feat(manager/copier): Implement manager (#29215)

docs/usage/configuration-options.md

@@ -2101,7 +2101,7 @@ In the case that a user is automatically added as reviewer (such as Renovate App
 
 ## ignoreScripts
 
-Applicable for npm and Composer only for now. Set this to `true` if running scripts causes problems.
+Applicable for npm, Composer and Copier only for now. Set this to `true` if running scripts causes problems.
 
 ## ignoreTests
 

lib/config/options/index.ts

@@ -888,7 +888,7 @@ const options: RenovateOptions[] = [
       'Set this to `false` if `allowScripts=true` and you wish to run scripts when updating lock files.',
     type: 'boolean',
     default: true,
-    supportedManagers: ['npm', 'composer'],
+    supportedManagers: ['npm', 'composer', 'copier'],
   },
   {
     name: 'platform',

lib/modules/manager/api.ts

@@ -22,6 +22,7 @@ import * as cloudbuild from './cloudbuild';
 import * as cocoapods from './cocoapods';
 import * as composer from './composer';
 import * as conan from './conan';
+import * as copier from './copier';
 import * as cpanfile from './cpanfile';
 import * as crossplane from './crossplane';
 import * as depsEdn from './deps-edn';
@@ -122,6 +123,7 @@ api.set('cloudbuild', cloudbuild);
 api.set('cocoapods', cocoapods);
 api.set('composer', composer);
 api.set('conan', conan);
+api.set('copier', copier);
 api.set('cpanfile', cpanfile);
 api.set('crossplane', crossplane);
 api.set('deps-edn', depsEdn);

lib/modules/manager/copier/artifacts.spec.ts

@@ -0,0 +1,387 @@
+import { mockDeep } from 'jest-mock-extended';
+import { join } from 'upath';
+import { mockExecAll } from '../../../../test/exec-util';
+import { fs, git, mocked, partial } from '../../../../test/util';
+import { GlobalConfig } from '../../../config/global';
+import type { RepoGlobalConfig } from '../../../config/types';
+import { logger } from '../../../logger';
+import type { StatusResult } from '../../../util/git/types';
+import * as _datasource from '../../datasource';
+import type { UpdateArtifactsConfig, Upgrade } from '../types';
+import { updateArtifacts } from '.';
+
+const datasource = mocked(_datasource);
+
+jest.mock('../../../util/git');
+jest.mock('../../../util/fs');
+jest.mock('../../datasource', () => mockDeep());
+
+process.env.CONTAINERBASE = 'true';
+
+const config: UpdateArtifactsConfig = {
+  ignoreScripts: true,
+};
+
+const upgrades: Upgrade[] = [
+  {
+    depName: 'https://github.com/foo/bar',
+    currentValue: '1.0.0',
+    newValue: '1.1.0',
+  },
+];
+
+const adminConfig: RepoGlobalConfig = {
+  localDir: join('/tmp/github/some/repo'),
+  cacheDir: join('/tmp/cache'),
+  containerbaseDir: join('/tmp/renovate/cache/containerbase'),
+  allowScripts: false,
+};
+
+describe('modules/manager/copier/artifacts', () => {
+  beforeEach(() => {
+    GlobalConfig.set(adminConfig);
+
+    // Mock git repo status
+    git.getRepoStatus.mockResolvedValue(
+      partial<StatusResult>({
+        conflicted: [],
+        modified: ['.copier-answers.yml'],
+        not_added: [],
+        deleted: [],
+        renamed: [],
+      }),
+    );
+  });
+
+  afterEach(() => {
+    fs.readLocalFile.mockClear();
+    git.getRepoStatus.mockClear();
+  });
+
+  describe('updateArtifacts()', () => {
+    it('returns null if newVersion is not provided', async () => {
+      const execSnapshots = mockExecAll();
+
+      const invalidUpgrade = [
+        { ...upgrades[0], newValue: undefined, newVersion: undefined },
+      ];
+
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: invalidUpgrade,
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(result).toEqual([
+        {
+          artifactError: {
+            lockFile: '.copier-answers.yml',
+            stderr: 'Missing copier template version to update to',
+          },
+        },
+      ]);
+      expect(execSnapshots).toEqual([]);
+    });
+
+    it('reports an error if no upgrade is specified', async () => {
+      const execSnapshots = mockExecAll();
+
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: [],
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(result).toEqual([
+        {
+          artifactError: {
+            lockFile: '.copier-answers.yml',
+            stderr: 'Unexpected number of dependencies: 0 (should be 1)',
+          },
+        },
+      ]);
+      expect(execSnapshots).toEqual([]);
+    });
+
+    it('invokes copier update with the correct options by default', async () => {
+      const execSnapshots = mockExecAll();
+
+      await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config: {},
+      });
+
+      expect(execSnapshots).toMatchObject([
+        {
+          cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0',
+        },
+      ]);
+    });
+
+    it.each`
+      pythonConstraint | copierConstraint
+      ${null}          | ${null}
+      ${'3.11.3'}      | ${null}
+      ${null}          | ${'9.1.0'}
+      ${'3.11.3'}      | ${'9.1.0'}
+    `(
+      `supports dynamic install with constraints python=$pythonConstraint copier=$copierConstraint`,
+      async ({ pythonConstraint, copierConstraint }) => {
+        GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+        const constraintConfig = {
+          python: pythonConstraint ?? '',
+          copier: copierConstraint ?? '',
+        };
+        if (!pythonConstraint) {
+          datasource.getPkgReleases.mockResolvedValueOnce({
+            releases: [{ version: '3.12.4' }],
+          });
+        }
+        if (!copierConstraint) {
+          datasource.getPkgReleases.mockResolvedValueOnce({
+            releases: [{ version: '9.2.0' }],
+          });
+        }
+        const execSnapshots = mockExecAll();
+
+        expect(
+          await updateArtifacts({
+            packageFileName: '.copier-answers.yml',
+            updatedDeps: upgrades,
+            newPackageFileContent: '',
+            config: {
+              ...config,
+              constraints: constraintConfig,
+            },
+          }),
+        ).not.toBeNull();
+
+        expect(execSnapshots).toMatchObject([
+          { cmd: `install-tool python ${pythonConstraint ?? '3.12.4'}` },
+          { cmd: `install-tool copier ${copierConstraint ?? '9.2.0'}` },
+          {
+            cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0',
+          },
+        ]);
+      },
+    );
+
+    it('includes --trust when allowScripts is true and ignoreScripts is false', async () => {
+      GlobalConfig.set({ ...adminConfig, allowScripts: true });
+      const execSnapshots = mockExecAll();
+
+      const trustConfig = {
+        ...config,
+        ignoreScripts: false,
+      };
+
+      await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config: trustConfig,
+      });
+
+      expect(execSnapshots).toMatchObject([
+        {
+          cmd: 'copier update --skip-answered --defaults --trust --answers-file .copier-answers.yml --vcs-ref 1.1.0',
+        },
+      ]);
+    });
+
+    it('does not include --trust when ignoreScripts is true', async () => {
+      GlobalConfig.set({ ...adminConfig, allowScripts: true });
+      const execSnapshots = mockExecAll();
+
+      await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(execSnapshots).toMatchObject([
+        {
+          cmd: 'copier update --skip-answered --defaults --answers-file .copier-answers.yml --vcs-ref 1.1.0',
+        },
+      ]);
+    });
+
+    it('handles exec errors', async () => {
+      mockExecAll(new Error('exec exception'));
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(result).toEqual([
+        {
+          artifactError: {
+            lockFile: '.copier-answers.yml',
+            stderr: 'exec exception',
+          },
+        },
+      ]);
+    });
+
+    it('does not report changes if answers-file was not changed', async () => {
+      mockExecAll();
+
+      git.getRepoStatus.mockResolvedValueOnce(
+        partial<StatusResult>({
+          conflicted: [],
+          modified: [],
+          not_added: ['new_file.py'],
+          deleted: ['old_file.py'],
+          renamed: [],
+        }),
+      );
+
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(result).toBeNull();
+    });
+
+    it('returns updated artifacts if repo status has changes', async () => {
+      mockExecAll();
+
+      git.getRepoStatus.mockResolvedValueOnce(
+        partial<StatusResult>({
+          conflicted: [],
+          modified: ['.copier-answers.yml'],
+          not_added: ['new_file.py'],
+          deleted: ['old_file.py'],
+          renamed: [{ from: 'renamed_old.py', to: 'renamed_new.py' }],
+        }),
+      );
+
+      fs.readLocalFile.mockResolvedValueOnce(
+        '_src: https://github.com/foo/bar\n_commit: 1.1.0',
+      );
+      fs.readLocalFile.mockResolvedValueOnce('new file contents');
+      fs.readLocalFile.mockResolvedValueOnce('renamed file contents');
+
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config,
+      });
+
+      expect(result).toEqual([
+        {
+          file: {
+            type: 'addition',
+            path: '.copier-answers.yml',
+            contents: '_src: https://github.com/foo/bar\n_commit: 1.1.0',
+          },
+        },
+        {
+          file: {
+            type: 'addition',
+            path: 'new_file.py',
+            contents: 'new file contents',
+          },
+        },
+        {
+          file: {
+            type: 'deletion',
+            path: 'old_file.py',
+          },
+        },
+        {
+          file: {
+            type: 'deletion',
+            path: 'renamed_old.py',
+          },
+        },
+        {
+          file: {
+            type: 'addition',
+            path: 'renamed_new.py',
+            contents: 'renamed file contents',
+          },
+        },
+      ]);
+    });
+
+    it('warns about, but adds conflicts', async () => {
+      mockExecAll();
+
+      git.getRepoStatus.mockResolvedValueOnce(
+        partial<StatusResult>({
+          conflicted: ['conflict_file.py'],
+          modified: ['.copier-answers.yml'],
+          not_added: ['new_file.py'],
+          deleted: ['old_file.py'],
+          renamed: [],
+        }),
+      );
+
+      fs.readLocalFile.mockResolvedValueOnce(
+        '_src: https://github.com/foo/bar\n_commit: 1.1.0',
+      );
+      fs.readLocalFile.mockResolvedValueOnce('new file contents');
+      fs.readLocalFile.mockResolvedValueOnce('conflict file contents');
+
+      const result = await updateArtifacts({
+        packageFileName: '.copier-answers.yml',
+        updatedDeps: upgrades,
+        newPackageFileContent: '',
+        config,
+      });
+      expect(logger.debug).toHaveBeenCalledWith(
+        {
+          depName: 'https://github.com/foo/bar',
+          packageFileName: '.copier-answers.yml',
+        },
+        'Updating the Copier template yielded 1 merge conflicts. Please check the proposed changes carefully! Conflicting files:\n  * conflict_file.py',
+      );
+      expect(result).toEqual([
+        {
+          file: {
+            type: 'addition',
+            path: '.copier-answers.yml',
+            contents: '_src: https://github.com/foo/bar\n_commit: 1.1.0',
+          },
+        },
+        {
+          file: {
+            type: 'addition',
+            path: 'new_file.py',
+            contents: 'new file contents',
+          },
+        },
+        {
+          file: {
+            type: 'addition',
+            path: 'conflict_file.py',
+            contents: 'conflict file contents',
+          },
+          notice: {
+            file: 'conflict_file.py',
+            message:
+              'This file had merge conflicts. Please check the proposed changes carefully!',
+          },
+        },
+        {
+          file: {
+            type: 'deletion',
+            path: 'old_file.py',
+          },
+        },
+      ]);
+    });
+  });
+});

lib/modules/manager/copier/artifacts.ts

@@ -0,0 +1,161 @@
+import { quote } from 'shlex';
+import { GlobalConfig } from '../../../config/global';
+import { logger } from '../../../logger';
+import { exec } from '../../../util/exec';
+import type { ExecOptions } from '../../../util/exec/types';
+import { readLocalFile } from '../../../util/fs';
+import { getRepoStatus } from '../../../util/git';
+import type {
+  UpdateArtifact,
+  UpdateArtifactsConfig,
+  UpdateArtifactsResult,
+} from '../types';
+import {
+  getCopierVersionConstraint,
+  getPythonVersionConstraint,
+} from './utils';
+
+const DEFAULT_COMMAND_OPTIONS = ['--skip-answered', '--defaults'];
+
+function buildCommand(
+  config: UpdateArtifactsConfig,
+  packageFileName: string,
+  newVersion: string,
+): string {
+  const command = ['copier', 'update', ...DEFAULT_COMMAND_OPTIONS];
+  if (GlobalConfig.get('allowScripts') && !config.ignoreScripts) {
+    command.push('--trust');
+  }
+  command.push(
+    '--answers-file',
+    quote(packageFileName),
+    '--vcs-ref',
+    quote(newVersion),
+  );
+  return command.join(' ');
+}
+
+function artifactError(
+  packageFileName: string,
+  message: string,
+): UpdateArtifactsResult[] {
+  return [
+    {
+      artifactError: {
+        lockFile: packageFileName,
+        stderr: message,
+      },
+    },
+  ];
+}
+
+export async function updateArtifacts({
+  packageFileName,
+  updatedDeps,
+  config,
+}: UpdateArtifact): Promise<UpdateArtifactsResult[] | null> {
+  if (!updatedDeps || updatedDeps.length !== 1) {
+    // Each answers file (~ packageFileName) has exactly one dependency to update.
+    return artifactError(
+      packageFileName,
+      `Unexpected number of dependencies: ${updatedDeps.length} (should be 1)`,
+    );
+  }
+
+  const newVersion = updatedDeps[0]?.newVersion ?? updatedDeps[0]?.newValue;
+  if (!newVersion) {
+    return artifactError(
+      packageFileName,
+      'Missing copier template version to update to',
+    );
+  }
+
+  const command = buildCommand(config, packageFileName, newVersion);
+  const execOptions: ExecOptions = {
+    docker: {},
+    userConfiguredEnv: config.env,
+    toolConstraints: [
+      {
+        toolName: 'python',
+        constraint: getPythonVersionConstraint(config),
+      },
+      {
+        toolName: 'copier',
+        constraint: getCopierVersionConstraint(config),
+      },
+    ],
+  };
+  try {
+    await exec(command, execOptions);
+  } catch (err) {
+    logger.debug({ err }, `Failed to update copier template: ${err.message}`);
+    return artifactError(packageFileName, err.message);
+  }
+
+  const status = await getRepoStatus();
+  // If the answers file didn't change, Copier did not update anything.
+  if (!status.modified.includes(packageFileName)) {
+    return null;
+  }
+
+  if (status.conflicted.length > 0) {
+    // Sometimes, Copier erroneously reports conflicts.
+    const msg =
+      `Updating the Copier template yielded ${status.conflicted.length} merge conflicts. ` +
+      'Please check the proposed changes carefully! Conflicting files:\n  * ' +
+      status.conflicted.join('\n  * ');
+    logger.debug({ packageFileName, depName: updatedDeps[0]?.depName }, msg);
+  }
+
+  const res: UpdateArtifactsResult[] = [];
+
+  for (const f of [
+    ...status.modified,
+    ...status.not_added,
+    ...status.conflicted,
+  ]) {
+    const fileRes: UpdateArtifactsResult = {
+      file: {
+        type: 'addition',
+        path: f,
+        contents: await readLocalFile(f),
+      },
+    };
+    if (status.conflicted.includes(f)) {
+      // Make the reviewer aware of the conflicts.
+      // This will be posted in a comment.
+      fileRes.notice = {
+        file: f,
+        message:
+          'This file had merge conflicts. Please check the proposed changes carefully!',
+      };
+    }
+    res.push(fileRes);
+  }
+  for (const f of status.deleted) {
+    res.push({
+      file: {
+        type: 'deletion',
+        path: f,
+      },
+    });
+  }
+  // `git status` might detect a rename, which is then not contained
+  // in not_added/deleted. Ensure we respect renames as well if they happen.
+  for (const f of status.renamed) {
+    res.push({
+      file: {
+        type: 'deletion',
+        path: f.from,
+      },
+    });
+    res.push({
+      file: {
+        type: 'addition',
+        path: f.to,
+        contents: await readLocalFile(f.to),
+      },
+    });
+  }
+  return res;
+}

lib/modules/manager/copier/extract.spec.ts

@@ -0,0 +1,58 @@
+import { extractPackageFile } from '.';
+
+describe('modules/manager/copier/extract', () => {
+  describe('extractPackageFile()', () => {
+    it('extracts repository and version from .copier-answers.yml', () => {
+      const content = `
+        _commit: v1.0.0
+        _src_path: https://github.com/username/template-repo
+      `;
+      const result = extractPackageFile(content);
+      expect(result).toEqual({
+        deps: [
+          {
+            depName: 'https://github.com/username/template-repo',
+            packageName: 'https://github.com/username/template-repo',
+            currentValue: 'v1.0.0',
+            datasource: 'git-tags',
+            depType: 'template',
+          },
+        ],
+      });
+    });
+
+    it('returns null for invalid .copier-answers.yml', () => {
+      const content = `
+        not_valid:
+          key: value
+      `;
+      const result = extractPackageFile(content);
+      expect(result).toBeNull();
+    });
+
+    it('returns null for invalid _src_path', () => {
+      const content = `
+        _commit: v1.0.0
+        _src_path: notaurl
+      `;
+      const result = extractPackageFile(content);
+      expect(result).toBeNull();
+    });
+
+    it('returns null for missing _commit field', () => {
+      const content = `
+        _src_path: https://github.com/username/template-repo
+      `;
+      const result = extractPackageFile(content);
+      expect(result).toBeNull();
+    });
+
+    it('returns null for missing _src_path field', () => {
+      const content = `
+        _commit: v1.0.0
+      `;
+      const result = extractPackageFile(content);
+      expect(result).toBeNull();
+    });
+  });
+});

lib/modules/manager/copier/extract.ts

@@ -0,0 +1,31 @@
+import { logger } from '../../../logger';
+import { GitTagsDatasource } from '../../datasource/git-tags';
+import type { PackageDependency, PackageFileContent } from '../types';
+import { CopierAnswersFile } from './schema';
+
+export function extractPackageFile(
+  content: string,
+  packageFile?: string,
+): PackageFileContent | null {
+  let parsed: CopierAnswersFile;
+  try {
+    parsed = CopierAnswersFile.parse(content);
+  } catch (err) {
+    logger.debug({ err, packageFile }, `Parsing Copier answers YAML failed`);
+    return null;
+  }
+
+  const deps: PackageDependency[] = [
+    {
+      datasource: GitTagsDatasource.id,
+      depName: parsed._src_path,
+      packageName: parsed._src_path,
+      depType: 'template',
+      currentValue: parsed._commit,
+    },
+  ];
+
+  return {
+    deps,
+  };
+}

lib/modules/manager/copier/index.ts

@@ -0,0 +1,12 @@
+import { GitTagsDatasource } from '../../datasource/git-tags';
+import * as pep440 from '../../versioning/pep440';
+export { updateArtifacts } from './artifacts';
+export { extractPackageFile } from './extract';
+export { updateDependency } from './update';
+
+export const defaultConfig = {
+  fileMatch: ['(^|/)\\.copier-answers(\\..+)?\\.ya?ml'],
+  versioning: pep440.id,
+};
+
+export const supportedDatasources = [GitTagsDatasource.id];

lib/modules/manager/copier/readme.md

@@ -0,0 +1,7 @@
+Keeps Copier templates up to date.
+Supports multiple `.copier-answers(...).y(a)ml` files in a single repository.
+If a template requires unsafe features, Copier must be invoked with the `--trust` flag.
+Enabling this behavior must be allowed in the [self-hosted configuration](../../../self-hosted-configuration.md) via `allowScripts`.
+Actually enable it in the [configuration](../../../configuration-options.md) by setting `ignoreScripts` to `false`.
+
+If you need to change the versioning format, read the [versioning](../../versioning/index.md) documentation to learn more.

lib/modules/manager/copier/schema.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod';
+import { Yaml } from '../../../util/schema-utils';
+
+export const CopierAnswersFile = Yaml.pipe(
+  z.object({
+    _commit: z.string(),
+    _src_path: z.string().url(),
+  }),
+);
+
+export type CopierAnswersFile = z.infer<typeof CopierAnswersFile>;

lib/modules/manager/copier/update.spec.ts

@@ -0,0 +1,25 @@
+import { codeBlock } from 'common-tags';
+import { updateDependency } from '.';
+
+describe('modules/manager/copier/update', () => {
+  describe('updateDependency', () => {
+    it('should append a new marking line at the end to trigger the artifact update', () => {
+      const fileContent = codeBlock`
+        _src_path: https://foo.bar/baz/quux
+        _commit: 1.0.0
+      `;
+      const ret = updateDependency({ fileContent, upgrade: {} });
+      expect(ret).toBe(`${fileContent}\n#copier updated`);
+    });
+
+    it('should not update again if the new line has been appended', () => {
+      const fileContent = codeBlock`
+        _src_path: https://foo.bar/baz/quux
+        _commit: 1.0.0
+        #copier updated
+      `;
+      const ret = updateDependency({ fileContent, upgrade: {} });
+      expect(ret).toBe(fileContent);
+    });
+  });
+});

lib/modules/manager/copier/update.ts

@@ -0,0 +1,22 @@
+import { logger } from '../../../logger';
+import type { UpdateDependencyConfig } from '../types';
+
+const updateLine = '#copier updated';
+
+/**
+ * updateDependency appends a comment line once.
+ * This is only for the purpose of triggering the artifact update.
+ * Copier needs to update its answers file itself.
+ */
+export function updateDependency({
+  fileContent,
+  upgrade,
+}: UpdateDependencyConfig): string | null {
+  logger.trace({ upgrade }, `copier.updateDependency()`);
+  if (!fileContent.endsWith(updateLine)) {
+    logger.debug(`append update line to the fileContent if it hasn't been`);
+    return `${fileContent}\n${updateLine}`;
+  }
+
+  return fileContent;
+}

lib/modules/manager/copier/utils.ts

@@ -0,0 +1,31 @@
+import is from '@sindresorhus/is';
+import { logger } from '../../../logger';
+import type { UpdateArtifactsConfig } from '../types';
+
+export function getPythonVersionConstraint(
+  config: UpdateArtifactsConfig,
+): string | undefined | null {
+  const { constraints = {} } = config;
+  const { python } = constraints;
+
+  if (is.nonEmptyString(python)) {
+    logger.debug('Using python constraint from config');
+    return python;
+  }
+
+  return undefined;
+}
+
+export function getCopierVersionConstraint(
+  config: UpdateArtifactsConfig,
+): string {
+  const { constraints = {} } = config;
+  const { copier } = constraints;
+
+  if (is.nonEmptyString(copier)) {
+    logger.debug('Using copier constraint from config');
+    return copier;
+  }
+
+  return '';
+}

lib/util/exec/containerbase.ts

@@ -38,6 +38,11 @@ const allToolConfig: Record<string, ToolConfig> = {
     packageName: 'composer/composer',
     versioning: composerVersioningId,
   },
+  copier: {
+    datasource: 'pypi',
+    packageName: 'copier',
+    versioning: pep440VersioningId,
+  },
   corepack: {
     datasource: 'npm',
     packageName: 'corepack',