feat(manager/pip-compile): Add support for non-default package repos to the pip-compile manager (#26853)

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

docs/usage/getting-started/private-packages.md

@@ -461,6 +461,34 @@ verify_ssl = true
 name = "pypi"
 ```
 
+### pip-compile
+
+The pip-compile manager extracts `--index-url` and `--extra-index-url` directives from its input file.
+Renovate will match those URLs with credentials from matching `hostRules` blocks in its configuration and pass appropriate `PIP_INDEX_URL` or `PIP_EXTRA_INDEX_URL` environment variables `pip-compile` with those credentials while generating the output file.
+
+```title="requirements.in"
+--extra-index-url https://pypi.my.domain/simple
+
+private-package==1.2.3
+```
+
+```json
+{
+  "pip-compile": {
+    "fileMatch": ["requirements.in"]
+  },
+  "hostRules": [
+    {
+      "matchHost": "pypi.my.domain",
+      "username": "myuser",
+      "password": "mypassword"
+    }
+  ]
+}
+```
+
+Note: When credentials are passed to `pip-compile` this way Renovate will also pass the `--no-emit-index-url` flag to avoid leaking plain-text credentials to the output file.
+
 ### poetry
 
 For every Poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`, where `X` represents the normalized name of the source in `pyproject.toml`.

lib/modules/manager/pip-compile/artifacts.spec.ts

@@ -279,16 +279,42 @@ describe('modules/manager/pip-compile/artifacts', () => {
           Fixtures.get('requirementsNoHeaders.txt'),
           'subdir/requirements.in',
           'subdir/requirements.txt',
+          false,
         ),
       ).toBe('pip-compile requirements.in');
     });
 
+    it('returns --no-emit-index-url when credentials are present in URLs', () => {
+      expect(
+        constructPipCompileCmd(
+          Fixtures.get('requirementsNoHeaders.txt'),
+          'subdir/requirements.in',
+          'subdir/requirements.txt',
+          true,
+        ),
+      ).toBe('pip-compile --no-emit-index-url requirements.in');
+    });
+
     it('returns extracted common arguments (like those featured in the README)', () => {
       expect(
         constructPipCompileCmd(
           Fixtures.get('requirementsWithHashes.txt'),
           'subdir/requirements.in',
           'subdir/requirements.txt',
+          false,
+        ),
+      ).toBe(
+        'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in',
+      );
+    });
+
+    it('returns --no-emit-index-url only once when its in the header and credentials are present in URLs', () => {
+      expect(
+        constructPipCompileCmd(
+          Fixtures.get('requirementsWithHashes.txt'),
+          'subdir/requirements.in',
+          'subdir/requirements.txt',
+          true,
         ),
       ).toBe(
         'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in',
@@ -301,6 +327,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
           Fixtures.get('requirementsWithUnknownArguments.txt'),
           'subdir/requirements.in',
           'subdir/requirements.txt',
+          false,
         ),
       ).toBe('pip-compile --generate-hashes requirements.in');
       expect(logger.trace).toHaveBeenCalledWith(
@@ -319,6 +346,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
           Fixtures.get('requirementsWithExploitingArguments.txt'),
           'subdir/requirements.in',
           'subdir/requirements.txt',
+          false,
         ),
       ).toBe(
         'pip-compile --generate-hashes --output-file=requirements.txt requirements.in',

lib/modules/manager/pip-compile/artifacts.ts

@@ -10,23 +10,30 @@ import {
 } from '../../../util/fs';
 import { getRepoStatus } from '../../../util/git';
 import { regEx } from '../../../util/regex';
+import * as pipRequirements from '../pip_requirements';
 import type { UpdateArtifact, UpdateArtifactsResult } from '../types';
 import {
   constraintLineRegex,
   deprecatedAllowedPipArguments,
   getExecOptions,
+  getRegistryUrlVarsFromPackageFile,
 } from './common';
 
 export function constructPipCompileCmd(
   content: string,
   inputFileName: string,
   outputFileName: string,
+  haveCredentials: boolean,
 ): string {
   const headers = constraintLineRegex.exec(content);
   const args = ['pip-compile'];
-  if (headers?.groups) {
+  if (!!headers?.groups || haveCredentials) {
-    logger.debug(`Found pip-compile header: ${headers[0]}`);
+    logger.debug(`Found pip-compile header: ${headers?.[0]}`);
-    for (const argument of split(headers.groups.arguments)) {
+    const headerArguments = split(headers?.groups?.arguments ?? '');
+    if (haveCredentials && !headerArguments.includes('--no-emit-index-url')) {
+      headerArguments.push('--no-emit-index-url');
+    }
+    for (const argument of headerArguments) {
       if (deprecatedAllowedPipArguments.includes(argument)) {
         args.push(argument);
       } else if (argument.startsWith('--output-file=')) {
@@ -93,13 +100,21 @@ export async function updateArtifacts({
       if (config.isLockFileMaintenance) {
         await deleteLocalFile(outputFileName);
       }
+      const packageFile = pipRequirements.extractPackageFile(newInputContent);
+      const registryUrlVars = getRegistryUrlVarsFromPackageFile(packageFile);
       const cmd = constructPipCompileCmd(
         existingOutput,
         inputFileName,
         outputFileName,
+        registryUrlVars.haveCredentials,
+      );
+      const execOptions = await getExecOptions(
+        config,
+        inputFileName,
+        registryUrlVars.environmentVars,
       );
-      const execOptions = await getExecOptions(config, inputFileName);
       logger.trace({ cmd }, 'pip-compile command');
+      logger.trace({ env: execOptions.extraEnv }, 'pip-compile extra env vars');
       await exec(cmd, execOptions);
       const status = await getRepoStatus();
       if (!status?.modified.includes(outputFileName)) {

lib/modules/manager/pip-compile/common.spec.ts

@@ -1,4 +1,12 @@
-import { allowedPipOptions, extractHeaderCommand } from './common';
+import { mockDeep } from 'jest-mock-extended';
+import { hostRules } from '../../../../test/util';
+import {
+  allowedPipOptions,
+  extractHeaderCommand,
+  getRegistryUrlVarsFromPackageFile,
+} from './common';
+
+jest.mock('../../../util/host-rules', () => mockDeep());
 
 function getCommandInHeader(command: string) {
   return `#
@@ -141,4 +149,93 @@ describe('modules/manager/pip-compile/common', () => {
       ).toHaveProperty('isCustomCommand', true);
     });
   });
+
+  describe('getRegistryUrlFlagsFromPackageFile()', () => {
+    it('handles both registryUrls and additionalRegistryUrls', () => {
+      hostRules.find.mockReturnValue({});
+      expect(
+        getRegistryUrlVarsFromPackageFile({
+          deps: [],
+          registryUrls: ['https://example.com/pypi/simple'],
+          additionalRegistryUrls: ['https://example2.com/pypi/simple'],
+        }),
+      ).toEqual({
+        haveCredentials: false,
+        environmentVars: {
+          PIP_INDEX_URL: 'https://example.com/pypi/simple',
+          PIP_EXTRA_INDEX_URL: 'https://example2.com/pypi/simple',
+        },
+      });
+    });
+
+    it('handles multiple additionalRegistryUrls', () => {
+      hostRules.find.mockReturnValue({});
+      expect(
+        getRegistryUrlVarsFromPackageFile({
+          deps: [],
+          additionalRegistryUrls: [
+            'https://example.com/pypi/simple',
+            'https://example2.com/pypi/simple',
+          ],
+        }),
+      ).toEqual({
+        haveCredentials: false,
+        environmentVars: {
+          PIP_EXTRA_INDEX_URL:
+            'https://example.com/pypi/simple https://example2.com/pypi/simple',
+        },
+      });
+    });
+
+    it('uses extra index URLs with no auth', () => {
+      hostRules.find.mockReturnValue({});
+      expect(
+        getRegistryUrlVarsFromPackageFile({
+          deps: [],
+          registryUrls: ['https://example.com/pypi/simple'],
+        }),
+      ).toEqual({
+        haveCredentials: false,
+        environmentVars: {
+          PIP_INDEX_URL: 'https://example.com/pypi/simple',
+        },
+      });
+    });
+
+    it('uses auth from extra index URLs matching host rules', () => {
+      hostRules.find.mockReturnValue({
+        username: 'user',
+        password: 'password',
+      });
+      expect(
+        getRegistryUrlVarsFromPackageFile({
+          deps: [],
+          registryUrls: ['https://example.com/pypi/simple'],
+        }),
+      ).toEqual({
+        haveCredentials: true,
+        environmentVars: {
+          PIP_INDEX_URL: 'https://user:password@example.com/pypi/simple',
+        },
+      });
+    });
+
+    it('handles invalid URLs', () => {
+      hostRules.find.mockReturnValue({});
+      expect(
+        getRegistryUrlVarsFromPackageFile({
+          deps: [],
+          additionalRegistryUrls: [
+            'https://example.com/pypi/simple',
+            'this is not a valid URL',
+          ],
+        }),
+      ).toEqual({
+        haveCredentials: false,
+        environmentVars: {
+          PIP_EXTRA_INDEX_URL: 'https://example.com/pypi/simple',
+        },
+      });
+    });
+  });
 });

lib/modules/manager/pip-compile/common.ts

@@ -1,11 +1,13 @@
 import is from '@sindresorhus/is';
 import { split } from 'shlex';
 import { logger } from '../../../logger';
+import { isNotNullOrUndefined } from '../../../util/array';
 import type { ExecOptions } from '../../../util/exec/types';
 import { ensureCacheDir } from '../../../util/fs';
+import * as hostRules from '../../../util/host-rules';
 import { regEx } from '../../../util/regex';
-import type { UpdateArtifactsConfig } from '../types';
+import type { PackageFileContent, UpdateArtifactsConfig } from '../types';
-import type { PipCompileArgs } from './types';
+import type { GetRegistryUrlVarsResult, PipCompileArgs } from './types';
 
 export function getPythonConstraint(
   config: UpdateArtifactsConfig,
@@ -35,6 +37,7 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string {
 export async function getExecOptions(
   config: UpdateArtifactsConfig,
   inputFileName: string,
+  extraEnv: Record<string, string>,
 ): Promise<ExecOptions> {
   const constraint = getPythonConstraint(config);
   const pipToolsConstraint = getPipToolsConstraint(config);
@@ -53,6 +56,7 @@ export async function getExecOptions(
     ],
     extraEnv: {
       PIP_CACHE_DIR: await ensureCacheDir('pip'),
+      ...extraEnv,
     },
   };
   return execOptions;
@@ -217,3 +221,72 @@ function throwForUnknownOption(arg: string): void {
   }
   throw new Error(`Option ${arg} not supported (yet)`);
 }
+
+function buildRegistryUrl(url: string): URL | null {
+  try {
+    const ret = new URL(url);
+    const hostRule = hostRules.find({ url });
+    if (!ret.username && !ret.password) {
+      ret.username = hostRule.username ?? '';
+      ret.password = hostRule.password ?? '';
+    }
+    return ret;
+  } catch {
+    return null;
+  }
+}
+
+function getRegistryUrlVarFromUrls(
+  varName: keyof GetRegistryUrlVarsResult['environmentVars'],
+  urls: URL[],
+): GetRegistryUrlVarsResult {
+  if (!urls.length) {
+    return {
+      haveCredentials: false,
+      environmentVars: {},
+    };
+  }
+
+  let haveCredentials = false;
+  for (const url of urls) {
+    if (url.username || url.password) {
+      haveCredentials = true;
+    }
+  }
+  const registryUrlsString = urls.map((url) => url.href).join(' ');
+  const ret: GetRegistryUrlVarsResult = {
+    haveCredentials,
+    environmentVars: {},
+  };
+  if (registryUrlsString) {
+    ret.environmentVars[varName] = registryUrlsString;
+  }
+  return ret;
+}
+
+export function getRegistryUrlVarsFromPackageFile(
+  packageFile: PackageFileContent | null,
+): GetRegistryUrlVarsResult {
+  // There should only ever be one element in registryUrls, since pip_requirements gets them from --index-url
+  // flags in the input file, and that only makes sense once
+  const indexUrl = getRegistryUrlVarFromUrls(
+    'PIP_INDEX_URL',
+    packageFile?.registryUrls
+      ?.map(buildRegistryUrl)
+      .filter(isNotNullOrUndefined) ?? [],
+  );
+  const extraIndexUrls = getRegistryUrlVarFromUrls(
+    'PIP_EXTRA_INDEX_URL',
+    packageFile?.additionalRegistryUrls
+      ?.map(buildRegistryUrl)
+      .filter(isNotNullOrUndefined) ?? [],
+  );
+
+  return {
+    haveCredentials: indexUrl.haveCredentials || extraIndexUrls.haveCredentials,
+    environmentVars: {
+      ...indexUrl.environmentVars,
+      ...extraIndexUrls.environmentVars,
+    },
+  };
+}

lib/modules/manager/pip-compile/types.ts

@@ -1,3 +1,11 @@
+export interface GetRegistryUrlVarsResult {
+  haveCredentials: boolean;
+  environmentVars: {
+    PIP_INDEX_URL?: string;
+    PIP_EXTRA_INDEX_URL?: string;
+  };
+}
+
 // managers supported by pip-tools Python package
 export type SupportedManagers =
   | 'pip_requirements'