From c3e57e7b51c47ac00a3cd11955ca54fa461a6b4a Mon Sep 17 00:00:00 2001 From: Miles Budnek Date: Mon, 12 Feb 2024 12:25:14 -0500 Subject: [PATCH] feat(manager/pip-compile): Add support for non-default package repos to the pip-compile manager (#26853) Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- .../usage/getting-started/private-packages.md | 28 ++++++ .../manager/pip-compile/artifacts.spec.ts | 28 ++++++ lib/modules/manager/pip-compile/artifacts.ts | 23 ++++- .../manager/pip-compile/common.spec.ts | 99 ++++++++++++++++++- lib/modules/manager/pip-compile/common.ts | 77 ++++++++++++++- lib/modules/manager/pip-compile/types.ts | 8 ++ 6 files changed, 256 insertions(+), 7 deletions(-) diff --git a/docs/usage/getting-started/private-packages.md b/docs/usage/getting-started/private-packages.md index e3090fb0f6..b481c94c07 100644 --- a/docs/usage/getting-started/private-packages.md +++ b/docs/usage/getting-started/private-packages.md @@ -461,6 +461,34 @@ verify_ssl = true name = "pypi" ``` +### pip-compile + +The pip-compile manager extracts `--index-url` and `--extra-index-url` directives from its input file. +Renovate will match those URLs with credentials from matching `hostRules` blocks in its configuration and pass appropriate `PIP_INDEX_URL` or `PIP_EXTRA_INDEX_URL` environment variables `pip-compile` with those credentials while generating the output file. + +```title="requirements.in" +--extra-index-url https://pypi.my.domain/simple + +private-package==1.2.3 +``` + +```json +{ + "pip-compile": { + "fileMatch": ["requirements.in"] + }, + "hostRules": [ + { + "matchHost": "pypi.my.domain", + "username": "myuser", + "password": "mypassword" + } + ] +} +``` + +Note: When credentials are passed to `pip-compile` this way Renovate will also pass the `--no-emit-index-url` flag to avoid leaking plain-text credentials to the output file. + ### poetry For every Poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`, where `X` represents the normalized name of the source in `pyproject.toml`. diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 14a92a54a4..4811afd5af 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -279,16 +279,42 @@ describe('modules/manager/pip-compile/artifacts', () => { Fixtures.get('requirementsNoHeaders.txt'), 'subdir/requirements.in', 'subdir/requirements.txt', + false, ), ).toBe('pip-compile requirements.in'); }); + it('returns --no-emit-index-url when credentials are present in URLs', () => { + expect( + constructPipCompileCmd( + Fixtures.get('requirementsNoHeaders.txt'), + 'subdir/requirements.in', + 'subdir/requirements.txt', + true, + ), + ).toBe('pip-compile --no-emit-index-url requirements.in'); + }); + it('returns extracted common arguments (like those featured in the README)', () => { expect( constructPipCompileCmd( Fixtures.get('requirementsWithHashes.txt'), 'subdir/requirements.in', 'subdir/requirements.txt', + false, + ), + ).toBe( + 'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in', + ); + }); + + it('returns --no-emit-index-url only once when its in the header and credentials are present in URLs', () => { + expect( + constructPipCompileCmd( + Fixtures.get('requirementsWithHashes.txt'), + 'subdir/requirements.in', + 'subdir/requirements.txt', + true, ), ).toBe( 'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in', @@ -301,6 +327,7 @@ describe('modules/manager/pip-compile/artifacts', () => { Fixtures.get('requirementsWithUnknownArguments.txt'), 'subdir/requirements.in', 'subdir/requirements.txt', + false, ), ).toBe('pip-compile --generate-hashes requirements.in'); expect(logger.trace).toHaveBeenCalledWith( @@ -319,6 +346,7 @@ describe('modules/manager/pip-compile/artifacts', () => { Fixtures.get('requirementsWithExploitingArguments.txt'), 'subdir/requirements.in', 'subdir/requirements.txt', + false, ), ).toBe( 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in', diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts index 9e3eeaf645..31633e75fc 100644 --- a/lib/modules/manager/pip-compile/artifacts.ts +++ b/lib/modules/manager/pip-compile/artifacts.ts @@ -10,23 +10,30 @@ import { } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; import { regEx } from '../../../util/regex'; +import * as pipRequirements from '../pip_requirements'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; import { constraintLineRegex, deprecatedAllowedPipArguments, getExecOptions, + getRegistryUrlVarsFromPackageFile, } from './common'; export function constructPipCompileCmd( content: string, inputFileName: string, outputFileName: string, + haveCredentials: boolean, ): string { const headers = constraintLineRegex.exec(content); const args = ['pip-compile']; - if (headers?.groups) { - logger.debug(`Found pip-compile header: ${headers[0]}`); - for (const argument of split(headers.groups.arguments)) { + if (!!headers?.groups || haveCredentials) { + logger.debug(`Found pip-compile header: ${headers?.[0]}`); + const headerArguments = split(headers?.groups?.arguments ?? ''); + if (haveCredentials && !headerArguments.includes('--no-emit-index-url')) { + headerArguments.push('--no-emit-index-url'); + } + for (const argument of headerArguments) { if (deprecatedAllowedPipArguments.includes(argument)) { args.push(argument); } else if (argument.startsWith('--output-file=')) { @@ -93,13 +100,21 @@ export async function updateArtifacts({ if (config.isLockFileMaintenance) { await deleteLocalFile(outputFileName); } + const packageFile = pipRequirements.extractPackageFile(newInputContent); + const registryUrlVars = getRegistryUrlVarsFromPackageFile(packageFile); const cmd = constructPipCompileCmd( existingOutput, inputFileName, outputFileName, + registryUrlVars.haveCredentials, + ); + const execOptions = await getExecOptions( + config, + inputFileName, + registryUrlVars.environmentVars, ); - const execOptions = await getExecOptions(config, inputFileName); logger.trace({ cmd }, 'pip-compile command'); + logger.trace({ env: execOptions.extraEnv }, 'pip-compile extra env vars'); await exec(cmd, execOptions); const status = await getRepoStatus(); if (!status?.modified.includes(outputFileName)) { diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts index fb9f1e5937..1a1f31e1d0 100644 --- a/lib/modules/manager/pip-compile/common.spec.ts +++ b/lib/modules/manager/pip-compile/common.spec.ts @@ -1,4 +1,12 @@ -import { allowedPipOptions, extractHeaderCommand } from './common'; +import { mockDeep } from 'jest-mock-extended'; +import { hostRules } from '../../../../test/util'; +import { + allowedPipOptions, + extractHeaderCommand, + getRegistryUrlVarsFromPackageFile, +} from './common'; + +jest.mock('../../../util/host-rules', () => mockDeep()); function getCommandInHeader(command: string) { return `# @@ -141,4 +149,93 @@ describe('modules/manager/pip-compile/common', () => { ).toHaveProperty('isCustomCommand', true); }); }); + + describe('getRegistryUrlFlagsFromPackageFile()', () => { + it('handles both registryUrls and additionalRegistryUrls', () => { + hostRules.find.mockReturnValue({}); + expect( + getRegistryUrlVarsFromPackageFile({ + deps: [], + registryUrls: ['https://example.com/pypi/simple'], + additionalRegistryUrls: ['https://example2.com/pypi/simple'], + }), + ).toEqual({ + haveCredentials: false, + environmentVars: { + PIP_INDEX_URL: 'https://example.com/pypi/simple', + PIP_EXTRA_INDEX_URL: 'https://example2.com/pypi/simple', + }, + }); + }); + + it('handles multiple additionalRegistryUrls', () => { + hostRules.find.mockReturnValue({}); + expect( + getRegistryUrlVarsFromPackageFile({ + deps: [], + additionalRegistryUrls: [ + 'https://example.com/pypi/simple', + 'https://example2.com/pypi/simple', + ], + }), + ).toEqual({ + haveCredentials: false, + environmentVars: { + PIP_EXTRA_INDEX_URL: + 'https://example.com/pypi/simple https://example2.com/pypi/simple', + }, + }); + }); + + it('uses extra index URLs with no auth', () => { + hostRules.find.mockReturnValue({}); + expect( + getRegistryUrlVarsFromPackageFile({ + deps: [], + registryUrls: ['https://example.com/pypi/simple'], + }), + ).toEqual({ + haveCredentials: false, + environmentVars: { + PIP_INDEX_URL: 'https://example.com/pypi/simple', + }, + }); + }); + + it('uses auth from extra index URLs matching host rules', () => { + hostRules.find.mockReturnValue({ + username: 'user', + password: 'password', + }); + expect( + getRegistryUrlVarsFromPackageFile({ + deps: [], + registryUrls: ['https://example.com/pypi/simple'], + }), + ).toEqual({ + haveCredentials: true, + environmentVars: { + PIP_INDEX_URL: 'https://user:password@example.com/pypi/simple', + }, + }); + }); + + it('handles invalid URLs', () => { + hostRules.find.mockReturnValue({}); + expect( + getRegistryUrlVarsFromPackageFile({ + deps: [], + additionalRegistryUrls: [ + 'https://example.com/pypi/simple', + 'this is not a valid URL', + ], + }), + ).toEqual({ + haveCredentials: false, + environmentVars: { + PIP_EXTRA_INDEX_URL: 'https://example.com/pypi/simple', + }, + }); + }); + }); }); diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts index 00bdb8c365..3279f14730 100644 --- a/lib/modules/manager/pip-compile/common.ts +++ b/lib/modules/manager/pip-compile/common.ts @@ -1,11 +1,13 @@ import is from '@sindresorhus/is'; import { split } from 'shlex'; import { logger } from '../../../logger'; +import { isNotNullOrUndefined } from '../../../util/array'; import type { ExecOptions } from '../../../util/exec/types'; import { ensureCacheDir } from '../../../util/fs'; +import * as hostRules from '../../../util/host-rules'; import { regEx } from '../../../util/regex'; -import type { UpdateArtifactsConfig } from '../types'; -import type { PipCompileArgs } from './types'; +import type { PackageFileContent, UpdateArtifactsConfig } from '../types'; +import type { GetRegistryUrlVarsResult, PipCompileArgs } from './types'; export function getPythonConstraint( config: UpdateArtifactsConfig, @@ -35,6 +37,7 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string { export async function getExecOptions( config: UpdateArtifactsConfig, inputFileName: string, + extraEnv: Record, ): Promise { const constraint = getPythonConstraint(config); const pipToolsConstraint = getPipToolsConstraint(config); @@ -53,6 +56,7 @@ export async function getExecOptions( ], extraEnv: { PIP_CACHE_DIR: await ensureCacheDir('pip'), + ...extraEnv, }, }; return execOptions; @@ -217,3 +221,72 @@ function throwForUnknownOption(arg: string): void { } throw new Error(`Option ${arg} not supported (yet)`); } + +function buildRegistryUrl(url: string): URL | null { + try { + const ret = new URL(url); + const hostRule = hostRules.find({ url }); + if (!ret.username && !ret.password) { + ret.username = hostRule.username ?? ''; + ret.password = hostRule.password ?? ''; + } + return ret; + } catch { + return null; + } +} + +function getRegistryUrlVarFromUrls( + varName: keyof GetRegistryUrlVarsResult['environmentVars'], + urls: URL[], +): GetRegistryUrlVarsResult { + if (!urls.length) { + return { + haveCredentials: false, + environmentVars: {}, + }; + } + + let haveCredentials = false; + for (const url of urls) { + if (url.username || url.password) { + haveCredentials = true; + } + } + const registryUrlsString = urls.map((url) => url.href).join(' '); + const ret: GetRegistryUrlVarsResult = { + haveCredentials, + environmentVars: {}, + }; + if (registryUrlsString) { + ret.environmentVars[varName] = registryUrlsString; + } + return ret; +} + +export function getRegistryUrlVarsFromPackageFile( + packageFile: PackageFileContent | null, +): GetRegistryUrlVarsResult { + // There should only ever be one element in registryUrls, since pip_requirements gets them from --index-url + // flags in the input file, and that only makes sense once + const indexUrl = getRegistryUrlVarFromUrls( + 'PIP_INDEX_URL', + packageFile?.registryUrls + ?.map(buildRegistryUrl) + .filter(isNotNullOrUndefined) ?? [], + ); + const extraIndexUrls = getRegistryUrlVarFromUrls( + 'PIP_EXTRA_INDEX_URL', + packageFile?.additionalRegistryUrls + ?.map(buildRegistryUrl) + .filter(isNotNullOrUndefined) ?? [], + ); + + return { + haveCredentials: indexUrl.haveCredentials || extraIndexUrls.haveCredentials, + environmentVars: { + ...indexUrl.environmentVars, + ...extraIndexUrls.environmentVars, + }, + }; +} diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts index c9e29b55c1..964c1cc9b6 100644 --- a/lib/modules/manager/pip-compile/types.ts +++ b/lib/modules/manager/pip-compile/types.ts @@ -1,3 +1,11 @@ +export interface GetRegistryUrlVarsResult { + haveCredentials: boolean; + environmentVars: { + PIP_INDEX_URL?: string; + PIP_EXTRA_INDEX_URL?: string; + }; +} + // managers supported by pip-tools Python package export type SupportedManagers = | 'pip_requirements'