renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-12 23:16:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(pip-compile): Provide credentials for registries in all input files (#28959)

Browse source
This commit is contained in:
Miles Budnek 2024-06-04 12:33:08 -04:00 committed by GitHub
parent 1f08846483
commit c27e0ecefb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 110 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/modules/manager/pip-compile/artifacts.spec.ts
View file

@ -86,6 +86,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns null if all unchanged', async () => { it('returns null if all unchanged', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll(); const execSnapshots = mockExecAll();
fs.readLocalFile.mockResolvedValueOnce('new lock'); fs.readLocalFile.mockResolvedValueOnce('new lock');
expect( expect(
@ -106,6 +107,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns null if no config.lockFiles', async () => { it('returns null if no config.lockFiles', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.readLocalFile.mockResolvedValueOnce('new lock'); fs.readLocalFile.mockResolvedValueOnce('new lock');
expect( expect(
await updateArtifacts({ await updateArtifacts({
@ -125,6 +127,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns updated requirements.txt', async () => { it('returns updated requirements.txt', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll(); const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue( git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({ partial<StatusResult>({
@ -162,6 +165,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
}), }),
); );
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip'); fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip');
expect( expect(
await updateArtifacts({ await updateArtifacts({
@ -215,6 +219,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
}), }),
); );
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
expect( expect(
await updateArtifacts({ await updateArtifacts({
packageFileName: 'requirements.in', packageFileName: 'requirements.in',
@ -328,6 +333,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('catches errors', async () => { it('catches errors', async () => {
const execSnapshots = mockExecAll(); const execSnapshots = mockExecAll();
fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt'); fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt');
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.writeLocalFile.mockImplementationOnce(() => { fs.writeLocalFile.mockImplementationOnce(() => {
throw new Error('not found'); throw new Error('not found');
}); });
@ -348,6 +354,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns updated requirements.txt when doing lockfile maintenance', async () => { it('returns updated requirements.txt when doing lockfile maintenance', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll(); const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue( git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({ partial<StatusResult>({
@ -370,6 +377,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('uses --upgrade-package only for isLockfileUpdate', async () => { it('uses --upgrade-package only for isLockfileUpdate', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll(); const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue( git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({ partial<StatusResult>({
@ -397,6 +405,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('uses pip-compile version from config', async () => { it('uses pip-compile version from config', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader); fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
GlobalConfig.set(dockerAdminConfig); GlobalConfig.set(dockerAdminConfig);
// pip-tools // pip-tools
datasource.getPkgReleases.mockResolvedValueOnce({ datasource.getPkgReleases.mockResolvedValueOnce({

30
lib/modules/manager/pip-compile/artifacts.ts
View file

@ -1,4 +1,5 @@
import { quote } from 'shlex'; import { quote } from 'shlex';
import upath from 'upath';
import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { TEMPORARY_ERROR } from '../../../constants/error-messages';
import { logger } from '../../../logger'; import { logger } from '../../../logger';
import { exec } from '../../../util/exec'; import { exec } from '../../../util/exec';
@ -8,13 +9,19 @@ import {
writeLocalFile, writeLocalFile,
} from '../../../util/fs'; } from '../../../util/fs';
import { getRepoStatus } from '../../../util/git'; import { getRepoStatus } from '../../../util/git';
import * as pipRequirements from '../pip_requirements'; import { extractPackageFileFlags as extractRequirementsFileFlags } from '../pip_requirements/common';
import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types'; import type {
PackageFileContent,
UpdateArtifact,
UpdateArtifactsResult,
Upgrade,
} from '../types';
import { import {
extractHeaderCommand, extractHeaderCommand,
extractPythonVersion, extractPythonVersion,
getExecOptions, getExecOptions,
getRegistryCredVarsFromPackageFile, getRegistryCredVarsFromPackageFiles,
matchManager,
} from './common'; } from './common';
import type { PipCompileArgs } from './types'; import type { PipCompileArgs } from './types';
import { inferCommandExecDir } from './utils'; import { inferCommandExecDir } from './utils';
@ -113,12 +120,25 @@ export async function updateArtifacts({
); );
const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile); const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile);
const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate); const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate);
const packageFile = pipRequirements.extractPackageFile(newInputContent); const packageFiles: PackageFileContent[] = [];
for (const name of compileArgs.sourceFiles) {
const manager = matchManager(name);
if (manager === 'pip_requirements') {
const path = upath.join(cwd, name);
const content = await readLocalFile(path, 'utf8');
if (content) {
const packageFile = extractRequirementsFileFlags(content);
if (packageFile) {
packageFiles.push(packageFile);
}
}
}
}
const cmd = constructPipCompileCmd(compileArgs, upgradePackages); const cmd = constructPipCompileCmd(compileArgs, upgradePackages);
const execOptions = await getExecOptions( const execOptions = await getExecOptions(
config, config,
cwd, cwd,
getRegistryCredVarsFromPackageFile(packageFile), getRegistryCredVarsFromPackageFiles(packageFiles),
pythonVersion, pythonVersion,
); );
logger.trace({ cwd, cmd }, 'pip-compile command'); logger.trace({ cwd, cmd }, 'pip-compile command');

92
lib/modules/manager/pip-compile/common.spec.ts
View file

@ -5,7 +5,7 @@ import {
allowedPipOptions, allowedPipOptions,
extractHeaderCommand, extractHeaderCommand,
extractPythonVersion, extractPythonVersion,
getRegistryCredVarsFromPackageFile, getRegistryCredVarsFromPackageFiles,
matchManager, matchManager,
} from './common'; } from './common';
import { inferCommandExecDir } from './utils'; import { inferCommandExecDir } from './utils';
@ -187,7 +187,7 @@ describe('modules/manager/pip-compile/common', () => {
}); });
}); });
describe('getCredentialVarsFromPackageFile()', () => { describe('getRegistryCredVarsFromPackageFiles()', () => {
it('handles both registryUrls and additionalRegistryUrls', () => { it('handles both registryUrls and additionalRegistryUrls', () => {
hostRules.find.mockReturnValueOnce({ hostRules.find.mockReturnValueOnce({
username: 'user1', username: 'user1',
@ -198,11 +198,13 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password2', password: 'password2',
}); });
expect( expect(
getRegistryCredVarsFromPackageFile({ getRegistryCredVarsFromPackageFiles([
deps: [], {
registryUrls: ['https://example.com/pypi/simple'], deps: [],
additionalRegistryUrls: ['https://example2.com/pypi/simple'], registryUrls: ['https://example.com/pypi/simple'],
}), additionalRegistryUrls: ['https://example2.com/pypi/simple'],
},
]),
).toEqual({ ).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1', KEYRING_SERVICE_USERNAME_0: 'user1',
@ -223,13 +225,15 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password2', password: 'password2',
}); });
expect( expect(
getRegistryCredVarsFromPackageFile({ getRegistryCredVarsFromPackageFiles([
deps: [], {
additionalRegistryUrls: [ deps: [],
'https://example.com/pypi/simple', additionalRegistryUrls: [
'https://example2.com/pypi/simple', 'https://example.com/pypi/simple',
], 'https://example2.com/pypi/simple',
}), ],
},
]),
).toEqual({ ).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1', KEYRING_SERVICE_USERNAME_0: 'user1',
@ -245,10 +249,12 @@ describe('modules/manager/pip-compile/common', () => {
username: 'user', username: 'user',
}); });
expect( expect(
getRegistryCredVarsFromPackageFile({ getRegistryCredVarsFromPackageFiles([
deps: [], {
additionalRegistryUrls: ['https://example.com/pypi/simple'], deps: [],
}), additionalRegistryUrls: ['https://example.com/pypi/simple'],
},
]),
).toEqual({ ).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user', KEYRING_SERVICE_USERNAME_0: 'user',
@ -261,10 +267,12 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password', password: 'password',
}); });
expect( expect(
getRegistryCredVarsFromPackageFile({ getRegistryCredVarsFromPackageFiles([
deps: [], {
additionalRegistryUrls: ['https://example.com/pypi/simple'], deps: [],
}), additionalRegistryUrls: ['https://example.com/pypi/simple'],
},
]),
).toEqual({ ).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com', KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: '', KEYRING_SERVICE_USERNAME_0: '',
@ -277,14 +285,46 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password', password: 'password',
}); });
expect( expect(
getRegistryCredVarsFromPackageFile({ getRegistryCredVarsFromPackageFiles([
deps: [], {
additionalRegistryUrls: ['invalid-url'], deps: [],
}), additionalRegistryUrls: ['invalid-url'],
},
]),
).toEqual({}); ).toEqual({});
}); });
}); });
it('handles multiple package files', () => {
hostRules.find.mockReturnValueOnce({
username: 'user1',
password: 'password1',
});
hostRules.find.mockReturnValueOnce({
username: 'user2',
password: 'password2',
});
expect(
getRegistryCredVarsFromPackageFiles([
{
deps: [],
registryUrls: ['https://example.com/pypi/simple'],
},
{
deps: [],
additionalRegistryUrls: ['https://example2.com/pypi/simple'],
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1',
KEYRING_SERVICE_PASSWORD_0: 'password1',
KEYRING_SERVICE_NAME_1: 'example2.com',
KEYRING_SERVICE_USERNAME_1: 'user2',
KEYRING_SERVICE_PASSWORD_1: 'password2',
});
});
describe('matchManager()', () => { describe('matchManager()', () => {
it('matches pip_setup setup.py', () => { it('matches pip_setup setup.py', () => {
expect(matchManager('setup.py')).toBe('pip_setup'); expect(matchManager('setup.py')).toBe('pip_setup');

16
lib/modules/manager/pip-compile/common.ts
View file

@ -281,13 +281,17 @@ function cleanUrl(url: string): URL | null {
} }
} }
export function getRegistryCredVarsFromPackageFile( export function getRegistryCredVarsFromPackageFiles(
packageFile: PackageFileContent | null, packageFiles: PackageFileContent[],
): ExtraEnv<string> { ): ExtraEnv<string> {
const urls = [ const urls: string[] = [];
...(packageFile?.registryUrls ?? []), for (const packageFile of packageFiles) {
...(packageFile?.additionalRegistryUrls ?? []), urls.push(
]; ...(packageFile.registryUrls ?? []),
...(packageFile.additionalRegistryUrls ?? []),
);
}
logger.debug(urls, 'Extracted registry URLs from package files');
const uniqueHosts = new Set<URL>( const uniqueHosts = new Set<URL>(
urls.map(cleanUrl).filter(isNotNullOrUndefined), urls.map(cleanUrl).filter(isNotNullOrUndefined),