renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-12 23:16:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(pip-compile): Provide credentials for registries in all input files (#28959)

Browse source
This commit is contained in:
Miles Budnek 2024-06-04 12:33:08 -04:00 committed by GitHub
parent 1f08846483
commit c27e0ecefb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 110 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/modules/manager/pip-compile/artifacts.spec.ts
View file

@ -86,6 +86,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns null if all unchanged', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll();
fs.readLocalFile.mockResolvedValueOnce('new lock');
expect(
@ -106,6 +107,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns null if no config.lockFiles', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.readLocalFile.mockResolvedValueOnce('new lock');
expect(
await updateArtifacts({
@ -125,6 +127,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns updated requirements.txt', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({
@ -162,6 +165,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
}),
);
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip');
expect(
await updateArtifacts({
@ -215,6 +219,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
}),
);
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
expect(
await updateArtifacts({
packageFileName: 'requirements.in',
@ -328,6 +333,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('catches errors', async () => {
const execSnapshots = mockExecAll();
fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt');
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
fs.writeLocalFile.mockImplementationOnce(() => {
throw new Error('not found');
});
@ -348,6 +354,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('returns updated requirements.txt when doing lockfile maintenance', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({
@ -370,6 +377,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('uses --upgrade-package only for isLockfileUpdate', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
const execSnapshots = mockExecAll();
git.getRepoStatus.mockResolvedValue(
partial<StatusResult>({
@ -397,6 +405,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
it('uses pip-compile version from config', async () => {
fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
GlobalConfig.set(dockerAdminConfig);
// pip-tools
datasource.getPkgReleases.mockResolvedValueOnce({

30
lib/modules/manager/pip-compile/artifacts.ts
View file

@ -1,4 +1,5 @@
import { quote } from 'shlex';
import upath from 'upath';
import { TEMPORARY_ERROR } from '../../../constants/error-messages';
import { logger } from '../../../logger';
import { exec } from '../../../util/exec';
@ -8,13 +9,19 @@ import {
writeLocalFile,
} from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import * as pipRequirements from '../pip_requirements';
import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types';
import { extractPackageFileFlags as extractRequirementsFileFlags } from '../pip_requirements/common';
import type {
PackageFileContent,
UpdateArtifact,
UpdateArtifactsResult,
Upgrade,
} from '../types';
import {
extractHeaderCommand,
extractPythonVersion,
getExecOptions,
getRegistryCredVarsFromPackageFile,
getRegistryCredVarsFromPackageFiles,
matchManager,
} from './common';
import type { PipCompileArgs } from './types';
import { inferCommandExecDir } from './utils';
@ -113,12 +120,25 @@ export async function updateArtifacts({
);
const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile);
const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate);
const packageFile = pipRequirements.extractPackageFile(newInputContent);
const packageFiles: PackageFileContent[] = [];
for (const name of compileArgs.sourceFiles) {
const manager = matchManager(name);
if (manager === 'pip_requirements') {
const path = upath.join(cwd, name);
const content = await readLocalFile(path, 'utf8');
if (content) {
const packageFile = extractRequirementsFileFlags(content);
if (packageFile) {
packageFiles.push(packageFile);
}
}
}
}
const cmd = constructPipCompileCmd(compileArgs, upgradePackages);
const execOptions = await getExecOptions(
config,
cwd,
getRegistryCredVarsFromPackageFile(packageFile),
getRegistryCredVarsFromPackageFiles(packageFiles),
pythonVersion,
);
logger.trace({ cwd, cmd }, 'pip-compile command');

64
lib/modules/manager/pip-compile/common.spec.ts
View file

@ -5,7 +5,7 @@ import {
allowedPipOptions,
extractHeaderCommand,
extractPythonVersion,
getRegistryCredVarsFromPackageFile,
getRegistryCredVarsFromPackageFiles,
matchManager,
} from './common';
import { inferCommandExecDir } from './utils';
@ -187,7 +187,7 @@ describe('modules/manager/pip-compile/common', () => {
});
});
describe('getCredentialVarsFromPackageFile()', () => {
describe('getRegistryCredVarsFromPackageFiles()', () => {
it('handles both registryUrls and additionalRegistryUrls', () => {
hostRules.find.mockReturnValueOnce({
username: 'user1',
@ -198,11 +198,13 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password2',
});
expect(
getRegistryCredVarsFromPackageFile({
getRegistryCredVarsFromPackageFiles([
{
deps: [],
registryUrls: ['https://example.com/pypi/simple'],
additionalRegistryUrls: ['https://example2.com/pypi/simple'],
}),
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1',
@ -223,13 +225,15 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password2',
});
expect(
getRegistryCredVarsFromPackageFile({
getRegistryCredVarsFromPackageFiles([
{
deps: [],
additionalRegistryUrls: [
'https://example.com/pypi/simple',
'https://example2.com/pypi/simple',
],
}),
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1',
@ -245,10 +249,12 @@ describe('modules/manager/pip-compile/common', () => {
username: 'user',
});
expect(
getRegistryCredVarsFromPackageFile({
getRegistryCredVarsFromPackageFiles([
{
deps: [],
additionalRegistryUrls: ['https://example.com/pypi/simple'],
}),
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user',
@ -261,10 +267,12 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password',
});
expect(
getRegistryCredVarsFromPackageFile({
getRegistryCredVarsFromPackageFiles([
{
deps: [],
additionalRegistryUrls: ['https://example.com/pypi/simple'],
}),
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: '',
@ -277,14 +285,46 @@ describe('modules/manager/pip-compile/common', () => {
password: 'password',
});
expect(
getRegistryCredVarsFromPackageFile({
getRegistryCredVarsFromPackageFiles([
{
deps: [],
additionalRegistryUrls: ['invalid-url'],
}),
},
]),
).toEqual({});
});
});
it('handles multiple package files', () => {
hostRules.find.mockReturnValueOnce({
username: 'user1',
password: 'password1',
});
hostRules.find.mockReturnValueOnce({
username: 'user2',
password: 'password2',
});
expect(
getRegistryCredVarsFromPackageFiles([
{
deps: [],
registryUrls: ['https://example.com/pypi/simple'],
},
{
deps: [],
additionalRegistryUrls: ['https://example2.com/pypi/simple'],
},
]),
).toEqual({
KEYRING_SERVICE_NAME_0: 'example.com',
KEYRING_SERVICE_USERNAME_0: 'user1',
KEYRING_SERVICE_PASSWORD_0: 'password1',
KEYRING_SERVICE_NAME_1: 'example2.com',
KEYRING_SERVICE_USERNAME_1: 'user2',
KEYRING_SERVICE_PASSWORD_1: 'password2',
});
});
describe('matchManager()', () => {
it('matches pip_setup setup.py', () => {
expect(matchManager('setup.py')).toBe('pip_setup');

16
lib/modules/manager/pip-compile/common.ts
View file

@ -281,13 +281,17 @@ function cleanUrl(url: string): URL | null {
}
}
export function getRegistryCredVarsFromPackageFile(
packageFile: PackageFileContent | null,
export function getRegistryCredVarsFromPackageFiles(
packageFiles: PackageFileContent[],
): ExtraEnv<string> {
const urls = [
...(packageFile?.registryUrls ?? []),
...(packageFile?.additionalRegistryUrls ?? []),
];
const urls: string[] = [];
for (const packageFile of packageFiles) {
urls.push(
...(packageFile.registryUrls ?? []),
...(packageFile.additionalRegistryUrls ?? []),
);
}
logger.debug(urls, 'Extracted registry URLs from package files');
const uniqueHosts = new Set<URL>(
urls.map(cleanUrl).filter(isNotNullOrUndefined),