renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-13 07:26:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: centralized cache directory (#10115)

Browse source 
Ignore environment variables and instead use static cache directory for third party tools.

Closes #10094

BREAKING CHANGE: Renovate will now override any package manager cache locations configured in env variables.
# Conflicts:
#	lib/manager/gomod/__snapshots__/artifacts.spec.ts.snap
#	lib/manager/gomod/artifacts.ts
This commit is contained in:
Rhys Arkins 2021-08-13 09:41:23 +02:00
parent 365ed0573a
commit 0e91fda451
15 changed files with 36 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/manager/bundler/artifacts.spec.ts
View file

@ -53,10 +53,7 @@ describe('bundler.updateArtifacts()', () => {
docker.resetPrefetchedImages();
setAdminConfig(adminConfig);
fs.ensureCacheDir.mockResolvedValueOnce(
join(adminConfig.cacheDir, './others/gem')
);
fs.ensureCacheDir.mockResolvedValue('/tmp/cache/others/gem');
});
afterEach(() => {
setAdminConfig();

4
lib/manager/bundler/artifacts.ts
View file

@ -172,13 +172,11 @@ export async function updateArtifacts(
);
}
const cacheDir = await ensureCacheDir('./others/gem', 'GEM_HOME');
const execOptions: ExecOptions = {
cwdFile: packageFileName,
extraEnv: {
...bundlerHostRulesVariables,
GEM_HOME: cacheDir,
GEM_HOME: await ensureCacheDir('bundler'),
},
docker: {
image: 'ruby',

4
lib/manager/cocoapods/artifacts.ts
View file

@ -67,13 +67,11 @@ export async function updateArtifacts({
);
const tagConstraint = match?.groups?.cocoapodsVersion ?? null;
const cacheDir = await ensureCacheDir('./others/cocoapods', 'CP_HOME_DIR');
const cmd = [...getPluginCommands(newPackageFileContent), 'pod install'];
const execOptions: ExecOptions = {
cwdFile: packageFileName,
extraEnv: {
CP_HOME_DIR: cacheDir,
CP_HOME_DIR: await ensureCacheDir('cocoapods'),
},
docker: {
image: 'cocoapods',

5
lib/manager/composer/artifacts.spec.ts
View file

@ -49,10 +49,7 @@ describe('.updateArtifacts()', () => {
docker.resetPrefetchedImages();
hostRules.clear();
setAdminConfig(adminConfig);
fs.ensureCacheDir.mockResolvedValue(
join(adminConfig.cacheDir, './others/composer')
);
fs.ensureCacheDir.mockResolvedValue('/tmp/renovate/cache/others/composer');
datasource.getPkgReleases.mockResolvedValueOnce({
releases: [
{ version: '1.0.0' },

7
lib/manager/composer/artifacts.ts
View file

@ -81,11 +81,6 @@ export async function updateArtifacts({
}: UpdateArtifact): Promise<UpdateArtifactsResult[] | null> {
logger.debug(`composer.updateArtifacts(${packageFileName})`);
const cacheDir = await ensureCacheDir(
'./others/composer',
'COMPOSER_CACHE_DIR'
);
const lockFileName = packageFileName.replace(/\.json$/, '.lock');
const existingLockFileContent = await readLocalFile(lockFileName, 'utf8');
if (!existingLockFileContent) {
@ -118,7 +113,7 @@ export async function updateArtifacts({
const execOptions: ExecOptions = {
cwdFile: packageFileName,
extraEnv: {
COMPOSER_CACHE_DIR: cacheDir,
COMPOSER_CACHE_DIR: await ensureCacheDir('composer'),
COMPOSER_AUTH: getAuthJson(),
},
docker: {

6
lib/manager/gomod/__snapshots__/artifacts.spec.ts.snap
View file

@ -284,7 +284,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/go\\":\\"/tmp/renovate/cache/others/go\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"git config --global url.\\\\\\"https://some-token@github.com/\\\\\\".insteadOf \\\\\\"https://github.com/\\\\\\" && go get -d ./...\\"",
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"git config --global url.\\\\\\"https://some-token@github.com/\\\\\\".insteadOf \\\\\\"https://github.com/\\\\\\" && go get -d ./...\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",
@ -326,7 +326,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/go\\":\\"/tmp/renovate/cache/others/go\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"go get -d ./... && go mod tidy && go mod tidy\\"",
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"go get -d ./... && go mod tidy && go mod tidy\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",
@ -368,7 +368,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/go\\":\\"/tmp/renovate/cache/others/go\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"go get -d ./...\\"",
"cmd": "docker run --rm --name=renovate_go --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e GOPATH -e GOPROXY -e GOPRIVATE -e GONOPROXY -e GONOSUMDB -e GOFLAGS -e CGO_ENABLED -w \\"/tmp/github/some/repo\\" renovate/go:latest bash -l -c \\"go get -d ./...\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",

5
lib/manager/gomod/artifacts.ts
View file

@ -94,8 +94,6 @@ export async function updateArtifacts({
}: UpdateArtifact): Promise<UpdateArtifactsResult[] | null> {
logger.debug(`gomod.updateArtifacts(${goModFileName})`);
const goPath = await ensureCacheDir('./others/go', 'GOPATH');
const sumFileName = goModFileName.replace(/\.mod$/, '.sum');
const existingGoSumContent = await readLocalFile(sumFileName);
if (!existingGoSumContent) {
@ -121,7 +119,7 @@ export async function updateArtifacts({
const execOptions: ExecOptions = {
cwdFile: goModFileName,
extraEnv: {
GOPATH: goPath,
GOPATH: await ensureCacheDir('go'),
GOPROXY: process.env.GOPROXY,
GOPRIVATE: process.env.GOPRIVATE,
GONOPROXY: process.env.GONOPROXY,
@ -133,7 +131,6 @@ export async function updateArtifacts({
image: 'go',
tagConstraint: config.constraints?.go,
tagScheme: 'npm',
volumes: [goPath],
preCommands: getPreCommands(),
},
};

21
lib/manager/npm/post-update/index.ts
View file

@ -420,17 +420,16 @@ export async function getAdditionalFiles(
const { additionalNpmrcContent, additionalYarnRcYml } = processHostRules();
const env = {
...getChildProcessEnv(),
NPM_CONFIG_CACHE: await ensureCacheDir('./others/npm', 'NPM_CONFIG_CACHE'),
YARN_CACHE_FOLDER: await ensureCacheDir(
'./others/yarn',
'YARN_CACHE_FOLDER'
),
YARN_GLOBAL_FOLDER: await ensureCacheDir('./others/berry'),
npm_config_store: await ensureCacheDir('./others/pnpm', 'npm_config_store'),
NODE_ENV: 'dev',
};
const env = getChildProcessEnv([
'NPM_CONFIG_CACHE',
'YARN_CACHE_FOLDER',
'npm_config_store',
]);
env.NPM_CONFIG_CACHE = await ensureCacheDir('npm');
env.YARN_CACHE_FOLDER = await ensureCacheDir('yarn');
env.YARN_GLOBAL_FOLDER = await ensureCacheDir('berry');
env.npm_config_store = await ensureCacheDir('pnpm');
env.NODE_ENV = 'dev';
let token = '';
try {

2
lib/manager/nuget/artifacts.spec.ts
View file

@ -46,7 +46,7 @@ describe('updateArtifacts', () => {
getDefaultRegistries.mockReturnValue([] as any);
env.getChildProcessEnv.mockReturnValue(envMock.basic);
fs.ensureCacheDir.mockImplementation((dirName: string) =>
Promise.resolve(dirName)
Promise.resolve(`others/${dirName}`)
);
getRandomString.mockReturnValue('not-so-random' as any);
setAdminConfig(adminConfig);

7
lib/manager/nuget/artifacts.ts
View file

@ -64,10 +64,9 @@ async function runDotnetRestore(
},
};
const nugetConfigDir = await ensureCacheDir(
`./others/nuget/${getRandomString()}`
);
const nugetConfigFile = join(nugetConfigDir, 'nuget.config');
const nugetCacheDir = await ensureCacheDir('nuget');
const nugetConfigDir = join(nugetCacheDir, `${getRandomString()}`);
const nugetConfigFile = join(nugetConfigDir, `nuget.config`);
await outputFile(
nugetConfigFile,
`<?xml version="1.0" encoding="utf-8"?>\n<configuration>\n</configuration>\n`

3
lib/manager/pip_setup/util.ts
View file

@ -15,8 +15,7 @@ export async function getExtractFile(): Promise<string> {
return extractPy;
}
const cacheDir = await ensureCacheDir('./others/pip_setup');
extractPy = join(cacheDir, EXTRACT);
extractPy = join(await ensureCacheDir('pip_setup'), EXTRACT);
await outputFile(extractPy, dataFiles.get('data/extract.py'));
return extractPy;

8
lib/manager/pipenv/__snapshots__/artifacts.spec.ts.snap
View file

@ -122,7 +122,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/pipenv\\":\\"/tmp/renovate/cache/others/pipenv\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python:latest bash -l -c \\"pip install --user pipenv && pipenv lock\\"",
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python:latest bash -l -c \\"pip install --user pipenv && pipenv lock\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",
@ -158,7 +158,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/pipenv\\":\\"/tmp/renovate/cache/others/pipenv\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.8.13 && pipenv lock\\"",
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.8.13 && pipenv lock\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",
@ -194,7 +194,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/pipenv\\":\\"/tmp/renovate/cache/others/pipenv\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.8.13 && pipenv lock\\"",
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.8.13 && pipenv lock\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",
@ -230,7 +230,7 @@ Array [
},
},
Object {
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -v \\"/tmp/renovate/cache/others/pipenv\\":\\"/tmp/renovate/cache/others/pipenv\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.1.1 && pipenv lock\\"",
"cmd": "docker run --rm --name=renovate_python --label=renovate_child -v \\"/tmp/github/some/repo\\":\\"/tmp/github/some/repo\\" -v \\"/tmp/renovate/cache\\":\\"/tmp/renovate/cache\\" -e PIPENV_CACHE_DIR -w \\"/tmp/github/some/repo\\" renovate/python bash -l -c \\"pip install --user pipenv==2020.1.1 && pipenv lock\\"",
"options": Object {
"cwd": "/tmp/github/some/repo",
"encoding": "utf-8",

5
lib/manager/pipenv/artifacts.ts
View file

@ -77,8 +77,6 @@ export async function updateArtifacts({
}: UpdateArtifact): Promise<UpdateArtifactsResult[] | null> {
logger.debug(`pipenv.updateArtifacts(${pipfileName})`);
const cacheDir = await ensureCacheDir('./others/pipenv', 'PIPENV_CACHE_DIR');
const lockFileName = pipfileName + '.lock';
const existingLockFileContent = await readLocalFile(lockFileName, 'utf8');
if (!existingLockFileContent) {
@ -99,7 +97,7 @@ export async function updateArtifacts({
const execOptions: ExecOptions = {
cwdFile: pipfileName,
extraEnv: {
PIPENV_CACHE_DIR: cacheDir,
PIPENV_CACHE_DIR: await ensureCacheDir('pipenv'),
},
docker: {
image: 'python',
@ -108,7 +106,6 @@ export async function updateArtifacts({
preCommands: [
`pip install --user ${quote(`pipenv${pipenvConstraint}`)}`,
],
volumes: [cacheDir],
},
};
logger.debug({ cmd }, 'pipenv lock command');

42
lib/util/fs/index.spec.ts
View file

@ -166,13 +166,12 @@ describe(getName(), () => {
dirFromEnv: string;
dirFromConfig: string;
} {
const dirFromEnv = join(root, join('/foo'));
const dirFromEnv = join(root, join('/bar/others/bundler'));
const dirFromConfig = join(root, join('/bar'));
jest.resetAllMocks();
env.getChildProcessEnv.mockReturnValueOnce({
...envMock.basic,
CUSTOM_CACHE_DIR: dirFromEnv,
});
setAdminConfig({
@ -185,45 +184,10 @@ describe(getName(), () => {
it('prefers environment variables over admin config', async () => {
await withDir(
async (tmpDir) => {
const { dirFromEnv, dirFromConfig } = setupMock(tmpDir.path);
const res = await ensureCacheDir(
'./deeply/nested',
'CUSTOM_CACHE_DIR'
);
const { dirFromEnv } = setupMock(tmpDir.path);
const res = await ensureCacheDir('bundler');
expect(res).toEqual(dirFromEnv);
expect(await exists(dirFromEnv)).toBeTrue();
expect(await exists(dirFromConfig)).toBeFalse();
},
{ unsafeCleanup: true }
);
});
it('is optional to pass environment variable', async () => {
await withDir(
async (tmpDir) => {
const { dirFromEnv, dirFromConfig } = setupMock(tmpDir.path);
const expected = join(`${dirFromConfig}/deeply/nested`);
const res = await ensureCacheDir('./deeply/nested');
expect(res).toEqual(expected);
expect(await exists(expected)).toBeTrue();
expect(await exists(dirFromEnv)).toBeFalse();
},
{ unsafeCleanup: true }
);
});
it('falls back to admin config', async () => {
await withDir(
async (tmpDir) => {
const { dirFromEnv, dirFromConfig } = setupMock(tmpDir.path);
const expected = join(`${dirFromConfig}/deeply/nested`);
const res = await ensureCacheDir(
'./deeply/nested',
'NO_SUCH_VARIABLE'
);
expect(res).toEqual(expected);
expect(await exists(expected)).toBeTrue();
expect(await exists(dirFromEnv)).toBeFalse();
},
{ unsafeCleanup: true }
);

29
lib/util/fs/index.ts
View file

@ -5,7 +5,6 @@ import * as fs from 'fs-extra';
import { isAbsolute, join, parse } from 'upath';
import { getAdminConfig } from '../../config/admin';
import { logger } from '../../logger';
import { getChildProcessEnv } from '../exec/env';
export * from './proxies';
@ -83,30 +82,10 @@ export async function ensureLocalDir(dirName: string): Promise<void> {
await fs.ensureDir(localDirName);
}
export async function ensureCacheDir(
adminCacheSubdir: string,
envCacheVar?: string
): Promise<string> {
let cacheDir: string;
if (envCacheVar) {
const env = getChildProcessEnv([envCacheVar]);
if (env[envCacheVar]) {
cacheDir = env[envCacheVar];
logger.debug(
{ cacheDir },
`Using cache directory from environment: ${envCacheVar}`
);
}
}
if (!cacheDir) {
const { cacheDir: adminCacheDir } = getAdminConfig();
cacheDir = join(adminCacheDir, adminCacheSubdir);
logger.debug({ cacheDir }, `Using cache directory from admin config`);
}
await fs.ensureDir(cacheDir);
return cacheDir;
export async function ensureCacheDir(name: string): Promise<string> {
const cacheDirName = join(getAdminConfig().cacheDir, `others/${name}`);
await fs.ensureDir(cacheDirName);
return cacheDirName;
}
/**