renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-09 21:46:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

refactor: simplify docker build (#27174)

Browse source
This commit is contained in:
Michael Kriese 2024-02-09 14:23:49 +01:00 committed by GitHub
parent c9151596f9
commit 19e49a6666
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 352 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.dockerignore Normal file
View file

@ -0,0 +1,8 @@
*
!tools/docker/bin
!dist/
!node_modules/
!package.json
!pnpm-lock.yaml
!renovate-schema.json
!license

24
.github/workflows/build.yml vendored
View file

@ -466,6 +466,11 @@ jobs:
- name: Build
run: pnpm build
- name: Build docker
run: pnpm build:docker build --tries=3
env:
LOG_LEVEL: debug
- name: Pack
run: pnpm test-e2e:pack
@ -550,6 +555,7 @@ jobs:
issues: write
pull-requests: write
id-token: write
packages: write
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -558,12 +564,24 @@ jobs:
show-progress: false
filter: blob:none # we don't need all blobs, only the full tree
- name: docker-config
uses: containerbase/internal-tools@e7bd2e8cedd99c9b24982865534cb7c9bf88620b # v3.0.55
with:
command: docker-config
- name: Setup Node.js
uses: ./.github/actions/setup-node
with:
node-version: ${{ env.NODE_VERSION }}
os: ${{ runner.os }}
- uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0
- name: Docker registry login
run: |
echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
- name: Check dry run
run: |
if [[ "${{github.event_name}}" == "workflow_dispatch" && "${{ github.event.inputs.dryRun }}" != "true" ]]; then
@ -574,13 +592,9 @@ jobs:
echo "DRY_RUN=false" >> "$GITHUB_ENV"
fi
# TODO: move to semantic-release prepare
- name: Build
run: pnpm build
- name: semantic-release
run: |
pnpm semantic-release --dry-run ${{env.DRY_RUN}}
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} # TODO: use action token?
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

4
.releaserc.json
View file

@ -18,8 +18,8 @@
[
"@semantic-release/exec",
{
"prepareCmd": "pnpm release:prepare --release=${nextRelease.version} --sha=${nextRelease.gitHead} --tag=${nextRelease.channel}",
"publishCmd": "pnpm release:publish --release=${nextRelease.version} --sha=${nextRelease.gitHead} --tag=${nextRelease.channel}"
"prepareCmd": "pnpm release:prepare --version=${nextRelease.version} --sha=${nextRelease.gitHead} --tries=3 --platform=linux/amd64,linux/arm64 --exit-on-error=false",
"publishCmd": "pnpm release:publish --version=${nextRelease.version} --sha=${nextRelease.gitHead} --platform=linux/amd64,linux/arm64 --exit-on-error=false"
}
]
],

8
package.json
View file

@ -9,8 +9,8 @@
},
"scripts": {
"build": "run-s clean 'generate:*' 'compile:*' create-json-schema",
"build:docker": "node tools/docker.mjs",
"build:docs": "run-s 'release:prepare {@}' --",
"build:docker": "ts-node tools/docker.ts",
"build:docs": "ts-node tools/generate-docs.ts",
"clean": "rimraf dist tmp",
"clean-cache": "node tools/clean-cache.mjs",
"compile:ts": "tsc -p tsconfig.app.json",
@ -45,8 +45,8 @@
"pretest": "run-s 'generate:*'",
"prettier": "prettier --cache --check '**/*.{ts,js,mjs,json,md,yml}'",
"prettier-fix": "prettier --write --cache '**/*.{ts,js,mjs,json,md,yml}'",
"release:prepare": "ts-node tools/generate-docs.ts",
"release:publish": "node tools/release.mjs",
"release:prepare": "ts-node tools/prepare-release.ts",
"release:publish": "ts-node tools/publish-release.ts",
"start": "ts-node lib/renovate.ts",
"test": "run-s lint test-schema jest",
"test-dirty": "git diff --exit-code",

47
tools/docker.mjs → tools/docker.ts
View file

@ -1,48 +1,23 @@
import { Command } from 'commander';
import { bake } from './utils/docker.mjs';
import { logger } from '../lib/logger';
import { parsePositiveInt, parseVersion } from './utils';
import { bake } from './utils/docker';
const program = new Command('pnpm build:docker');
/**
*
* @param {string | undefined} val
*/
function parseInt(val) {
if (!val) {
return 0;
}
const r = Number.parseInt(val, 10);
if (!Number.isFinite(r) || r < 0) {
throw new Error(`Invalid number: ${val}`);
}
return r;
}
/**
*
* @param {string | undefined} val
*/
function parseVersion(val) {
if (!val) {
return val;
}
if (!/^\d+\.\d+\.\d+(?:-.+)?$/.test(val)) {
throw new Error(`Invalid version: ${val}`);
}
return val;
}
program
.command('build')
.description('Build docker images')
.option('--platform <type>', 'docker platforms to build')
.option('--version <version>', 'version to use as tag', parseVersion)
.option('--tries <tries>', 'number of tries on failure', parseInt)
.option('--tries <tries>', 'number of tries on failure', parsePositiveInt)
.option(
'--delay <delay>',
'delay between tries for docker build (eg. 5s, 10m, 1h)',
'30s',
)
.action(async (opts) => {
console.log('Building docker images ...');
logger.info('Building docker images ...');
await bake('build', opts, opts.tries - 1);
});
@ -52,7 +27,7 @@ program
.option('--platform <type>', 'docker platforms to build')
.option('--version <version>', 'version to use as tag', parseVersion)
.action(async (opts) => {
console.log('Publishing docker images ...');
logger.info('Publishing docker images ...');
await bake('push', opts);
});

45
tools/docker/Dockerfile
View file

@ -1,4 +1,3 @@
ARG RENOVATE_VERSION
ARG BASE_IMAGE_TYPE=slim
# --------------------------------------
@ -11,6 +10,37 @@ FROM ghcr.io/renovatebot/base-image:1.22.0@sha256:59606f80b6194a99f9d7d4a2667dcc
# --------------------------------------
FROM ghcr.io/renovatebot/base-image:1.22.0-full@sha256:7a371dcfff219fc638301ce1856d92ee2a09993f628a7b641d8da12c6e23eb0d AS full-base
# --------------------------------------
# build image
# --------------------------------------
FROM slim-base as build
WORKDIR /usr/local/renovate
ENV CI=1 npm_config_modules_cache_max_age=0
COPY pnpm-lock.yaml ./
# only fetch deps from lockfile https://pnpm.io/cli/fetch
RUN pnpm fetch --prod
COPY . ./
# install
ENV RE2_DOWNLOAD_MIRROR=https://github.com/containerbase/node-re2-prebuild/releases/download RE2_DOWNLOAD_SKIP_PATH=1
RUN set -ex; \
pnpm install --prod --offline --ignore-scripts; \
npm explore re2 -- npm run install; \
true
# test
COPY tools/docker/bin/ /usr/local/bin/
RUN set -ex; \
renovate --version; \
renovate-config-validator; \
node -e "new require('re2')('.*').exec('test')"; \
true
# --------------------------------------
# final image
# --------------------------------------
@ -21,19 +51,24 @@ LABEL org.opencontainers.image.source="https://github.com/renovatebot/renovate"
org.opencontainers.image.url="https://renovatebot.com" \
org.opencontainers.image.licenses="AGPL-3.0-only"
WORKDIR /usr/src/app
ENV RENOVATE_X_IGNORE_NODE_WARN=true
COPY bin/ /usr/local/bin/
COPY tools/docker/bin/ /usr/local/bin/
CMD ["renovate"]
ARG RENOVATE_VERSION
RUN install-tool renovate
COPY --from=build --chown=root:root /usr/local/renovate/ /usr/local/renovate/
# Compabillity, so `config.js` can access renovate and deps
RUN ln -sf /opt/containerbase/tools/renovate/${RENOVATE_VERSION}/node_modules ./node_modules;
RUN set -ex; \
mkdir /opt/containerbase/tools/renovate; \
echo "${RENOVATE_VERSION}" > /opt/containerbase/versions/renovate; \
ln -sf /usr/local/renovate /opt/containerbase/tools/renovate/${RENOVATE_VERSION}; \
ln -sf /usr/local/renovate/node_modules ./node_modules; \
true
RUN set -ex; \
renovate --version; \

18
tools/docker/bake.hcl
View file

@ -42,8 +42,15 @@ group "push" {
]
}
group "push-cache" {
targets = [
"push-cache-slim",
"push-cache-full",
]
}
target "settings" {
context = "tools/docker"
dockerfile = "tools/docker/Dockerfile"
args = {
APT_HTTP_PROXY = "${APT_HTTP_PROXY}"
CONTAINERBASE_DEBUG = "${CONTAINERBASE_DEBUG}"
@ -54,7 +61,7 @@ target "settings" {
target "slim" {
cache-from = [
"type=registry,ref=ghcr.io/${OWNER}/docker-build-cache:${FILE}-${RENOVATE_VERSION}",
"type=registry,ref=ghcr.io/${OWNER}/docker-build-cache:${FILE}",
]
tags = [
"ghcr.io/${OWNER}/${FILE}:${RENOVATE_VERSION}",
@ -67,7 +74,7 @@ target "full" {
BASE_IMAGE_TYPE = "full"
}
cache-from = [
"type=registry,ref=ghcr.io/${OWNER}/docker-build-cache:${FILE}-${RENOVATE_VERSION}-full",
"type=registry,ref=ghcr.io/${OWNER}/docker-build-cache:${FILE}-full",
]
tags = [
"ghcr.io/${OWNER}/${FILE}:${RENOVATE_VERSION}-full",
@ -87,7 +94,7 @@ target "push-cache-slim" {
"slim",
]
tags = [
"ghcr.io/${OWNER}/docker-build-cache:${FILE}-${RENOVATE_VERSION}",
"ghcr.io/${OWNER}/docker-build-cache:${FILE}",
]
}
@ -98,7 +105,7 @@ target "push-cache-full" {
"full",
]
tags = [
"ghcr.io/${OWNER}/docker-build-cache:${FILE}-${RENOVATE_VERSION}-full",
"ghcr.io/${OWNER}/docker-build-cache:${FILE}-full",
]
}
@ -108,7 +115,6 @@ target "build-slim" {
target "build-full" {
inherits = ["settings", "full"]
}
target "push-slim" {

8
tools/docker/bin/renovate Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
if [[ -f "/usr/local/etc/env" && -z "${CONTAINERBASE_ENV+x}" ]]; then
# shellcheck source=/dev/null
. /usr/local/etc/env
fi
node /usr/local/renovate/dist/renovate.js "$@"

8
tools/docker/bin/renovate-config-validator Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
if [[ -f "/usr/local/etc/env" && -z "${CONTAINERBASE_ENV+x}" ]]; then
# shellcheck source=/dev/null
. /usr/local/etc/env
fi
node /usr/local/renovate/dist/config-validator.js "$@"

81
tools/docs/index.ts Normal file
View file

@ -0,0 +1,81 @@
import { ERROR } from 'bunyan';
import fs from 'fs-extra';
import * as tar from 'tar';
import { getProblems, logger } from '../../lib/logger';
import { generateConfig } from './config';
import { generateDatasources } from './datasources';
import { getOpenGitHubItems } from './github-query-items';
import { generateManagers } from './manager';
import { generateManagerAsdfSupportedPlugins } from './manager-asdf-supported-plugins';
import { generatePlatforms } from './platforms';
import { generatePresets } from './presets';
import { generateSchema } from './schema';
import { generateTemplates } from './templates';
import { generateVersioning } from './versioning';
export async function generateDocs(): Promise<void> {
try {
const dist = 'tmp/docs';
logger.info('generating docs');
await fs.mkdir(`${dist}/`, { recursive: true });
logger.info('* static');
await fs.copy('docs/usage/.', `${dist}`);
logger.info('* fetching open GitHub issues');
const openItems = await getOpenGitHubItems();
logger.info('* platforms');
await generatePlatforms(dist, openItems.platforms);
// versionings
logger.info('* versionings');
await generateVersioning(dist);
// datasources
logger.info('* datasources');
await generateDatasources(dist, openItems.datasources);
// managers
logger.info('* managers');
await generateManagers(dist, openItems.managers);
// managers/asdf supported plugins
logger.info('* managers/asdf/supported-plugins');
await generateManagerAsdfSupportedPlugins(dist);
// presets
logger.info('* presets');
await generatePresets(dist);
// templates
logger.info('* templates');
await generateTemplates(dist);
// configuration-options
logger.info('* configuration-options');
await generateConfig(dist);
// self-hosted-configuration
logger.info('* self-hosted-configuration');
await generateConfig(dist, true);
// json-schema
logger.info('* json-schema');
await generateSchema(dist);
await tar.create(
{ file: './tmp/docs.tgz', cwd: './tmp/docs', gzip: true },
['.'],
);
} catch (err) {
logger.error({ err }, 'Unexpected error');
} finally {
const loggerErrors = getProblems().filter((p) => p.level >= ERROR);
if (loggerErrors.length) {
process.exit(1);
}
}
}

84
tools/generate-docs.ts
View file

@ -1,17 +1,5 @@
import { ERROR } from 'bunyan';
import fs from 'fs-extra';
import * as tar from 'tar';
import { getProblems, logger } from '../lib/logger';
import { generateConfig } from './docs/config';
import { generateDatasources } from './docs/datasources';
import { getOpenGitHubItems } from './docs/github-query-items';
import { generateManagers } from './docs/manager';
import { generateManagerAsdfSupportedPlugins } from './docs/manager-asdf-supported-plugins';
import { generatePlatforms } from './docs/platforms';
import { generatePresets } from './docs/presets';
import { generateSchema } from './docs/schema';
import { generateTemplates } from './docs/templates';
import { generateVersioning } from './docs/versioning';
import { logger } from '../lib/logger';
import { generateDocs } from './docs';
process.on('unhandledRejection', (err) => {
// Will print "unhandledRejection err is not defined"
@ -19,70 +7,4 @@ process.on('unhandledRejection', (err) => {
process.exit(-1);
});
// eslint-disable-next-line @typescript-eslint/no-floating-promises
(async () => {
try {
const dist = 'tmp/docs';
logger.info('generating docs');
await fs.mkdir(`${dist}/`, { recursive: true });
logger.info('* static');
await fs.copy('docs/usage/.', `${dist}`);
logger.info('* fetching open GitHub issues');
const openItems = await getOpenGitHubItems();
logger.info('* platforms');
await generatePlatforms(dist, openItems.platforms);
// versionings
logger.info('* versionings');
await generateVersioning(dist);
// datasources
logger.info('* datasources');
await generateDatasources(dist, openItems.datasources);
// managers
logger.info('* managers');
await generateManagers(dist, openItems.managers);
// managers/asdf supported plugins
logger.info('* managers/asdf/supported-plugins');
await generateManagerAsdfSupportedPlugins(dist);
// presets
logger.info('* presets');
await generatePresets(dist);
// templates
logger.info('* templates');
await generateTemplates(dist);
// configuration-options
logger.info('* configuration-options');
await generateConfig(dist);
// self-hosted-configuration
logger.info('* self-hosted-configuration');
await generateConfig(dist, true);
// json-schema
logger.info('* json-schema');
await generateSchema(dist);
await tar.create(
{ file: './tmp/docs.tgz', cwd: './tmp/docs', gzip: true },
['.'],
);
} catch (err) {
logger.error({ err }, 'Unexpected error');
} finally {
const loggerErrors = getProblems().filter((p) => p.level >= ERROR);
if (loggerErrors.length) {
process.exit(1);
}
}
})();
void generateDocs();

34
tools/prepare-release.ts Normal file
View file

@ -0,0 +1,34 @@
import { Command } from 'commander';
import { logger } from '../lib/logger';
import { generateDocs } from './docs';
import { parseVersion } from './utils';
import { bake } from './utils/docker';
process.on('unhandledRejection', (err) => {
// Will print "unhandledRejection err is not defined"
logger.error({ err }, 'unhandledRejection');
process.exit(-1);
});
const program = new Command('pnpm release:prepare')
.description('Build docker images')
.option('--platform <type>', 'docker platforms to build')
.option('--version <version>', 'version to use as tag', parseVersion)
.option('--tries <tries>', 'number of tries for docker build', parseInt)
.option(
'--delay <delay>',
'delay between tries for docker build (eg. 5s, 10m, 1h)',
'30s',
)
.option('--exit-on-error [boolean]', 'exit on docker error', (s) =>
s ? s !== 'false' : undefined,
)
.option('-d, --debug', 'output docker build');
void (async () => {
await program.parseAsync();
const opts = program.opts();
logger.info(`Preparing v${opts.version} ...`);
await generateDocs();
await bake('build', opts, opts.tries);
})();

25
tools/publish-release.ts Normal file
View file

@ -0,0 +1,25 @@
import { Command } from 'commander';
import { logger } from '../lib/logger';
import { parseVersion } from './utils';
import { bake } from './utils/docker';
process.on('unhandledRejection', (err) => {
// Will print "unhandledRejection err is not defined"
logger.error({ err }, 'unhandledRejection');
process.exit(-1);
});
const program = new Command('pnpm release:prepare')
.description('Build docker images')
.option('--platform <type>', 'docker platforms to build')
.option('--version <version>', 'version to use as tag', parseVersion)
.option('--exit-on-error', 'exit on docker error')
.option('-d, --debug', 'output docker build');
void (async () => {
await program.parseAsync();
const opts = program.opts();
logger.info(`Publishing v${opts.version}...`);
logger.info(`TODO: publish docker images`);
await bake('push-cache', opts);
})();

8
tools/release.mjs
View file

@ -1,8 +0,0 @@
import { options } from './utils/options.mjs';
const version = options.release;
console.log(`Publishing version: ${version}`);
// eslint-disable-next-line promise/valid-params,@typescript-eslint/no-floating-promises
import('./dispatch-release.mjs').catch();

43
tools/utils/docker.mjs
View file

@ -1,43 +0,0 @@
import { setTimeout } from 'timers/promises';
import { exec } from './exec.mjs';
const file = 'tools/docker/bake.hcl';
/**
*
* @param {string} target
* @param {{platform?:string, version?: string, args?: string[]}} opts
* @param {number} tries
*/
export async function bake(target, opts, tries = 0) {
if (opts.version) {
console.log(`Using version: ${opts.version}`);
process.env.RENOVATE_VERSION = opts.version;
}
const args = ['buildx', 'bake', '--file', file];
if (opts.platform) {
console.log(`Using platform: ${opts.platform}`);
args.push('--set', `settings.platform=${opts.platform}`);
}
if (Array.isArray(opts.args)) {
console.log(`Using args: ${opts.args.join(' ')}`);
args.push(...opts.args);
}
args.push(target);
const result = exec(`docker`, args);
if (result.status !== 0) {
if (tries > 0) {
console.log(`Error occured:`, result.stderr || result.stdout);
console.warn(`Retrying in 30s ...`);
await setTimeout(30000);
return bake(target, opts, tries - 1);
} else {
throw new Error(result.stderr || result.stdout);
}
}
}

60
tools/utils/docker.ts Normal file
View file

@ -0,0 +1,60 @@
import { setTimeout } from 'timers/promises';
import { logger } from '../../lib/logger';
import { toMs } from '../../lib/util/pretty-time';
import { exec } from './exec';
const file = 'tools/docker/bake.hcl';
export async function bake(
target: string,
opts: {
platform?: string;
version?: string;
args?: string[];
delay?: string;
exitOnError?: boolean;
},
tries: number = 0,
): Promise<void> {
if (opts.version) {
console.log(`Using version: ${opts.version}`);
process.env.RENOVATE_VERSION = opts.version;
}
const args = ['buildx', 'bake', '--file', file];
if (opts.platform) {
console.log(`Using platform: ${opts.platform}`);
args.push('--set', `settings.platform=${opts.platform}`);
}
if (Array.isArray(opts.args)) {
console.log(`Using args: ${opts.args.join(' ')}`);
args.push(...opts.args);
}
args.push(target);
const result = exec(`docker`, args);
if (result.signal) {
logger.error(`Signal received: ${result.signal}`);
process.exit(1);
} else if (result.status && result.status !== 0) {
if (tries > 0) {
logger.debug(`Error occured:\n ${result.stderr}`);
const delay = opts.delay ? toMs(opts.delay) : null;
if (delay) {
logger.info(`Retrying in ${opts.delay} ...`);
await setTimeout(delay);
}
return bake(target, opts, tries - 1);
} else {
logger.error(`Error occured:\n${result.stderr}`);
if (opts.exitOnError !== false) {
process.exit(result.status);
}
}
} else {
logger.debug(`${target} succeeded:\n${result.stdout || result.stderr}`);
}
}

7
tools/utils/exec.mjs → tools/utils/exec.ts
View file

@ -1,4 +1,4 @@
import { spawnSync } from 'node:child_process';
import { type SpawnSyncReturns, spawnSync } from 'node:child_process';
const maxBuffer = 20 * 1024 * 1024;
@ -7,7 +7,10 @@ const maxBuffer = 20 * 1024 * 1024;
* @param {string} cmd
* @param {string[]} args
*/
export function exec(cmd, args = []) {
export function exec(
cmd: string,
args: string[] = [],
): SpawnSyncReturns<string> {
// args from shelljs
return spawnSync(cmd, args, { maxBuffer, encoding: 'utf8' });
}

32
tools/utils/index.ts
View file

@ -65,3 +65,35 @@ export function readFile(file: string): Promise<string> {
}
return Promise.resolve('');
}
/**
*
* @param val
*/
export function parsePositiveInt(val: string | undefined): number {
if (!val) {
return 0;
}
const r = Number.parseInt(val, 10);
if (!Number.isFinite(r) || r < 0) {
throw new Error(`Invalid number: ${val}`);
}
return r;
}
/**
*
* @param val
*/
export function parseVersion(val: string | undefined): string | undefined {
if (!val) {
return val;
}
if (!/^\d+\.\d+\.\d+(?:-.+)?$/.test(val)) {
throw new Error(`Invalid version: ${val}`);
}
return val;
}