From 10a4a8bb26caa4753bc0ab5a1f599338a852e31b Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Tue, 7 May 2024 00:01:01 +0545 Subject: [PATCH] feat(self-hosted): `autodiscoverRepoSort` and `autodiscoverRepoOrder` (#28738) Co-authored-by: Rhys Arkins --- docs/usage/self-hosted-configuration.md | 12 +++++++ docs/usage/self-hosted-experimental.md | 37 ------------------- lib/config/global.ts | 2 ++ lib/config/options/index.ts | 20 +++++++++++ lib/config/types.ts | 3 ++ lib/modules/platform/gitea/index.spec.ts | 10 +++--- lib/modules/platform/gitea/index.ts | 40 ++++++++++++++++----- lib/modules/platform/gitea/readme.md | 2 +- lib/modules/platform/gitea/types.ts | 6 +--- lib/modules/platform/types.ts | 11 ++++++ lib/workers/global/autodiscover.ts | 2 ++ lib/workers/global/config/parse/env.spec.ts | 12 +++++++ lib/workers/global/config/parse/env.ts | 25 +++++++++++++ 13 files changed, 124 insertions(+), 58 deletions(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index ffead1ce36..b96039d95c 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -233,6 +233,18 @@ This feature is useful for users who want Renovate to only work on repositories The `autodiscoverProjects` config option takes an array of minimatch-compatible globs or RE2-compatible regex strings. For more details on this syntax see Renovate's [string pattern matching documentation](./string-pattern-matching.md). +## autodiscoverRepoOrder + +The order method for autodiscover server side repository search. + +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + +## autodiscoverRepoSort + +The sort method for autodiscover server side repository search. + +> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. + ## autodiscoverTopics Some platforms allow you to add tags, or topics, to repositories and retrieve repository lists by specifying those diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index 2e59653983..f6118f1f15 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -32,43 +32,6 @@ Skipping the check will speed things up, but may result in versions being return If set to any value, Renovate will always paginate requests to GitHub fully, instead of stopping after 10 pages. -## `RENOVATE_X_AUTODISCOVER_REPO_ORDER` - - -!!! note - For the Forgejo and Gitea platform only. - -The order method for autodiscover server side repository search. - -> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. - -Allowed values: - -- `asc` -- `desc` - -Default value: `asc`. - -## `RENOVATE_X_AUTODISCOVER_REPO_SORT` - - -!!! note - For the Forgejo and Gitea platform only. - -The sort method for autodiscover server side repository search. - -> If multiple `autodiscoverTopics` are used resulting order will be per topic not global. - -Allowed values: - -- `alpha` -- `created` -- `updated` -- `size` -- `id` - -Default value: `alpha`. - ## `RENOVATE_X_DELETE_CONFIG_FILE` If `true` Renovate tries to delete the self-hosted config file after reading it. diff --git a/lib/config/global.ts b/lib/config/global.ts index c713f8635f..01bd31a965 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -33,6 +33,8 @@ export class GlobalConfig { 'platform', 'endpoint', 'httpCacheTtlDays', + 'autodiscoverRepoSort', + 'autodiscoverRepoOrder', 'userAgent', ]; diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 949db96e06..08ae23725f 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -22,6 +22,26 @@ const options: RenovateOptions[] = [ globalOnly: true, patternMatch: true, }, + { + name: 'autodiscoverRepoOrder', + description: + 'The order method for autodiscover server side repository search.', + type: 'string', + default: null, + globalOnly: true, + allowedValues: ['asc', 'desc'], + supportedPlatforms: ['gitea'], + }, + { + name: 'autodiscoverRepoSort', + description: + 'The sort method for autodiscover server side repository search.', + type: 'string', + default: null, + globalOnly: true, + allowedValues: ['alpha', 'created', 'updated', 'size', 'id'], + supportedPlatforms: ['gitea'], + }, { name: 'allowedEnv', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index f900fae3b8..2aa2cd1552 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -2,6 +2,7 @@ import type { LogLevel } from 'bunyan'; import type { PlatformId } from '../constants'; import type { LogLevelRemap } from '../logger/types'; import type { CustomManager } from '../modules/manager/custom/types'; +import type { RepoSortMethod, SortMethod } from '../modules/platform/types'; import type { HostRule } from '../types'; import type { GitNoVerifyOption } from '../util/git/types'; import type { MergeConfidence } from '../util/merge-confidence/types'; @@ -159,6 +160,8 @@ export interface RepoGlobalConfig { privateKey?: string; privateKeyOld?: string; httpCacheTtlDays?: number; + autodiscoverRepoSort?: RepoSortMethod; + autodiscoverRepoOrder?: SortMethod; userAgent?: string; } diff --git a/lib/modules/platform/gitea/index.spec.ts b/lib/modules/platform/gitea/index.spec.ts index 77b6cb695a..f74c4270c1 100644 --- a/lib/modules/platform/gitea/index.spec.ts +++ b/lib/modules/platform/gitea/index.spec.ts @@ -224,9 +224,6 @@ describe('modules/platform/gitea/index', () => { hostRules.clear(); setBaseUrl('https://gitea.renovatebot.com/'); - - delete process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT; - delete process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER; }); async function initFakePlatform( @@ -421,8 +418,6 @@ describe('modules/platform/gitea/index', () => { }); it('Sorts repos', async () => { - process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT = 'updated'; - process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER = 'desc'; const scope = httpMock .scope('https://gitea.com/api/v1') .get('/repos/search') @@ -438,7 +433,10 @@ describe('modules/platform/gitea/index', () => { }); await initFakePlatform(scope); - const repos = await gitea.getRepos(); + const repos = await gitea.getRepos({ + sort: 'updated', + order: 'desc', + }); expect(repos).toEqual(['a/b', 'c/d']); }); }); diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 1117fe91cd..b4859dfe99 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -34,6 +34,8 @@ import type { Pr, RepoParams, RepoResult, + RepoSortMethod, + SortMethod, UpdatePrConfig, } from '../types'; import { repoFingerprint } from '../util'; @@ -49,8 +51,6 @@ import type { PRMergeMethod, PRUpdateParams, Repo, - RepoSortMethod, - SortMethod, } from './types'; import { DRAFT_PREFIX, @@ -159,7 +159,17 @@ async function lookupLabelByName(name: string): Promise { return labelList.find((l) => l.name === name)?.id ?? null; } -async function fetchRepositories(topic?: string): Promise { +interface FetchRepositoriesArgs { + topic?: string; + sort?: RepoSortMethod; + order?: SortMethod; +} + +async function fetchRepositories({ + topic, + sort, + order, +}: FetchRepositoriesArgs): Promise { const repos = await helper.searchRepos({ uid: botUserID, archived: false, @@ -167,11 +177,11 @@ async function fetchRepositories(topic?: string): Promise { topic: true, q: topic, }), - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT && { - sort: process.env.RENOVATE_X_AUTODISCOVER_REPO_SORT as RepoSortMethod, + ...(sort && { + sort, }), - ...(process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER && { - order: process.env.RENOVATE_X_AUTODISCOVER_REPO_ORDER as SortMethod, + ...(order && { + order, }), }); return repos.filter(usableRepo).map((r) => r.full_name); @@ -330,7 +340,16 @@ const platform: Platform = { try { if (config?.topics) { logger.debug({ topics: config.topics }, 'Auto-discovering by topics'); - const repos = await map(config.topics, fetchRepositories); + const fetchRepoArgs: FetchRepositoriesArgs[] = config.topics.map( + (topic) => { + return { + topic, + sort: config.sort, + order: config.order, + }; + }, + ); + const repos = await map(fetchRepoArgs, fetchRepositories); return deduplicateArray(repos.flat()); } else if (config?.namespaces) { logger.debug( @@ -348,7 +367,10 @@ const platform: Platform = { ); return deduplicateArray(repos.flat()); } else { - return await fetchRepositories(); + return await fetchRepositories({ + sort: config?.sort, + order: config?.order, + }); } } catch (err) { logger.error({ err }, 'Gitea getRepos() error'); diff --git a/lib/modules/platform/gitea/readme.md b/lib/modules/platform/gitea/readme.md index 3ce5a31cf1..b130a4aa68 100644 --- a/lib/modules/platform/gitea/readme.md +++ b/lib/modules/platform/gitea/readme.md @@ -48,5 +48,5 @@ Repositories are ignored when one of the following conditions is met: - Pull requests are disabled for that repository You can change the default server-side sort method and order for autodiscover API. -Set those via [`RENOVATE_X_AUTODISCOVER_REPO_SORT`](../../../self-hosted-experimental.md#renovate_x_autodiscover_repo_sort) and [`RENOVATE_X_AUTODISCOVER_REPO_ORDER`](../../../self-hosted-experimental.md#renovate_x_autodiscover_repo_order). +Set those via [`autodiscoverRepoSort`](../../../self-hosted-configuration.md#autodiscoverRepoSort) and [`autodiscoverRepoOrder`](../../../self-hosted-configuration.md#autodiscoverRepoOrder). Read the [Gitea swagger docs](https://try.gitea.io/api/swagger#/repository/repoSearch) for more details. diff --git a/lib/modules/platform/gitea/types.ts b/lib/modules/platform/gitea/types.ts index 32fa785469..3025a89633 100644 --- a/lib/modules/platform/gitea/types.ts +++ b/lib/modules/platform/gitea/types.ts @@ -1,5 +1,5 @@ import type { LongCommitSha } from '../../../util/git/types'; -import type { Pr } from '../types'; +import type { Pr, RepoSortMethod, SortMethod } from '../types'; export interface PrReviewersParams { reviewers?: string[]; @@ -147,10 +147,6 @@ export interface CombinedCommitStatus { statuses: CommitStatus[]; } -export type RepoSortMethod = 'alpha' | 'created' | 'updated' | 'size' | 'id'; - -export type SortMethod = 'asc' | 'desc'; - export interface RepoSearchParams { uid?: number; archived?: boolean; diff --git a/lib/modules/platform/types.ts b/lib/modules/platform/types.ts index d51e27e1e4..28f622cd61 100644 --- a/lib/modules/platform/types.ts +++ b/lib/modules/platform/types.ts @@ -201,8 +201,19 @@ export type EnsureCommentRemovalConfig = export type EnsureIssueResult = 'updated' | 'created'; +export type RepoSortMethod = + | 'alpha' + | 'created' + | 'updated' + | 'size' + | 'id' + | null; + +export type SortMethod = 'asc' | 'desc' | null; export interface AutodiscoverConfig { topics?: string[]; + sort?: RepoSortMethod; + order?: SortMethod; includeMirrors?: boolean; namespaces?: string[]; projects?: string[]; diff --git a/lib/workers/global/autodiscover.ts b/lib/workers/global/autodiscover.ts index e6e633df55..e4da1e8c40 100644 --- a/lib/workers/global/autodiscover.ts +++ b/lib/workers/global/autodiscover.ts @@ -38,6 +38,8 @@ export async function autodiscoverRepositories( // Autodiscover list of repositories let discovered = await platform.getRepos({ topics: config.autodiscoverTopics, + sort: config.autodiscoverRepoSort, + order: config.autodiscoverRepoOrder, includeMirrors: config.includeMirrors, namespaces: config.autodiscoverNamespaces, projects: config.autodiscoverProjects, diff --git a/lib/workers/global/config/parse/env.spec.ts b/lib/workers/global/config/parse/env.spec.ts index 664ca14299..1acaa0b5d5 100644 --- a/lib/workers/global/config/parse/env.spec.ts +++ b/lib/workers/global/config/parse/env.spec.ts @@ -267,6 +267,18 @@ describe('workers/global/config/parse/env', () => { expect(config.token).toBe('a'); }); + it('massages converted experimental env vars', async () => { + const envParam: NodeJS.ProcessEnv = { + RENOVATE_X_AUTODISCOVER_REPO_SORT: 'alpha', + RENOVATE_X_DOCKER_MAX_PAGES: '10', + RENOVATE_AUTODISCOVER_REPO_ORDER: 'desc', + }; + const config = await env.getConfig(envParam); + expect(config.autodiscoverRepoSort).toBe('alpha'); + expect(config.autodiscoverRepoOrder).toBe('desc'); + expect(config.dockerMaxPages).toBeUndefined(); + }); + describe('RENOVATE_CONFIG tests', () => { let processExit: jest.SpyInstance; diff --git a/lib/workers/global/config/parse/env.ts b/lib/workers/global/config/parse/env.ts index cc1c4f8fb3..a37141cff4 100644 --- a/lib/workers/global/config/parse/env.ts +++ b/lib/workers/global/config/parse/env.ts @@ -83,6 +83,30 @@ function massageEnvKeyValues(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { return result; } +const convertedExperimentalEnvVars = [ + 'RENOVATE_X_AUTODISCOVER_REPO_SORT', + 'RENOVATE_X_AUTODISCOVER_REPO_ORDER', +]; + +/** + * Massages the experimental env vars which have been converted to config options + * + * e.g. RENOVATE_X_AUTODISCOVER_REPO_SORT -> RENOVATE_AUTODISCOVER_REPO_SORT + */ +function massageConvertedExperimentalVars( + env: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + const result = { ...env }; + for (const key of convertedExperimentalEnvVars) { + if (env[key] !== undefined) { + const newKey = key.replace('RENOVATE_X_', 'RENOVATE_'); + result[newKey] = env[key]; + delete result[key]; + } + } + return result; +} + export async function getConfig( inputEnv: NodeJS.ProcessEnv, ): Promise { @@ -91,6 +115,7 @@ export async function getConfig( env = renameEnvKeys(env); // massage the values of migrated configuration keys env = massageEnvKeyValues(env); + env = massageConvertedExperimentalVars(env); const options = getOptions();