Merge 43c6eeea01 into b7f96b2ea1

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.devcontainer/Dockerfile

@@ -1 +1 @@
-FROM ghcr.io/containerbase/devcontainer:13.5.6
+FROM ghcr.io/containerbase/devcontainer:13.5.7

lib/config/options/index.ts

@@ -516,7 +516,7 @@ const options: RenovateOptions[] = [
     description:
       'Change this value to override the default Renovate sidecar image.',
     type: 'string',
-    default: 'ghcr.io/containerbase/sidecar:13.5.6',
+    default: 'ghcr.io/containerbase/sidecar:13.5.7',
     globalOnly: true,
   },
   {

lib/modules/datasource/pod/index.spec.ts

@@ -1,10 +1,21 @@
+import { setTimeout } from 'timers/promises';
+import fs from 'fs-extra';
+import _simpleGit, { SimpleGit } from 'simple-git';
+import { DirectoryResult, dir } from 'tmp-promise';
+import { dirname, join } from 'upath';
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../../test/http-mock';
+import { GlobalConfig } from '../../../config/global';
+import type { RepoGlobalConfig } from '../../../config/types';
 import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages';
+import * as memCache from '../../../util/cache/memory';
 import * as hostRules from '../../../util/host-rules';
 import * as rubyVersioning from '../../versioning/ruby';
 import { PodDatasource } from '.';
 
+jest.mock('simple-git');
+const simpleGit: jest.Mock<Partial<SimpleGit>> = _simpleGit as never;
+
 const config = {
   versioning: rubyVersioning.id,
   datasource: PodDatasource.id,
@@ -16,11 +27,51 @@ const githubApiHost = 'https://api.github.com';
 const githubEntApiHost = 'https://github.foo.com';
 const githubEntApiHost2 = 'https://ghe.foo.com';
 const cocoapodsHost = 'https://cdn.cocoapods.org';
+const privateRepositoryURL =
+  'https://myorg.visualstudio.com/myproject/_git/podspecs';
+
+function setupGitMocks(delayMs?: number): { mockClone: jest.Mock<any, any> } {
+  const mockClone = jest
+    .fn()
+    .mockName('clone')
+    .mockImplementation(
+      async (_registryUrl: string, clonePath: string, _opts) => {
+        if (delayMs && delayMs > 0) {
+          await setTimeout(delayMs);
+        }
+
+        const path = `${clonePath}/foo/1.2.3/foo.podspec.json`;
+        fs.mkdirSync(dirname(path), { recursive: true });
+        fs.writeFileSync(path, '', { encoding: 'utf8' });
+      },
+    );
+
+  simpleGit.mockReturnValue({
+    clone: mockClone,
+  });
+
+  return { mockClone };
+}
 
 describe('modules/datasource/pod/index', () => {
+  let tmpDir: DirectoryResult | null;
+  let adminConfig: RepoGlobalConfig;
+
   describe('getReleases', () => {
-    beforeEach(() => {
+    beforeEach(async () => {
       hostRules.clear();
+      delete process.env.COCOAPODS_GIT_REPOSITORIES;
+
+      tmpDir = await dir({ unsafeCleanup: true });
+
+      adminConfig = {
+        localDir: join(tmpDir.path, 'local'),
+        cacheDir: join(tmpDir.path, 'cache'),
+      };
+      GlobalConfig.set(adminConfig);
+
+      simpleGit.mockReset();
+      memCache.init();
     });
 
     it('returns null for invalid inputs', async () => {
@@ -341,5 +392,23 @@ describe('modules/datasource/pod/index', () => {
         ],
       });
     });
+
+    it('processes data from a private git https cocoapods specs repository', async () => {
+      process.env.COCOAPODS_GIT_REPOSITORIES = privateRepositoryURL;
+      const { mockClone } = setupGitMocks();
+      const res = await getPkgReleases({
+        ...config,
+        registryUrls: [privateRepositoryURL],
+      });
+      expect(res).toEqual({
+        registryUrl: privateRepositoryURL,
+        releases: [
+          {
+            version: '1.2.3',
+          },
+        ],
+      });
+      expect(mockClone).toHaveBeenCalled();
+    });
   });
 });

lib/modules/datasource/pod/index.ts

@@ -1,11 +1,19 @@
 import crypto from 'node:crypto';
+import { pathExistsSync, readdir } from 'fs-extra';
+import Git from 'simple-git';
+import upath from 'upath';
 import { HOST_DISABLED } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
+import * as memCache from '../../../util/cache/memory';
 import { cache } from '../../../util/cache/package/decorator';
+import { privateCacheDir } from '../../../util/fs';
+import { simpleGitConfig } from '../../../util/git/config';
+import { toSha256 } from '../../../util/hash';
 import type { HttpError } from '../../../util/http';
 import { GithubHttp } from '../../../util/http/github';
 import { newlineRegex, regEx } from '../../../util/regex';
+import { parseUrl } from '../../../util/url';
 import { Datasource } from '../datasource';
 import { massageGithubUrl } from '../metadata';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
@@ -136,6 +144,91 @@ export class PodDatasource extends Datasource {
     return null;
   }
 
+  private async getReleasesFromGit(
+    packageName: string,
+    registryUrl: string,
+  ): Promise<ReleaseResult | null> {
+    const cacheKey = `crate-datasource/registry-clone-path/${registryUrl}`;
+    const cacheKeyForError = `crate-datasource/registry-clone-path/${registryUrl}/error`;
+
+    // We need to ensure we don't run `git clone` in parallel. Therefore we store
+    // a promise of the running operation in the mem cache, which in the end resolves
+    // to the file path of the cloned repository.
+
+    const clonePathPromise: Promise<string> | null = memCache.get(cacheKey);
+    let clonePath: string;
+
+    if (clonePathPromise) {
+      clonePath = await clonePathPromise;
+    } else {
+      const url = parseUrl(registryUrl);
+      if (!url) {
+        logger.debug(`Could not parse registry URL ${registryUrl}`);
+        return null;
+      }
+
+      clonePath = upath.join(
+        privateCacheDir(),
+        PodDatasource.cacheDirFromUrl(url),
+      );
+      logger.info(
+        { clonePath, registryUrl },
+        `Cloning private cocoapods registry`,
+      );
+
+      const git = Git({ ...simpleGitConfig(), maxConcurrentProcesses: 1 });
+      const clonePromise = git.clone(registryUrl, clonePath, {
+        '--depth': 1,
+      });
+
+      memCache.set(
+        cacheKey,
+        clonePromise.then(() => clonePath).catch(() => null),
+      );
+
+      try {
+        await clonePromise;
+      } catch (err) {
+        logger.warn(
+          { err, packageName, registryUrl },
+          'failed cloning git registry',
+        );
+        memCache.set(cacheKeyForError, err);
+
+        return null;
+      }
+    }
+
+    if (!clonePath) {
+      const err = memCache.get(cacheKeyForError);
+      logger.warn(
+        { err, packageName, registryUrl },
+        'Previous git clone failed, bailing out.',
+      );
+
+      return null;
+    }
+    // Recursively get directory contents
+    const modulePath = upath.join(clonePath, packageName);
+    if (!pathExistsSync(modulePath)) {
+      return null;
+    }
+    const spec_files = (await readdir(modulePath, { recursive: true }))
+      .map((item) => item.toString()) // Convert buffers to strings
+      .filter(
+        (item) => item.endsWith('.podspec.json') || item.endsWith('.podspec'),
+      );
+    const modulesByVersion = spec_files.map((item) => {
+      const parts = item.split('/');
+      const version = parts[0];
+      return {
+        version,
+      };
+    });
+    // todo: Not return but use a single let result
+    return { releases: modulesByVersion };
+  }
+
   private async getReleasesFromGithub(
     packageName: string,
     opts: { hostURL: string; account: string; repo: string },
@@ -202,6 +295,14 @@ export class PodDatasource extends Datasource {
     return null;
   }
 
+  private static cacheDirFromUrl(url: URL): string {
+    const proto = url.protocol.replace(regEx(/:$/), '');
+    const host = url.hostname;
+    const hash = toSha256(url.pathname).substring(0, 7);
+
+    return `cocoapods-registry-${proto}-${host}-${hash}`;
+  }
+
   @cache({
     ttlMinutes: 30,
     namespace: `datasource-${PodDatasource.id}`,
@@ -224,15 +325,19 @@ export class PodDatasource extends Datasource {
     if (isDefaultRepo(baseUrl)) {
       [baseUrl] = this.defaultRegistryUrls;
     }
-
     let result: ReleaseResult | null = null;
     const match = githubRegex.exec(baseUrl);
-    if (match?.groups) {
+    const privateGitRegistry =
+      process.env.COCOAPODS_GIT_REPOSITORIES?.split(',');
+
+    if (privateGitRegistry?.includes(registryUrl)) {
+      result = await this.getReleasesFromGit(podName, registryUrl);
+    } else if (match?.groups) {
       baseUrl = massageGithubUrl(baseUrl);
       const { hostURL, account, repo } = match.groups;
       const opts = { hostURL, account, repo };
       result = await this.getReleasesFromGithub(podName, opts);
-    } else {
+    } else if (this.defaultRegistryUrls.includes(baseUrl)) {
       result = await this.getReleasesFromCDN(podName, baseUrl);
     }
 

lib/modules/datasource/pod/readme.md

@@ -0,0 +1,38 @@
+This datasource will return releases from the Cocoapods public CDN by default.
+
+### Private Cocoapods Repositories
+
+It can also be configured to return releases from a private Cocoapods repository via the environment variable `COCOAPODS_GIT_REPOSITORIES` which takes a comma separated list of one or more repository URLs.
+These URLs should be exactly the same as what is in the Podfile of your project like so:
+
+Podfile:
+
+```ruby
+source 'https://myorg.visualstudio.com/myproject/_git/podspecs'
+source 'https://cdn.cocoapods.org/'
+
+target 'MyApp' do
+  pod 'RxSwift' # Comes from 'https://cdn.cocoapods.org'
+  pod 'MyPrivatePod' # Comes from 'https://myorg.visualstudio.com/myproject/_git/podspecs'
+end
+```
+
+config.js:
+
+```javascript
+module.exports = {
+  platform: "azure",
+  endpoint: "https://myorg.visualstudio.com",
+  token: process.env.TOKEN,
+  COCOAPODS_GIT_REPOSITORIES: "https://myorg.visualstudio.com/myproject/_git/podspecs",
+  repositories: [
+    "myproject/my-mobile-app",
+  ],
+  packageRules: [
+    {
+      "matchDatasources": ["pod"],
+      "matchPackageNames": ["MyPrivatePod"]
+    }
+  ]
+};
+```

lib/workers/global/config/parse/env.spec.ts

@@ -306,7 +306,11 @@ describe('workers/global/config/parse/env', () => {
 
       it('crashes', async () => {
         const envParam: NodeJS.ProcessEnv = { RENOVATE_CONFIG: '!@#' };
-        await env.getConfig(envParam);
+        processExit.mockImplementationOnce(() => {
+          throw new Error('terminate function to simulate process.exit call');
+        });
+
+        await expect(env.getConfig(envParam)).toReject();
         expect(processExit).toHaveBeenCalledWith(1);
       });
 

lib/workers/global/config/parse/env.ts

@@ -3,6 +3,7 @@ import JSON5 from 'json5';
 import { getOptions } from '../../../../config/options';
 import type { AllConfig } from '../../../../config/types';
 import { logger } from '../../../../logger';
+import { parseJson } from '../../../../util/common';
 import { coersions } from './coersions';
 import type { ParseConfigOptions } from './types';
 import { migrateAndValidateConfig } from './util';
@@ -118,9 +119,9 @@ function massageConvertedExperimentalVars(
 
 export async function getConfig(
   inputEnv: NodeJS.ProcessEnv,
+  configEnvKey = 'RENOVATE_CONFIG',
 ): Promise<AllConfig> {
-  let env = inputEnv;
+  let env = normalizePrefixes(inputEnv, inputEnv.ENV_PREFIX);
-  env = normalizePrefixes(inputEnv, inputEnv.ENV_PREFIX);
   env = massageConvertedExperimentalVars(env);
   env = renameEnvKeys(env);
   // massage the values of migrated configuration keys
@@ -128,92 +129,82 @@ export async function getConfig(
 
   const options = getOptions();
 
-  let config: AllConfig = {};
+  const config = await parseAndValidateOrExit(env, configEnvKey);
-
-  if (env.RENOVATE_CONFIG) {
-    try {
-      config = JSON5.parse(env.RENOVATE_CONFIG);
-      logger.debug({ config }, 'Detected config in env RENOVATE_CONFIG');
-
-      config = await migrateAndValidateConfig(config, 'RENOVATE_CONFIG');
-    } catch (err) {
-      logger.fatal({ err }, 'Could not parse RENOVATE_CONFIG');
-      process.exit(1);
-    }
-  }
 
   config.hostRules ??= [];
 
-  options.forEach((option) => {
+  for (const option of options) {
-    if (option.env !== false) {
+    if (option.env === false) {
-      const envName = getEnvName(option);
+      continue;
-      const envVal = env[envName];
+    }
-      if (envVal) {
+
-        if (option.type === 'array' && option.subType === 'object') {
+    const envName = getEnvName(option);
-          try {
+    const envVal = env[envName];
-            const parsed = JSON5.parse(envVal);
+    if (!envVal) {
-            if (is.array(parsed)) {
+      continue;
-              config[option.name] = parsed;
+    }
-            } else {
+
-              logger.debug(
+    if (option.type === 'array' && option.subType === 'object') {
-                { val: envVal, envName },
+      try {
-                'Could not parse object array',
+        const parsed = JSON5.parse(envVal);
-              );
+        if (is.array(parsed)) {
-            }
+          config[option.name] = parsed;
-          } catch {
-            logger.debug(
-              { val: envVal, envName },
-              'Could not parse environment variable',
-            );
-          }
         } else {
-          const coerce = coersions[option.type];
+          logger.debug(
-          config[option.name] = coerce(envVal);
+            { val: envVal, envName },
-          if (option.name === 'dryRun') {
+            'Could not parse object array',
-            if ((config[option.name] as string) === 'true') {
+          );
-              logger.warn(
+        }
-                'env config dryRun property has been changed to full',
+      } catch {
-              );
+        logger.debug(
-              config[option.name] = 'full';
+          { val: envVal, envName },
-            } else if ((config[option.name] as string) === 'false') {
+          'Could not parse environment variable',
-              logger.warn(
+        );
-                'env config dryRun property has been changed to null',
+      }
-              );
+    } else {
-              delete config[option.name];
+      const coerce = coersions[option.type];
-            } else if ((config[option.name] as string) === 'null') {
+      config[option.name] = coerce(envVal);
-              delete config[option.name];
+      if (option.name === 'dryRun') {
-            }
+        if ((config[option.name] as string) === 'true') {
-          }
+          logger.warn('env config dryRun property has been changed to full');
-          if (option.name === 'requireConfig') {
+          config[option.name] = 'full';
-            if ((config[option.name] as string) === 'true') {
+        } else if ((config[option.name] as string) === 'false') {
-              logger.warn(
+          logger.warn('env config dryRun property has been changed to null');
-                'env config requireConfig property has been changed to required',
+          delete config[option.name];
-              );
+        } else if ((config[option.name] as string) === 'null') {
-              config[option.name] = 'required';
+          delete config[option.name];
-            } else if ((config[option.name] as string) === 'false') {
+        }
-              logger.warn(
+      }
-                'env config requireConfig property has been changed to optional',
+
-              );
+      if (option.name === 'requireConfig') {
-              config[option.name] = 'optional';
+        if ((config[option.name] as string) === 'true') {
-            }
+          logger.warn(
-          }
+            'env config requireConfig property has been changed to required',
-          if (option.name === 'platformCommit') {
+          );
-            if ((config[option.name] as string) === 'true') {
+          config[option.name] = 'required';
-              logger.warn(
+        } else if ((config[option.name] as string) === 'false') {
-                'env config platformCommit property has been changed to enabled',
+          logger.warn(
-              );
+            'env config requireConfig property has been changed to optional',
-              config[option.name] = 'enabled';
+          );
-            } else if ((config[option.name] as string) === 'false') {
+          config[option.name] = 'optional';
-              logger.warn(
+        }
-                'env config platformCommit property has been changed to disabled',
+      }
-              );
+
-              config[option.name] = 'disabled';
+      if (option.name === 'platformCommit') {
-            }
+        if ((config[option.name] as string) === 'true') {
-          }
+          logger.warn(
+            'env config platformCommit property has been changed to enabled',
+          );
+          config[option.name] = 'enabled';
+        } else if ((config[option.name] as string) === 'false') {
+          logger.warn(
+            'env config platformCommit property has been changed to disabled',
+          );
+          config[option.name] = 'disabled';
         }
       }
     }
-  });
+  }
 
   if (env.GITHUB_COM_TOKEN) {
     logger.debug(`Converting GITHUB_COM_TOKEN into a global host rule`);
@@ -237,7 +228,31 @@ export async function getConfig(
     'VSTS_TOKEN',
   ];
 
-  unsupportedEnv.forEach((val) => delete env[val]);
+  for (const val of unsupportedEnv) {
+    delete env[val];
+  }
 
   return config;
 }
+
+async function parseAndValidateOrExit(
+  env: NodeJS.ProcessEnv,
+  configEnvKey: string,
+): Promise<AllConfig> {
+  if (!env[configEnvKey]) {
+    return {};
+  }
+
+  try {
+    const config = parseJson(
+      env[configEnvKey],
+      `${configEnvKey}.env.json5`,
+    ) as AllConfig;
+    logger.debug({ config }, `Detected config in env ${configEnvKey}`);
+
+    return await migrateAndValidateConfig(config, `${configEnvKey}`);
+  } catch (err) {
+    logger.fatal({ err }, `Could not parse ${configEnvKey}`);
+    process.exit(1);
+  }
+}

package.json

@@ -143,12 +143,12 @@
     "pnpm": "9.15.1"
   },
   "dependencies": {
-    "@aws-sdk/client-codecommit": "3.699.0",
+    "@aws-sdk/client-codecommit": "3.716.0",
-    "@aws-sdk/client-ec2": "3.701.0",
+    "@aws-sdk/client-ec2": "3.716.0",
-    "@aws-sdk/client-ecr": "3.699.0",
+    "@aws-sdk/client-ecr": "3.720.0",
-    "@aws-sdk/client-rds": "3.699.0",
+    "@aws-sdk/client-rds": "3.719.1",
-    "@aws-sdk/client-s3": "3.701.0",
+    "@aws-sdk/client-s3": "3.717.0",
-    "@aws-sdk/credential-providers": "3.699.0",
+    "@aws-sdk/credential-providers": "3.716.0",
     "@breejs/later": "4.2.0",
     "@cdktf/hcl2json": "0.20.10",
     "@opentelemetry/api": "1.9.0",
@@ -311,8 +311,8 @@
     "@types/url-join": "4.0.3",
     "@types/validate-npm-package-name": "4.0.2",
     "@types/xmldoc": "1.1.9",
-    "@typescript-eslint/eslint-plugin": "8.18.1",
+    "@typescript-eslint/eslint-plugin": "8.18.2",
-    "@typescript-eslint/parser": "8.18.1",
+    "@typescript-eslint/parser": "8.18.2",
     "aws-sdk-client-mock": "4.1.0",
     "callsite": "1.0.0",
     "common-tags": "1.8.2",

tools/docker/Dockerfile

@@ -5,19 +5,19 @@ ARG BASE_IMAGE_TYPE=slim
 # --------------------------------------
 # slim image
 # --------------------------------------
-FROM ghcr.io/renovatebot/base-image:9.27.10@sha256:bb66c6760180eca52655624f8cfb86e1581ba56ba014d788eb5edddd3fb4405f AS slim-base
+FROM ghcr.io/renovatebot/base-image:9.27.12@sha256:4e40805172da37b74583f26b811aa47bde3f4e68afea511d451dad0ddc86d40a AS slim-base
 
 # --------------------------------------
 # full image
 # --------------------------------------
-FROM ghcr.io/renovatebot/base-image:9.27.10-full@sha256:7345a26aba660efc97621a291785e951079f3e03077a6e57b06a022a0cb22c54 AS full-base
+FROM ghcr.io/renovatebot/base-image:9.27.12-full@sha256:5925cfd3e6ade0d1c7ca0b6c20be80ee055ce3d18e2d1d75b92fce1e0604f5a0 AS full-base
 
 ENV RENOVATE_BINARY_SOURCE=global
 
 # --------------------------------------
 # build image
 # --------------------------------------
-FROM --platform=$BUILDPLATFORM ghcr.io/renovatebot/base-image:9.27.10@sha256:bb66c6760180eca52655624f8cfb86e1581ba56ba014d788eb5edddd3fb4405f AS build
+FROM --platform=$BUILDPLATFORM ghcr.io/renovatebot/base-image:9.27.12@sha256:4e40805172da37b74583f26b811aa47bde3f4e68afea511d451dad0ddc86d40a AS build
 
 # We want a specific node version here
 # renovate: datasource=node-version