feat(rubygems): support GitHub Packages (#11107)

lib/datasource/rubygems/common.ts

@@ -1 +1,40 @@
+import Marshal from 'marshal';
+import urlJoin from 'url-join';
+import { logger } from '../../logger';
+import { Http } from '../../util/http';
+import { getQueryString } from '../../util/url';
+
 export const id = 'rubygems';
+export const http = new Http(id);
+
+export const knownFallbackHosts = ['rubygems.pkg.github.com', 'gitlab.com'];
+
+export async function fetchJson<T>(
+  dependency: string,
+  registry: string,
+  path: string
+): Promise<T> {
+  const url = urlJoin(registry, path, `${dependency}.json`);
+
+  logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+  const response = (await http.getJson<T>(url)) || {
+    body: undefined,
+  };
+
+  return response.body;
+}
+
+export async function fetchBuffer<T>(
+  dependency: string,
+  registry: string,
+  path: string
+): Promise<T> {
+  const url = `${urlJoin(registry, path)}?${getQueryString({
+    gems: dependency,
+  })}`;
+
+  logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+  const response = await http.getBuffer(url);
+
+  return new Marshal(response.body).parsed as T;
+}

lib/datasource/rubygems/get-rubygems-org.ts

@@ -1,11 +1,8 @@
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import { getElapsedMinutes } from '../../util/date';
-import { Http } from '../../util/http';
 import type { ReleaseResult } from '../types';
-import { id } from './common';
+import { http } from './common';
-
-const http = new Http(id);
 
 let lastSync = new Date('2000-01-01');
 let packageReleases: Record<string, string[]> = Object.create(null); // Because we might need a "constructor" key

lib/datasource/rubygems/get.ts

@@ -1,40 +1,65 @@
-import urlJoin from 'url-join';
 import { logger } from '../../logger';
-import { Http } from '../../util/http';
+import { HttpError } from '../../util/http/types';
-import type { OutgoingHttpHeaders } from '../../util/http/types';
+import type { Release, ReleaseResult } from '../types';
-import type { ReleaseResult } from '../types';
+import { fetchBuffer, fetchJson } from './common';
-import { id } from './common';
+import type {
-
+  JsonGemVersions,
-const http = new Http(id);
+  JsonGemsInfo,
+  MarshalledVersionInfo,
+} from './types';
 
 const INFO_PATH = '/api/v1/gems';
 const VERSIONS_PATH = '/api/v1/versions';
+const DEPENDENCIES_PATH = '/api/v1/dependencies';
 
-const getHeaders = (): OutgoingHttpHeaders => ({ hostType: id });
+export async function getDependencyFallback(
-
-export async function fetch(
   dependency: string,
-  registry: string,
+  registry: string
-  path: string
+): Promise<ReleaseResult | null> {
-): Promise<any> {
+  logger.debug(
-  const headers = getHeaders();
+    { dependency, api: DEPENDENCIES_PATH },
-
+    'RubyGems lookup for dependency'
-  const url = urlJoin(registry, path, `${dependency}.json`);
+  );
-
+  const info = await fetchBuffer<MarshalledVersionInfo[]>(
-  logger.trace({ dependency }, `RubyGems lookup request: ${String(url)}`);
+    dependency,
-  const response = (await http.getJson(url, { headers })) || {
+    registry,
-    body: undefined,
+    DEPENDENCIES_PATH
+  );
+  if (!info || info.length === 0) {
+    return null;
+  }
+  const releases = info.map(({ number: version, platform: rubyPlatform }) => ({
+    version,
+    rubyPlatform,
+  }));
+  return {
+    releases,
+    homepage: null,
+    sourceUrl: null,
+    changelogUrl: null,
   };
-
-  return response.body;
 }
 
 export async function getDependency(
   dependency: string,
   registry: string
 ): Promise<ReleaseResult | null> {
-  logger.debug({ dependency }, 'RubyGems lookup for dependency');
+  logger.debug(
-  const info = await fetch(dependency, registry, INFO_PATH);
+    { dependency, api: INFO_PATH },
+    'RubyGems lookup for dependency'
+  );
+  let info: JsonGemsInfo;
+
+  try {
+    info = await fetchJson(dependency, registry, INFO_PATH);
+  } catch (error) {
+    // fallback to deps api on 404
+    if (error instanceof HttpError && error.response?.statusCode === 404) {
+      return await getDependencyFallback(dependency, registry);
+    }
+    throw error;
+  }
+
   if (!info) {
     logger.debug({ dependency }, 'RubyGems package not found.');
     return null;
@@ -48,10 +73,10 @@ export async function getDependency(
     return null;
   }
 
-  let versions = [];
+  let versions: JsonGemVersions[] = [];
-  let releases = [];
+  let releases: Release[] = [];
   try {
-    versions = await fetch(dependency, registry, VERSIONS_PATH);
+    versions = await fetchJson(dependency, registry, VERSIONS_PATH);
   } catch (err) {
     if (err.statusCode === 400 || err.statusCode === 404) {
       logger.debug(
@@ -63,13 +88,15 @@ export async function getDependency(
     }
   }
 
+  // TODO: invalid properties for `Release` see #11312
+
   if (versions.length === 0 && info.version) {
     logger.warn('falling back to the version from the info endpoint');
     releases = [
       {
         version: info.version,
         rubyPlatform: info.platform,
-      },
+      } as Release,
     ];
   } else {
     releases = versions.map(

lib/datasource/rubygems/index.spec.ts

@@ -1,6 +1,10 @@
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../test/http-mock';
-import { loadFixture, loadJsonFixture } from '../../../test/util';
+import {
+  loadBinaryFixture,
+  loadFixture,
+  loadJsonFixture,
+} from '../../../test/util';
 import * as rubyVersioning from '../../versioning/ruby';
 import { resetCache } from './get-rubygems-org';
 import * as rubygems from '.';
@@ -8,6 +12,8 @@ import * as rubygems from '.';
 const rubygemsOrgVersions = loadFixture('rubygems-org.txt');
 const railsInfo = loadJsonFixture('rails/info.json');
 const railsVersions = loadJsonFixture('rails/versions.json');
+const railsDependencies = loadBinaryFixture('dependencies-rails.dat');
+const emptyMarshalArray = Buffer.from([4, 8, 91, 0]);
 
 describe('datasource/rubygems/index', () => {
   describe('getReleases', () => {
@@ -149,6 +155,7 @@ describe('datasource/rubygems/index', () => {
       expect(await getPkgReleases(params)).toBeNull();
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
+
     it('falls back to info when version request fails', async () => {
       httpMock
         .scope('https://thirdparty.com/')
@@ -173,5 +180,39 @@ describe('datasource/rubygems/index', () => {
         .reply(500);
       expect(await getPkgReleases(params)).toBeNull();
     });
+
+    it('falls back to dependencies api', async () => {
+      httpMock
+        .scope('https://thirdparty.com/')
+        .get('/api/v1/gems/rails.json')
+        .reply(404, railsInfo)
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, railsDependencies);
+
+      const res = await getPkgReleases(params);
+      expect(res?.releases).toHaveLength(339);
+    });
+
+    it('returns null for GitHub Packages package miss', async () => {
+      const newparams = { ...params };
+      newparams.registryUrls = ['https://rubygems.pkg.github.com/example'];
+      httpMock
+        .scope('https://rubygems.pkg.github.com/example')
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, emptyMarshalArray);
+      expect(await getPkgReleases(newparams)).toBeNull();
+    });
+
+    it('returns a dep for GitHub Packages package hit', async () => {
+      const newparams = { ...params };
+      newparams.registryUrls = ['https://rubygems.pkg.github.com/example'];
+      httpMock
+        .scope('https://rubygems.pkg.github.com/example')
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, railsDependencies);
+      const res = await getPkgReleases(newparams);
+      expect(res.releases).toHaveLength(339);
+      expect(res).toMatchSnapshot();
+    });
   });
 });

lib/datasource/rubygems/releases.ts

@@ -1,14 +1,18 @@
+import { parseUrl } from '../../util/url';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { getDependency } from './get';
+import { knownFallbackHosts } from './common';
+import { getDependency, getDependencyFallback } from './get';
 import { getRubygemsOrgDependency } from './get-rubygems-org';
 
 export function getReleases({
   lookupName,
   registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  // prettier-ignore
+  if (parseUrl(registryUrl)?.hostname === 'rubygems.org') {
-  if (registryUrl.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization]
+    return getRubygemsOrgDependency(lookupName);
-      return getRubygemsOrgDependency(lookupName);
+  }
-    }
+  if (knownFallbackHosts.includes(parseUrl(registryUrl)?.hostname)) {
+    return getDependencyFallback(lookupName, registryUrl);
+  }
   return getDependency(lookupName, registryUrl);
 }

lib/datasource/rubygems/types.ts

@@ -0,0 +1,44 @@
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1dependenciesgemscomma-delimited-gem-names
+ */
+export interface MarshalledVersionInfo {
+  name: string;
+  number: string;
+  platform: string;
+  dependencies: MarshalledDependency[];
+}
+
+export type MarshalledDependency = [name: string, version: string];
+
+export interface JsonGemDependency {
+  name: string;
+  requirements: string;
+}
+
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1gemsgem-namejsonyaml
+ */
+export interface JsonGemsInfo {
+  // FIXME: This property doesn't exist in api
+  changelog_uri: string;
+  dependencies: {
+    development: JsonGemDependency;
+    runtime: JsonGemDependency;
+  };
+  homepage_uri: string;
+  name: string;
+  platform?: string;
+  source_code_uri: string;
+  version?: string;
+}
+
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1versionsgem-namejsonyaml
+ */
+export interface JsonGemVersions {
+  created_at: string;
+  number: string;
+  platform: string;
+  rubygems_version: string;
+  ruby_version: string;
+}

lib/types/marshal.d.ts

@@ -0,0 +1,17 @@
+declare module 'marshal' {
+  class Marshal {
+    public parsed?: unknown;
+
+    constructor();
+    constructor(buffer: Buffer);
+    constructor(buffer: string, encoding: BufferEncoding);
+
+    public load(buffer: Buffer): this;
+    public load(buffer: string, encoding: BufferEncoding): this;
+
+    public toString(encoding?: BufferEncoding): string;
+    public toJSON(): unknown;
+  }
+
+  export = Marshal;
+}

lib/util/http/index.ts

@@ -10,7 +10,12 @@ import { resolveBaseUrl } from '../url';
 import { applyAuthorization, removeAuthorization } from './auth';
 import { applyHostRules } from './host-rules';
 import { getQueue } from './queue';
-import type { GotOptions, OutgoingHttpHeaders, RequestStats } from './types';
+import type {
+  GotJSONOptions,
+  GotOptions,
+  OutgoingHttpHeaders,
+  RequestStats,
+} from './types';
 
 // TODO: refactor code to remove this (#9651)
 import './legacy';
@@ -31,7 +36,7 @@ export interface HttpPostOptions extends HttpOptions {
 
 export interface InternalHttpOptions extends HttpOptions {
   json?: Record<string, unknown>;
-  responseType?: 'json';
+  responseType?: 'json' | 'buffer';
   method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
 }
 
@@ -69,6 +74,11 @@ function applyDefaultHeaders(options: Options): void {
   };
 }
 
+// Note on types:
+// options.requestType can be either 'json' or 'buffer', but `T` should be
+// `Buffer` in the latter case.
+// We don't declare overload signatures because it's immediately wrapped by
+// `request`.
 async function gotRoutine<T>(
   url: string,
   options: GotOptions,
@@ -76,7 +86,9 @@ async function gotRoutine<T>(
 ): Promise<Response<T>> {
   logger.trace({ url, options }, 'got request');
 
-  const resp = await got<T>(url, options);
+  // Cheat the TS compiler using `as` to pick a specific overload.
+  // Otherwise it doesn't typecheck.
+  const resp = await got<T>(url, options as GotJSONOptions);
   const duration = resp.timings.phases.total || 0;
 
   const httpRequests = memCache.get('http-requests') || [];
@@ -172,6 +184,23 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
     return this.request<string>(url, { ...options, method: 'head' });
   }
 
+  protected requestBuffer(
+    url: string | URL,
+    httpOptions?: InternalHttpOptions
+  ): Promise<HttpResponse<Buffer> | null> {
+    return this.request<Buffer>(url, {
+      ...httpOptions,
+      responseType: 'buffer',
+    });
+  }
+
+  getBuffer(
+    url: string,
+    options: HttpOptions = {}
+  ): Promise<HttpResponse<Buffer> | null> {
+    return this.requestBuffer(url, options);
+  }
+
   private async requestJson<T = unknown>(
     url: string,
     options: InternalHttpOptions

lib/util/http/types.ts

@@ -1,11 +1,19 @@
-import { OptionsOfJSONResponseBody, RequestError as RequestError_ } from 'got';
+import {
+  OptionsOfBufferResponseBody,
+  OptionsOfJSONResponseBody,
+  RequestError as RequestError_,
+} from 'got';
 
 export type GotContextOptions = {
   authType?: string;
 } & Record<string, unknown>;
 
 // TODO: Move options to context
-export type GotOptions = OptionsOfJSONResponseBody & {
+export type GotOptions = GotBufferOptions | GotJSONOptions;
+export type GotBufferOptions = OptionsOfBufferResponseBody & GotExtraOptions;
+export type GotJSONOptions = OptionsOfJSONResponseBody & GotExtraOptions;
+
+export type GotExtraOptions = {
   abortOnError?: boolean;
   abortIgnoreStatusCodes?: number[];
   token?: string;

package.json

@@ -167,6 +167,7 @@
     "luxon": "2.0.2",
     "markdown-it": "12.2.0",
     "markdown-table": "2.0.0",
+    "marshal": "0.5.2",
     "minimatch": "3.0.4",
     "moo": "0.5.1",
     "node-html-parser": "3.3.6",

test/util.ts

@@ -80,6 +80,14 @@ export function getFixturePath(fixtureFile: string, fixtureRoot = '.'): string {
   return upath.join(callerDir, fixtureRoot, '__fixtures__', fixtureFile);
 }
 
+export function loadBinaryFixture(
+  fixtureFile: string,
+  fixtureRoot = '.'
+): Buffer {
+  const fixtureAbsFile = getFixturePath(fixtureFile, fixtureRoot);
+  return readFileSync(fixtureAbsFile);
+}
+
 export function loadFixture(fixtureFile: string, fixtureRoot = '.'): string {
   const fixtureAbsFile = getFixturePath(fixtureFile, fixtureRoot);
   return readFileSync(fixtureAbsFile, { encoding: 'utf8' });

yarn.lock

@@ -3381,6 +3381,13 @@ debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1:
   dependencies:
     ms "2.1.2"
 
+debug@4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
+  integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==
+  dependencies:
+    ms "^2.1.1"
+
 debug@^2.2.0, debug@^2.3.3, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -6512,6 +6519,13 @@ marked@^2.0.0:
   resolved "https://registry.yarnpkg.com/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753"
   integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA==
 
+marshal@0.5.2:
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/marshal/-/marshal-0.5.2.tgz#4a6e6a20c5f59053a5b86d7fac7ad28081214b36"
+  integrity sha512-f6zOFkXq8k8AJbACRR06s4RFCP19ugCFLn9fX4v01yinL634v8gzSf8mVVHrXFhn6GpMKXR0n8LU2wIUL+c32Q==
+  dependencies:
+    debug "4.1.1"
+
 matcher@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/matcher/-/matcher-3.0.0.tgz#bd9060f4c5b70aa8041ccc6f80368760994f30ca"