feat(rubygems): support GitHub Packages

lib/datasource/rubygems/common.ts

@@ -1 +1,40 @@
+import Marshal from 'marshal';
+import urlJoin from 'url-join';
+import { logger } from '../../logger';
+import { Http } from '../../util/http';
+import { getQueryString } from '../../util/url';
+
 export const id = 'rubygems';
+export const http = new Http(id);
+
+export const knownFallbackHosts = ['rubygems.pkg.github.com', 'gitlab.com'];
+
+export async function fetchJson<T>(
+  dependency: string,
+  registry: string,
+  path: string
+): Promise<T> {
+  const url = urlJoin(registry, path, `${dependency}.json`);
+
+  logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+  const response = (await http.getJson<T>(url)) || {
+    body: undefined,
+  };
+
+  return response.body;
+}
+
+export async function fetchBuffer<T>(
+  dependency: string,
+  registry: string,
+  path: string
+): Promise<T> {
+  const url = `${urlJoin(registry, path)}?${getQueryString({
+    gems: dependency,
+  })}`;
+
+  logger.trace({ registry, dependency, url }, `RubyGems lookup request`);
+  const response = await http.getBuffer(url);
+
+  return new Marshal(response.body).parsed as T;
+}

lib/datasource/rubygems/get-rubygems-org.ts

@@ -1,11 +1,8 @@
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import { getElapsedMinutes } from '../../util/date';
-import { Http } from '../../util/http';
 import type { ReleaseResult } from '../types';
-import { id } from './common';
-
-const http = new Http(id);
+import { http } from './common';
 
 let lastSync = new Date('2000-01-01');
 let packageReleases: Record<string, string[]> = Object.create(null); // Because we might need a "constructor" key

lib/datasource/rubygems/get.ts

@@ -1,40 +1,65 @@
-import urlJoin from 'url-join';
 import { logger } from '../../logger';
-import { Http } from '../../util/http';
-import type { OutgoingHttpHeaders } from '../../util/http/types';
-import type { ReleaseResult } from '../types';
-import { id } from './common';
-
-const http = new Http(id);
+import { HttpError } from '../../util/http/types';
+import type { Release, ReleaseResult } from '../types';
+import { fetchBuffer, fetchJson } from './common';
+import type {
+  JsonGemVersions,
+  JsonGemsInfo,
+  MarshalledVersionInfo,
+} from './types';
 
 const INFO_PATH = '/api/v1/gems';
 const VERSIONS_PATH = '/api/v1/versions';
+const DEPENDENCIES_PATH = '/api/v1/dependencies';
 
-const getHeaders = (): OutgoingHttpHeaders => ({ hostType: id });
-
-export async function fetch(
+export async function getDependencyFallback(
   dependency: string,
-  registry: string,
-  path: string
-): Promise<any> {
-  const headers = getHeaders();
-
-  const url = urlJoin(registry, path, `${dependency}.json`);
-
-  logger.trace({ dependency }, `RubyGems lookup request: ${String(url)}`);
-  const response = (await http.getJson(url, { headers })) || {
-    body: undefined,
+  registry: string
+): Promise<ReleaseResult | null> {
+  logger.debug(
+    { dependency, api: DEPENDENCIES_PATH },
+    'RubyGems lookup for dependency'
+  );
+  const info = await fetchBuffer<MarshalledVersionInfo[]>(
+    dependency,
+    registry,
+    DEPENDENCIES_PATH
+  );
+  if (!info || info.length === 0) {
+    return null;
+  }
+  const releases = info.map(({ number: version, platform: rubyPlatform }) => ({
+    version,
+    rubyPlatform,
+  }));
+  return {
+    releases,
+    homepage: null,
+    sourceUrl: null,
+    changelogUrl: null,
   };
-
-  return response.body;
 }
 
 export async function getDependency(
   dependency: string,
   registry: string
 ): Promise<ReleaseResult | null> {
-  logger.debug({ dependency }, 'RubyGems lookup for dependency');
-  const info = await fetch(dependency, registry, INFO_PATH);
+  logger.debug(
+    { dependency, api: INFO_PATH },
+    'RubyGems lookup for dependency'
+  );
+  let info: JsonGemsInfo;
+
+  try {
+    info = await fetchJson(dependency, registry, INFO_PATH);
+  } catch (error) {
+    // fallback to deps api on 404
+    if (error instanceof HttpError && error.response?.statusCode === 404) {
+      return await getDependencyFallback(dependency, registry);
+    }
+    throw error;
+  }
+
   if (!info) {
     logger.debug({ dependency }, 'RubyGems package not found.');
     return null;
@@ -48,10 +73,10 @@ export async function getDependency(
     return null;
   }
 
-  let versions = [];
-  let releases = [];
+  let versions: JsonGemVersions[] = [];
+  let releases: Release[] = [];
   try {
-    versions = await fetch(dependency, registry, VERSIONS_PATH);
+    versions = await fetchJson(dependency, registry, VERSIONS_PATH);
   } catch (err) {
     if (err.statusCode === 400 || err.statusCode === 404) {
       logger.debug(
@@ -63,13 +88,15 @@ export async function getDependency(
     }
   }
 
+  // TODO: invalid properties for `Release` see #11312
+
   if (versions.length === 0 && info.version) {
     logger.warn('falling back to the version from the info endpoint');
     releases = [
       {
         version: info.version,
         rubyPlatform: info.platform,
-      },
+      } as Release,
     ];
   } else {
     releases = versions.map(

lib/datasource/rubygems/index.spec.ts

@@ -1,13 +1,19 @@
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../test/http-mock';
-import { loadFixture, loadJsonFixture } from '../../../test/util';
+import {
+  loadBinaryFixture,
+  loadFixture,
+  loadJsonFixture,
+} from '../../../test/util';
 import * as rubyVersioning from '../../versioning/ruby';
 import { resetCache } from './get-rubygems-org';
 import * as rubygems from '.';
 
 const rubygemsOrgVersions = loadFixture('rubygems-org.txt');
 const railsInfo = loadJsonFixture('rails/info.json');
 const railsVersions = loadJsonFixture('rails/versions.json');
+const railsDependencies = loadBinaryFixture('dependencies-rails.dat');
+const emptyMarshalArray = Buffer.from([4, 8, 91, 0]);
 
 describe('datasource/rubygems/index', () => {
   describe('getReleases', () => {
@@ -149,6 +155,7 @@ describe('datasource/rubygems/index', () => {
       expect(await getPkgReleases(params)).toBeNull();
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
+
     it('falls back to info when version request fails', async () => {
       httpMock
         .scope('https://thirdparty.com/')
@@ -173,5 +180,39 @@ describe('datasource/rubygems/index', () => {
         .reply(500);
       expect(await getPkgReleases(params)).toBeNull();
     });
+
+    it('falls back to dependencies api', async () => {
+      httpMock
+        .scope('https://thirdparty.com/')
+        .get('/api/v1/gems/rails.json')
+        .reply(404, railsInfo)
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, railsDependencies);
+
+      const res = await getPkgReleases(params);
+      expect(res?.releases).toHaveLength(339);
+    });
+
+    it('returns null for GitHub Packages package miss', async () => {
+      const newparams = { ...params };
+      newparams.registryUrls = ['https://rubygems.pkg.github.com/example'];
+      httpMock
+        .scope('https://rubygems.pkg.github.com/example')
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, emptyMarshalArray);
+      expect(await getPkgReleases(newparams)).toBeNull();
+    });
+
+    it('returns a dep for GitHub Packages package hit', async () => {
+      const newparams = { ...params };
+      newparams.registryUrls = ['https://rubygems.pkg.github.com/example'];
+      httpMock
+        .scope('https://rubygems.pkg.github.com/example')
+        .get('/api/v1/dependencies?gems=rails')
+        .reply(200, railsDependencies);
+      const res = await getPkgReleases(newparams);
+      expect(res.releases).toHaveLength(339);
+      expect(res).toMatchSnapshot();
+    });
   });
 });

lib/datasource/rubygems/releases.ts

@@ -1,14 +1,18 @@
+import { parseUrl } from '../../util/url';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
-import { getDependency } from './get';
+import { knownFallbackHosts } from './common';
+import { getDependency, getDependencyFallback } from './get';
 import { getRubygemsOrgDependency } from './get-rubygems-org';
 
 export function getReleases({
   lookupName,
   registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
-  // prettier-ignore
-  if (registryUrl.endsWith('rubygems.org')) { // lgtm [js/incomplete-url-substring-sanitization]
-      return getRubygemsOrgDependency(lookupName);
-    }
+  if (parseUrl(registryUrl)?.hostname === 'rubygems.org') {
+    return getRubygemsOrgDependency(lookupName);
+  }
+  if (knownFallbackHosts.includes(parseUrl(registryUrl)?.hostname)) {
+    return getDependencyFallback(lookupName, registryUrl);
+  }
   return getDependency(lookupName, registryUrl);
 }

lib/datasource/rubygems/types.ts

@@ -0,0 +1,44 @@
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1dependenciesgemscomma-delimited-gem-names
+ */
+export interface MarshalledVersionInfo {
+  name: string;
+  number: string;
+  platform: string;
+  dependencies: MarshalledDependency[];
+}
+
+export type MarshalledDependency = [name: string, version: string];
+
+export interface JsonGemDependency {
+  name: string;
+  requirements: string;
+}
+
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1gemsgem-namejsonyaml
+ */
+export interface JsonGemsInfo {
+  // FIXME: This property doesn't exist in api
+  changelog_uri: string;
+  dependencies: {
+    development: JsonGemDependency;
+    runtime: JsonGemDependency;
+  };
+  homepage_uri: string;
+  name: string;
+  platform?: string;
+  source_code_uri: string;
+  version?: string;
+}
+
+/**
+ * see https://guides.rubygems.org/rubygems-org-api/#get---apiv1versionsgem-namejsonyaml
+ */
+export interface JsonGemVersions {
+  created_at: string;
+  number: string;
+  platform: string;
+  rubygems_version: string;
+  ruby_version: string;
+}

lib/types/marshal.d.ts

@@ -0,0 +1,17 @@
+declare module 'marshal' {
+  class Marshal {
+    public parsed?: unknown;
+
+    constructor();
+    constructor(buffer: Buffer);
+    constructor(buffer: string, encoding: BufferEncoding);
+
+    public load(buffer: Buffer): this;
+    public load(buffer: string, encoding: BufferEncoding): this;
+
+    public toString(encoding?: BufferEncoding): string;
+    public toJSON(): unknown;
+  }
+
+  export = Marshal;
+}