feat(maven): S3 Support (#14938)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/datasource/maven/__fixtures__/metadata-s3.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.example</groupId>
+  <artifactId>package</artifactId>
+  <versioning>
+    <latest>1.0.2</latest>
+    <release>1.0.2</release>
+    <versions>
+      <version>0.0.1</version>
+      <version>1.0.0</version>
+      <version>1.0.1</version>
+      <version>1.0.2</version>
+      <version>1.0.3</version>
+    </versions>
+    <lastUpdated>20210101000000</lastUpdated>
+  </versioning>
+</metadata>

lib/modules/datasource/maven/index.ts

@@ -14,7 +14,7 @@ import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
 import { MAVEN_REPO } from './common';
 import type { MavenDependency, ReleaseMap } from './types';
 import {
-  checkHttpResource,
+  checkResource,
   createUrlForDependencyPom,
   downloadHttpProtocol,
   downloadMavenXml,
@@ -249,7 +249,7 @@ export class MavenDatasource extends Datasource {
         const artifactUrl = getMavenUrl(dependency, repoUrl, pomUrl);
         const release: Release = { version };
 
-        const res = await checkHttpResource(this.http, artifactUrl);
+        const res = await checkResource(this.http, artifactUrl);
 
         if (is.date(res)) {
           release.releaseTimestamp = res.toISOString();

lib/modules/datasource/maven/s3.spec.ts

@@ -0,0 +1,204 @@
+import { Readable } from 'stream';
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
+import { DateTime } from 'luxon';
+import { ReleaseResult, getPkgReleases } from '..';
+import { Fixtures } from '../../../../test/fixtures';
+import { logger } from '../../../../test/util';
+import * as hostRules from '../../../util/host-rules';
+import { id as versioning } from '../../versioning/maven';
+import { MavenDatasource } from '.';
+
+const datasource = MavenDatasource.id;
+
+const baseUrlS3 = 's3://repobucket';
+
+function get(
+  depName = 'org.example:package',
+  ...registryUrls: string[]
+): Promise<ReleaseResult | null> {
+  const conf = { versioning, datasource, depName };
+  return getPkgReleases(registryUrls ? { ...conf, registryUrls } : conf);
+}
+
+const meta = Readable.from(
+  Buffer.from(Fixtures.get('metadata-s3.xml'), 'utf-8')
+);
+
+describe('modules/datasource/maven/s3', () => {
+  const s3mock = mockClient(S3Client);
+
+  beforeEach(() => {
+    hostRules.add({
+      hostType: datasource,
+      matchHost: 'custom.registry.renovatebot.com',
+      token: '123test',
+    });
+    jest.resetAllMocks();
+  });
+
+  afterEach(() => {
+    s3mock.reset();
+    hostRules.clear();
+  });
+
+  describe('S3', () => {
+    it('returns releases', async () => {
+      s3mock
+        .on(GetObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/maven-metadata.xml',
+        })
+        .resolvesOnce({ Body: meta })
+        .on(HeadObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/0.0.1/package-0.0.1.pom',
+        })
+        .resolvesOnce({ DeleteMarker: true })
+        .on(HeadObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/1.0.0/package-1.0.0.pom',
+        })
+        .rejectsOnce('NoSuchKey')
+        .on(HeadObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/1.0.1/package-1.0.1.pom',
+        })
+        .rejectsOnce('Unknown')
+        .on(HeadObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/1.0.2/package-1.0.2.pom',
+        })
+        .resolvesOnce({})
+        .on(HeadObjectCommand, {
+          Bucket: 'repobucket',
+          Key: 'org/example/package/1.0.3/package-1.0.3.pom',
+        })
+        .resolvesOnce({
+          LastModified: DateTime.fromISO(`2020-01-01T00:00:00.000Z`).toJSDate(),
+        });
+
+      const res = await get('org.example:package', baseUrlS3);
+
+      expect(res).toEqual({
+        display: 'org.example:package',
+        group: 'org.example',
+        name: 'package',
+        registryUrl: 's3://repobucket',
+        releases: [
+          { version: '1.0.2' },
+          { version: '1.0.3', releaseTimestamp: '2020-01-01T00:00:00.000Z' },
+        ],
+      });
+    });
+
+    describe('errors', () => {
+      it('returns null on auth error', async () => {
+        class CredentialsProviderError extends Error {
+          constructor() {
+            super();
+            this.name = 'CredentialsProviderError';
+          }
+        }
+
+        s3mock
+          .on(GetObjectCommand, {
+            Bucket: 'repobucket',
+            Key: 'org/example/package/maven-metadata.xml',
+          })
+          .rejectsOnce(new CredentialsProviderError());
+
+        const res = await get('org.example:package', baseUrlS3);
+
+        expect(res).toBeNull();
+        expect(logger.logger.debug).toHaveBeenCalledWith(
+          {
+            failedUrl: 's3://repobucket/org/example/package/maven-metadata.xml',
+          },
+          'Dependency lookup authorization failed. Please correct AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars'
+        );
+      });
+
+      it('returns null for incorrect region', async () => {
+        s3mock
+          .on(GetObjectCommand, {
+            Bucket: 'repobucket',
+            Key: 'org/example/package/maven-metadata.xml',
+          })
+          .rejectsOnce('Region is missing');
+
+        const res = await get('org.example:package', baseUrlS3);
+
+        expect(res).toBeNull();
+        expect(logger.logger.debug).toHaveBeenCalledWith(
+          {
+            failedUrl: 's3://repobucket/org/example/package/maven-metadata.xml',
+          },
+          'Dependency lookup failed. Please a correct AWS_REGION env var'
+        );
+      });
+
+      it('returns null for NoSuchKey error', async () => {
+        s3mock
+          .on(GetObjectCommand, {
+            Bucket: 'repobucket',
+            Key: 'org/example/package/maven-metadata.xml',
+          })
+          .rejectsOnce('NoSuchKey');
+
+        const res = await get('org.example:package', baseUrlS3);
+
+        expect(res).toBeNull();
+        expect(logger.logger.trace).toHaveBeenCalledWith(
+          {
+            failedUrl: 's3://repobucket/org/example/package/maven-metadata.xml',
+          },
+          'S3 url not found'
+        );
+      });
+
+      it('returns null for NotFound error', async () => {
+        s3mock
+          .on(GetObjectCommand, {
+            Bucket: 'repobucket',
+            Key: 'org/example/package/maven-metadata.xml',
+          })
+          .rejectsOnce('NotFound');
+
+        const res = await get('org.example:package', baseUrlS3);
+
+        expect(res).toBeNull();
+        expect(logger.logger.trace).toHaveBeenCalledWith(
+          {
+            failedUrl: 's3://repobucket/org/example/package/maven-metadata.xml',
+          },
+          'S3 url not found'
+        );
+      });
+
+      it('returns null for unknown error', async () => {
+        s3mock
+          .on(GetObjectCommand, {
+            Bucket: 'repobucket',
+            Key: 'org/example/package/maven-metadata.xml',
+          })
+          .rejectsOnce('Unknown error');
+
+        const res = await get('org.example:package', baseUrlS3);
+
+        expect(res).toBeNull();
+        expect(logger.logger.debug).toHaveBeenCalledWith(
+          {
+            failedUrl: 's3://repobucket/org/example/package/maven-metadata.xml',
+            message: 'Unknown error',
+          },
+          'Unknown S3 download error'
+        );
+      });
+    });
+  });
+});

lib/modules/datasource/maven/util.spec.ts

@@ -0,0 +1,50 @@
+import { parseUrl } from '../../../util/url';
+import {
+  checkResource,
+  checkS3Resource,
+  downloadMavenXml,
+  downloadS3Protocol,
+} from './util';
+
+describe('modules/datasource/maven/util', () => {
+  describe('downloadMavenXml', () => {
+    it('returns empty object for unsupported protocols', async () => {
+      const res = await downloadMavenXml(
+        null,
+        parseUrl('unsupported://server.com/')
+      );
+      expect(res).toEqual({});
+    });
+
+    it('returns empty object for invalid URLs', async () => {
+      const res = await downloadMavenXml(null, null);
+      expect(res).toEqual({});
+    });
+  });
+
+  describe('downloadS3Protocol', () => {
+    it('returns null for non-S3 URLs', async () => {
+      const res = await downloadS3Protocol(parseUrl('http://not-s3.com/'));
+      expect(res).toBeNull();
+    });
+  });
+
+  describe('checkResource', () => {
+    it('returns not found for unsupported protocols', async () => {
+      const res = await checkResource(null, 'unsupported://server.com/');
+      expect(res).toBe('not-found');
+    });
+
+    it('returns error for invalid URLs', async () => {
+      const res = await checkResource(null, 'not-a-valid-url');
+      expect(res).toBe('error');
+    });
+  });
+
+  describe('checkS3Resource', () => {
+    it('returns error for non-S3 URLs', async () => {
+      const res = await checkS3Resource(parseUrl('http://not-s3.com/'));
+      expect(res).toBe('error');
+    });
+  });
+});

lib/modules/datasource/maven/util.ts

@@ -1,3 +1,5 @@
+import { Blob } from 'buffer';
+import { Readable } from 'stream';
 import { DateTime } from 'luxon';
 import { XmlDocument } from 'xmldoc';
 import { HOST_DISABLED } from '../../../constants/error-messages';
@@ -6,9 +8,10 @@ import { ExternalHostError } from '../../../types/errors/external-host-error';
 import type { Http } from '../../../util/http';
 import type { HttpResponse } from '../../../util/http/types';
 import { regEx } from '../../../util/regex';
+import { getS3Client, parseS3Url } from '../../../util/s3';
+import { streamToString } from '../../../util/streams';
 import { parseUrl } from '../../../util/url';
 import { normalizeDate } from '../metadata';
-
 import type { ReleaseResult } from '../types';
 import { MAVEN_REPO } from './common';
 import type {
@@ -93,15 +96,60 @@ export async function downloadHttpProtocol(
       // istanbul ignore next
       logger.debug({ failedUrl }, 'Unsupported host');
     } else {
-      logger.info({ failedUrl, err }, 'Unknown error');
+      logger.info({ failedUrl, err }, 'Unknown HTTP download error');
     }
     return {};
   }
 }
 
-export async function checkHttpResource(
+function isS3NotFound(err: Error): boolean {
+  return err.message === 'NotFound' || err.message === 'NoSuchKey';
+}
+
+export async function downloadS3Protocol(pkgUrl: URL): Promise<string | null> {
+  logger.trace({ url: pkgUrl.toString() }, `Attempting to load S3 dependency`);
+  try {
+    const s3Url = parseS3Url(pkgUrl);
+    if (s3Url === null) {
+      return null;
+    }
+    const { Body: res } = await getS3Client().getObject(s3Url);
+
+    // istanbul ignore if
+    if (res instanceof Blob) {
+      return res.toString();
+    }
+
+    if (res instanceof Readable) {
+      return streamToString(res);
+    }
+  } catch (err) {
+    const failedUrl = pkgUrl.toString();
+    if (err.name === 'CredentialsProviderError') {
+      logger.debug(
+        { failedUrl },
+        'Dependency lookup authorization failed. Please correct AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars'
+      );
+    } else if (err.message === 'Region is missing') {
+      logger.debug(
+        { failedUrl },
+        'Dependency lookup failed. Please a correct AWS_REGION env var'
+      );
+    } else if (isS3NotFound(err)) {
+      logger.trace({ failedUrl }, `S3 url not found`);
+    } else {
+      logger.debug(
+        { failedUrl, message: err.message },
+        'Unknown S3 download error'
+      );
+    }
+  }
+  return null;
+}
+
+async function checkHttpResource(
   http: Http,
-  pkgUrl: URL | string
+  pkgUrl: URL
 ): Promise<HttpResourceCheckResult> {
   try {
     const res = await http.head(pkgUrl.toString());
@@ -130,6 +178,58 @@ export async function checkHttpResource(
   }
 }
 
+export async function checkS3Resource(
+  pkgUrl: URL
+): Promise<HttpResourceCheckResult> {
+  try {
+    const s3Url = parseS3Url(pkgUrl);
+    if (s3Url === null) {
+      return 'error';
+    }
+    const response = await getS3Client().headObject(s3Url);
+    if (response.DeleteMarker) {
+      return 'not-found';
+    }
+    if (response.LastModified) {
+      return response.LastModified;
+    }
+    return 'found';
+  } catch (err) {
+    if (isS3NotFound(err)) {
+      return 'not-found';
+    } else {
+      logger.debug(
+        { pkgUrl, name: err.name, message: err.message },
+        `Can't check S3 resource existence`
+      );
+    }
+    return 'error';
+  }
+}
+
+export async function checkResource(
+  http: Http,
+  pkgUrl: URL | string
+): Promise<HttpResourceCheckResult> {
+  const parsedUrl = typeof pkgUrl === 'string' ? parseUrl(pkgUrl) : pkgUrl;
+  if (parsedUrl === null) {
+    return 'error';
+  }
+  switch (parsedUrl.protocol) {
+    case 'http:':
+    case 'https:':
+      return await checkHttpResource(http, parsedUrl);
+    case 's3:':
+      return await checkS3Resource(parsedUrl);
+    default:
+      logger.debug(
+        { url: pkgUrl.toString() },
+        `Unsupported Maven protocol in check resource`
+      );
+      return 'not-found';
+  }
+}
+
 function containsPlaceholder(str: string): boolean {
   return regEx(/\${.*?}/g).test(str);
 }
@@ -146,7 +246,6 @@ export async function downloadMavenXml(
   http: Http,
   pkgUrl: URL | null
 ): Promise<MavenXml> {
-  /* istanbul ignore if */
   if (!pkgUrl) {
     return {};
   }
@@ -166,8 +265,8 @@ export async function downloadMavenXml(
       } = await downloadHttpProtocol(http, pkgUrl));
       break;
     case 's3:':
-      logger.debug('Skipping s3 dependency');
+      rawContent = (await downloadS3Protocol(pkgUrl)) ?? undefined;
-      return {};
+      break;
     default:
       logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`);
       return {};

lib/modules/platform/azure/azure-helper.ts

@@ -4,12 +4,12 @@ import {
   GitRef,
 } from 'azure-devops-node-api/interfaces/GitInterfaces.js';
 import { logger } from '../../../logger';
+import { streamToString } from '../../../util/streams';
 import * as azureApi from './azure-got-wrapper';
 import {
   getBranchNameWithoutRefsPrefix,
   getBranchNameWithoutRefsheadsPrefix,
   getNewBranchName,
-  streamToString,
 } from './util';
 
 const mergePolicyGuid = 'fa4e907d-c16b-4a4c-9dfa-4916e5d171ab'; // Magic GUID for merge strategy policy configurations

lib/modules/platform/azure/index.ts

@@ -22,6 +22,7 @@ import * as git from '../../../util/git';
 import * as hostRules from '../../../util/host-rules';
 import { regEx } from '../../../util/regex';
 import { sanitize } from '../../../util/sanitize';
+import { streamToString } from '../../../util/streams';
 import { ensureTrailingSlash } from '../../../util/url';
 import type {
   BranchStatusConfig,
@@ -53,7 +54,6 @@ import {
   getRepoByName,
   getStorageExtraCloneOpts,
   max4000Chars,
-  streamToString,
 } from './util';
 
 interface Config {

lib/modules/platform/azure/util.spec.ts

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { streamToString } from '../../../util/streams';
 import {
   getBranchNameWithoutRefsheadsPrefix,
   getGitStatusContextCombinedName,
@@ -9,7 +10,6 @@ import {
   getRepoByName,
   getStorageExtraCloneOpts,
   max4000Chars,
-  streamToString,
 } from './util';
 
 describe('modules/platform/azure/util', () => {

lib/modules/platform/azure/util.ts

@@ -118,19 +118,6 @@ export function getRenovatePRFormat(azurePr: GitPullRequest): AzurePr {
   } as AzurePr;
 }
 
-export async function streamToString(
-  stream: NodeJS.ReadableStream
-): Promise<string> {
-  const chunks: Uint8Array[] = [];
-
-  const p = await new Promise<string>((resolve, reject) => {
-    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
-    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
-    stream.on('error', (err) => reject(err));
-  });
-  return p;
-}
-
 export function getStorageExtraCloneOpts(config: HostRule): GitOptions {
   let authType: string;
   let authValue: string;

lib/util/s3.spec.ts

@@ -0,0 +1,24 @@
+import { getS3Client, parseS3Url } from './s3';
+
+describe('util/s3', () => {
+  it('parses S3 URLs', () => {
+    expect(parseS3Url('s3://bucket/key/path')).toEqual({
+      Bucket: 'bucket',
+      Key: 'key/path',
+    });
+  });
+
+  it('returns null for non-S3 URLs', () => {
+    expect(parseS3Url('http://example.com/key/path')).toBeNull();
+  });
+
+  it('returns null for invalid URLs', () => {
+    expect(parseS3Url('thisisnotaurl')).toBeNull();
+  });
+
+  it('returns a singleton S3 client instance', () => {
+    const client1 = getS3Client();
+    const client2 = getS3Client();
+    expect(client1).toBe(client2);
+  });
+});

lib/util/s3.ts

@@ -0,0 +1,30 @@
+// Singleton S3 instance initialized on-demand.
+import { S3 } from '@aws-sdk/client-s3';
+import { parseUrl } from './url';
+
+let s3Instance: S3 | undefined;
+export function getS3Client(): S3 {
+  if (!s3Instance) {
+    s3Instance = new S3({});
+  }
+  return s3Instance;
+}
+
+export interface S3UrlParts {
+  Bucket: string;
+  Key: string;
+}
+
+export function parseS3Url(rawUrl: URL | string): S3UrlParts | null {
+  const parsedUrl = typeof rawUrl === 'string' ? parseUrl(rawUrl) : rawUrl;
+  if (parsedUrl === null) {
+    return null;
+  }
+  if (parsedUrl.protocol !== 's3:') {
+    return null;
+  }
+  return {
+    Bucket: parsedUrl.host,
+    Key: parsedUrl.pathname.substring(1),
+  };
+}

lib/util/streams.spec.ts

@@ -0,0 +1,11 @@
+import { Readable } from 'stream';
+import { streamToString } from './streams';
+
+describe('util/streams', () => {
+  describe('streamToString', () => {
+    it('handles Readables', async () => {
+      const res = await streamToString(Readable.from(['abc', 'zxc']));
+      expect(res).toBe('abczxc');
+    });
+  });
+});

lib/util/streams.ts

@@ -0,0 +1,14 @@
+import { Readable } from 'stream';
+
+export async function streamToString(
+  stream: NodeJS.ReadableStream
+): Promise<string> {
+  const readable = Readable.from(stream);
+  const chunks: Uint8Array[] = [];
+  const p = await new Promise<string>((resolve, reject) => {
+    readable.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+    readable.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+    readable.on('error', (err) => reject(err));
+  });
+  return p;
+}

package.json

@@ -134,6 +134,7 @@
   "dependencies": {
     "@aws-sdk/client-ec2": "3.72.0",
     "@aws-sdk/client-ecr": "3.72.0",
+    "@aws-sdk/client-s3": "3.72.0",
     "@breejs/later": "4.1.0",
     "@cheap-glitch/mi-cron": "1.0.1",
     "@iarna/toml": "2.2.5",
@@ -215,7 +216,6 @@
   },
   "devDependencies": {
     "@actions/core": "1.7.0",
-    "@aws-sdk/client-s3": "3.72.0",
     "@jest/globals": "27.5.1",
     "@jest/reporters": "27.5.1",
     "@jest/test-result": "27.5.1",