Merge ae76089e03 into adede1d309

lib/modules/datasource/maven/index.spec.ts

@@ -1,4 +1,6 @@
-import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { Readable } from 'node:stream';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { sdkStreamMixin } from '@smithy/util-stream';
 import { mockClient } from 'aws-sdk-client-mock';
 import { GoogleAuth as _googleAuth } from 'google-auth-library';
 import { DateTime } from 'luxon';
@@ -622,10 +624,7 @@ describe('modules/datasource/maven/index', () => {
 
   describe('post-fetch release validation', () => {
     it('returns null for 404', async () => {
-      httpMock
+      httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(404);
-        .scope(MAVEN_REPO)
-        .head('/foo/bar/1.2.3/bar-1.2.3.pom')
-        .reply(404);
 
       const res = await postprocessRelease(
         { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
@@ -635,25 +634,23 @@ describe('modules/datasource/maven/index', () => {
       expect(res).toBeNull();
     });
 
-    it('returns null for unknown error', async () => {
+    it('returns original value for unknown error', async () => {
       httpMock
         .scope(MAVEN_REPO)
-        .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+        .get('/foo/bar/1.2.3/bar-1.2.3.pom')
         .replyWithError('unknown error');
 
+      const releaseOrig: Release = { version: '1.2.3' };
       const res = await postprocessRelease(
         { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
-        { version: '1.2.3' },
+        releaseOrig,
       );
 
-      expect(res).toBeNull();
+      expect(res).toBe(releaseOrig);
     });
 
     it('returns original value for 200 response', async () => {
-      httpMock
+      httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(200);
-        .scope(MAVEN_REPO)
-        .head('/foo/bar/1.2.3/bar-1.2.3.pom')
-        .reply(200);
       const releaseOrig: Release = { version: '1.2.3' };
 
       const res = await postprocessRelease(
@@ -665,10 +662,7 @@ describe('modules/datasource/maven/index', () => {
     });
 
     it('returns original value for 200 response with versionOrig', async () => {
-      httpMock
+      httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(200);
-        .scope(MAVEN_REPO)
-        .head('/foo/bar/1.2.3/bar-1.2.3.pom')
-        .reply(200);
       const releaseOrig: Release = { version: '1.2', versionOrig: '1.2.3' };
 
       const res = await postprocessRelease(
@@ -683,13 +677,13 @@ describe('modules/datasource/maven/index', () => {
       const releaseOrig: Release = { version: '1.2.3' };
       expect(
         await postprocessRelease(
-          { datasource, registryUrl: MAVEN_REPO },
+          { datasource, registryUrl: MAVEN_REPO }, // packageName is missing
           releaseOrig,
         ),
       ).toBe(releaseOrig);
       expect(
         await postprocessRelease(
-          { datasource, packageName: 'foo:bar' },
+          { datasource, packageName: 'foo:bar' }, // registryUrl is missing
           releaseOrig,
         ),
       ).toBe(releaseOrig);
@@ -698,7 +692,7 @@ describe('modules/datasource/maven/index', () => {
     it('adds releaseTimestamp', async () => {
       httpMock
         .scope(MAVEN_REPO)
-        .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+        .get('/foo/bar/1.2.3/bar-1.2.3.pom')
         .reply(200, '', { 'Last-Modified': '2024-01-01T00:00:00.000Z' });
 
       const res = await postprocessRelease(
@@ -719,13 +713,22 @@ describe('modules/datasource/maven/index', () => {
         s3mock.reset();
       });
 
+      function body(input: string): ReturnType<typeof sdkStreamMixin> {
+        const result = new Readable();
+        result.push(input);
+        result.push(null);
+        return sdkStreamMixin(result);
+      }
+
       it('checks package', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
-          .resolvesOnce({});
+          .resolvesOnce({
+            Body: body('foo'),
+          });
 
         const res = await postprocessRelease(
           { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
@@ -737,11 +740,12 @@ describe('modules/datasource/maven/index', () => {
 
       it('supports timestamp', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
           .resolvesOnce({
+            Body: body('foo'),
             LastModified: DateTime.fromISO(
               '2024-01-01T00:00:00.000Z',
             ).toJSDate(),
@@ -760,7 +764,7 @@ describe('modules/datasource/maven/index', () => {
 
       it('returns null for deleted object', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
@@ -778,7 +782,7 @@ describe('modules/datasource/maven/index', () => {
 
       it('returns null for NotFound response', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
@@ -796,7 +800,7 @@ describe('modules/datasource/maven/index', () => {
 
       it('returns null for NoSuchKey response', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
@@ -812,9 +816,9 @@ describe('modules/datasource/maven/index', () => {
         expect(res).toBeNull();
       });
 
-      it('returns null for unknown error', async () => {
+      it('returns original value for any other error', async () => {
         s3mock
-          .on(HeadObjectCommand, {
+          .on(GetObjectCommand, {
             Bucket: 'bucket',
             Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
           })
@@ -827,7 +831,7 @@ describe('modules/datasource/maven/index', () => {
           releaseOrig,
         );
 
-        expect(res).toBeNull();
+        expect(res).toBe(releaseOrig);
       });
     });
   });

lib/modules/datasource/maven/index.ts

@@ -1,9 +1,9 @@
-import is from '@sindresorhus/is';
 import type { XmlDocument } from 'xmldoc';
 import { GlobalConfig } from '../../../config/global';
 import { logger } from '../../../logger';
 import * as packageCache from '../../../util/cache/package';
 import { cache } from '../../../util/cache/package/decorator';
+import { Result } from '../../../util/result';
 import { ensureTrailingSlash } from '../../../util/url';
 import mavenVersion from '../../versioning/maven';
 import * as mavenVersioning from '../../versioning/maven';
@@ -18,10 +18,10 @@ import type {
   ReleaseResult,
 } from '../types';
 import { MAVEN_REPO } from './common';
-import type { MavenDependency } from './types';
+import type { MavenDependency, MavenFetchError } from './types';
 import {
-  checkResource,
   createUrlForDependencyPom,
+  downloadMaven,
   downloadMavenXml,
   getDependencyInfo,
   getDependencyParts,
@@ -93,24 +93,29 @@ export class MavenDatasource extends Datasource {
       return cachedVersions;
     }
 
-    const { isCacheable, xml: mavenMetadata } = await downloadMavenXml(
+    const metadataXmlResult = await downloadMavenXml(this.http, metadataUrl);
-      this.http,
+    return metadataXmlResult
-      metadataUrl,
+      .transform(
-    );
+        async ({ isCacheable, data: mavenMetadata }): Promise<string[]> => {
-    if (!mavenMetadata) {
-      return [];
-    }
-
           const versions = extractVersions(mavenMetadata);
           const cachePrivatePackages = GlobalConfig.get(
             'cachePrivatePackages',
             false,
           );
+
           if (cachePrivatePackages || isCacheable) {
             await packageCache.set(cacheNamespace, cacheKey, versions, 30);
           }
 
           return versions;
+        },
+      )
+      .onError((err) => {
+        logger.debug(
+          `Maven: error fetching versions for "${dependency.display}": ${err.type}`,
+        );
+      })
+      .unwrapOr([]);
   }
 
   async getReleases({
@@ -190,17 +195,22 @@ export class MavenDatasource extends Datasource {
     );
 
     const artifactUrl = getMavenUrl(dependency, registryUrl, pomUrl);
-
+    const fetchResult = await downloadMaven(this.http, artifactUrl);
-    const res = await checkResource(this.http, artifactUrl);
+    return fetchResult
-
+      .transform((res): PostprocessReleaseResult => {
-    if (res === 'not-found' || res === 'error') {
+        if (res.lastModified) {
-      return 'reject';
+          release.releaseTimestamp = res.lastModified;
-    }
-
-    if (is.date(res)) {
-      release.releaseTimestamp = res.toISOString();
         }
 
         return release;
+      })
+      .catch((err): Result<PostprocessReleaseResult, MavenFetchError> => {
+        if (err.type === 'not-found') {
+          return Result.ok('reject');
+        }
+
+        return Result.ok(release);
+      })
+      .unwrapOr(release);
   }
 }

lib/modules/datasource/maven/types.ts

@@ -1,4 +1,3 @@
-import type { XmlDocument } from 'xmldoc';
 import type { Result } from '../../../util/result';
 import type { ReleaseResult } from '../types';
 
@@ -9,13 +8,6 @@ export interface MavenDependency {
   dependencyUrl: string;
 }
 
-export interface MavenXml {
-  isCacheable?: boolean;
-  xml?: XmlDocument;
-}
-
-export type HttpResourceCheckResult = 'found' | 'not-found' | 'error' | Date;
-
 export type DependencyInfo = Pick<
   ReleaseResult,
   'homepage' | 'sourceUrl' | 'packageScope'
@@ -41,6 +33,7 @@ export type MavenFetchError =
   | { type: 'unsupported-protocol' }
   | { type: 'credentials-error' }
   | { type: 'missing-aws-region' }
+  | { type: 'xml-parse-error'; err: Error }
   | { type: 'unknown'; err: Error };
 
 export type MavenFetchResult<T = string> = Result<

lib/modules/datasource/maven/util.spec.ts

@@ -4,7 +4,6 @@ import { HOST_DISABLED } from '../../../constants/error-messages';
 import { Http, HttpError } from '../../../util/http';
 import type { MavenFetchError } from './types';
 import {
-  checkResource,
   downloadHttpProtocol,
   downloadMavenXml,
   downloadS3Protocol,
@@ -46,17 +45,36 @@ function httpError({
 
 describe('modules/datasource/maven/util', () => {
   describe('downloadMavenXml', () => {
-    it('returns empty object for unsupported protocols', async () => {
+    it('returns error for unsupported protocols', async () => {
       const res = await downloadMavenXml(
         http,
         new URL('unsupported://server.com/'),
       );
-      expect(res).toEqual({});
+      expect(res.unwrap()).toEqual({
+        ok: false,
+        err: { type: 'unsupported-protocol' } satisfies MavenFetchError,
+      });
+    });
+
+    it('returns error for xml parse error', async () => {
+      const http = partial<Http>({
+        get: () =>
+          Promise.resolve({
+            statusCode: 200,
+            body: 'invalid xml',
+            headers: {},
+          }),
+      });
+      const res = await downloadMavenXml(http, 'https://example.com/');
+      expect(res.unwrap()).toEqual({
+        ok: false,
+        err: { type: 'xml-parse-error', err: expect.any(Error) },
+      });
     });
   });
 
   describe('downloadS3Protocol', () => {
-    it('fails for non-S3 URLs', async () => {
+    it('returns error for non-S3 URLs', async () => {
       const res = await downloadS3Protocol(new URL('http://not-s3.com/'));
       expect(res.unwrap()).toEqual({
         ok: false,
@@ -122,16 +140,4 @@ describe('modules/datasource/maven/util', () => {
       });
     });
   });
-
-  describe('checkResource', () => {
-    it('returns not found for unsupported protocols', async () => {
-      const res = await checkResource(http, 'unsupported://server.com/');
-      expect(res).toBe('not-found');
-    });
-
-    it('returns error for invalid URLs', async () => {
-      const res = await checkResource(http, 'not-a-valid-url');
-      expect(res).toBe('error');
-    });
-  });
 });

lib/modules/datasource/maven/util.ts

@@ -1,6 +1,5 @@
 import { Readable } from 'node:stream';
-import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
+import { GetObjectCommand } from '@aws-sdk/client-s3';
-import { DateTime } from 'luxon';
 import { XmlDocument } from 'xmldoc';
 import { HOST_DISABLED } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
@@ -9,7 +8,6 @@ import { type Http, HttpError } from '../../../util/http';
 import type { HttpOptions, HttpResponse } from '../../../util/http/types';
 import { regEx } from '../../../util/regex';
 import { Result } from '../../../util/result';
-import type { S3UrlParts } from '../../../util/s3';
 import { getS3Client, parseS3Url } from '../../../util/s3';
 import { streamToString } from '../../../util/streams';
 import { ensureTrailingSlash, parseUrl } from '../../../util/url';
@@ -18,11 +16,9 @@ import { getGoogleAuthToken } from '../util';
 import { MAVEN_REPO } from './common';
 import type {
   DependencyInfo,
-  HttpResourceCheckResult,
   MavenDependency,
   MavenFetchResult,
   MavenFetchSuccess,
-  MavenXml,
 } from './types';
 
 function getHost(url: string): string | null {
@@ -268,92 +264,6 @@ export async function downloadArtifactRegistryProtocol(
   return downloadHttpProtocol(http, url, opts);
 }
 
-async function checkHttpResource(
-  http: Http,
-  pkgUrl: URL,
-): Promise<HttpResourceCheckResult> {
-  try {
-    const res = await http.head(pkgUrl.toString());
-    const timestamp = res?.headers?.['last-modified'];
-    if (timestamp) {
-      const isoTimestamp = normalizeDate(timestamp);
-      if (isoTimestamp) {
-        const releaseDate = DateTime.fromISO(isoTimestamp, {
-          zone: 'UTC',
-        }).toJSDate();
-        return releaseDate;
-      }
-    }
-    return 'found';
-  } catch (err) {
-    if (isNotFoundError(err)) {
-      return 'not-found';
-    }
-
-    const failedUrl = pkgUrl.toString();
-    logger.debug(
-      { failedUrl, statusCode: err.statusCode },
-      `Can't check HTTP resource existence`,
-    );
-    return 'error';
-  }
-}
-
-export async function checkS3Resource(
-  s3Url: S3UrlParts,
-): Promise<HttpResourceCheckResult> {
-  try {
-    const response = await getS3Client().send(new HeadObjectCommand(s3Url));
-    if (response.DeleteMarker) {
-      return 'not-found';
-    }
-    if (response.LastModified) {
-      return response.LastModified;
-    }
-    return 'found';
-  } catch (err) {
-    if (isS3NotFound(err)) {
-      return 'not-found';
-    } else {
-      logger.debug(
-        {
-          bucket: s3Url.Bucket,
-          key: s3Url.Key,
-          name: err.name,
-          message: err.message,
-        },
-        `Can't check S3 resource existence`,
-      );
-    }
-    return 'error';
-  }
-}
-
-export async function checkResource(
-  http: Http,
-  pkgUrl: URL | string,
-): Promise<HttpResourceCheckResult> {
-  const parsedUrl = typeof pkgUrl === 'string' ? parseUrl(pkgUrl) : pkgUrl;
-  if (parsedUrl === null) {
-    return 'error';
-  }
-
-  const s3Url = parseS3Url(parsedUrl);
-  if (s3Url) {
-    return await checkS3Resource(s3Url);
-  }
-
-  if (parsedUrl.protocol === 'http:' || parsedUrl.protocol === 'https:') {
-    return await checkHttpResource(http, parsedUrl);
-  }
-
-  logger.debug(
-    { url: pkgUrl.toString() },
-    `Unsupported Maven protocol in check resource`,
-  );
-  return 'not-found';
-}
-
 function containsPlaceholder(str: string): boolean {
   return regEx(/\${.*?}/g).test(str);
 }
@@ -369,44 +279,57 @@ export function getMavenUrl(
   );
 }
 
-export async function downloadMavenXml(
+export async function downloadMaven(
   http: Http,
-  pkgUrl: URL,
+  url: URL | string,
-): Promise<MavenXml> {
+): Promise<MavenFetchResult> {
-  const protocol = pkgUrl.protocol;
+  const pkgUrl = url instanceof URL ? url : parseUrl(url);
-
+  // istanbul ignore if
-  if (protocol === 'http:' || protocol === 'https:') {
+  if (!pkgUrl) {
-    const rawResult = await downloadHttpProtocol(http, pkgUrl);
+    return Result.err({ type: 'invalid-url' });
-    const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
-      const xml = new XmlDocument(data);
-      return { isCacheable, xml };
-    });
-    return xmlResult.unwrapOr({});
   }
 
-  if (protocol === 'artifactregistry:') {
+  const protocol = pkgUrl.protocol.slice(0, -1);
-    const rawResult = await downloadArtifactRegistryProtocol(http, pkgUrl);
+
-    const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
+  let result: MavenFetchResult = Result.err({ type: 'unsupported-protocol' });
-      const xml = new XmlDocument(data);
+
-      return { isCacheable, xml };
+  if (protocol === 'http' || protocol === 'https') {
-    });
+    result = await downloadHttpProtocol(http, pkgUrl);
-    return xmlResult.unwrapOr({});
   }
 
-  if (protocol === 's3:') {
+  if (protocol === 'artifactregistry') {
-    const rawResult = await downloadS3Protocol(pkgUrl);
+    result = await downloadArtifactRegistryProtocol(http, pkgUrl);
-    const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
-      const xml = new XmlDocument(data);
-      return { xml };
-    });
-    return xmlResult.unwrapOr({});
   }
 
+  if (protocol === 's3') {
+    result = await downloadS3Protocol(pkgUrl);
+  }
+
+  return result.onError((err) => {
+    if (err.type === 'unsupported-protocol') {
       logger.debug(
         { url: pkgUrl.toString() },
-    `Content is not found for Maven url`,
+        `Maven lookup error: unsupported protocol (${protocol})`,
       );
-  return {};
+    }
+  });
+}
+
+export async function downloadMavenXml(
+  http: Http,
+  url: URL | string,
+): Promise<MavenFetchResult<XmlDocument>> {
+  const rawResult = await downloadMaven(http, url);
+  return rawResult.transform((result): MavenFetchResult<XmlDocument> => {
+    try {
+      return Result.ok({
+        ...result,
+        data: new XmlDocument(result.data),
+      });
+    } catch (err) {
+      return Result.err({ type: 'xml-parse-error', err });
+    }
+  });
 }
 
 export function getDependencyParts(packageName: string): MavenDependency {
@@ -458,13 +381,14 @@ async function getSnapshotFullVersion(
     `${version}/maven-metadata.xml`,
   );
 
-  const { xml: mavenMetadata } = await downloadMavenXml(http, metadataUrl);
+  const metadataXmlResult = await downloadMavenXml(http, metadataUrl);
-  // istanbul ignore if: hard to test
-  if (!mavenMetadata) {
-    return null;
-  }
 
-  return extractSnapshotVersion(mavenMetadata);
+  return metadataXmlResult
+    .transform(({ data }) => {
+      const nullErr = { type: 'snapshot-extract-error' };
+      return Result.wrapNullable(extractSnapshotVersion(data), nullErr);
+    })
+    .unwrapOrNull();
 }
 
 function isSnapshotVersion(version: string): boolean {
@@ -508,7 +432,6 @@ export async function getDependencyInfo(
   version: string,
   recursionLimit = 5,
 ): Promise<DependencyInfo> {
-  const result: DependencyInfo = {};
   const path = await createUrlForDependencyPom(
     http,
     version,
@@ -517,11 +440,10 @@ export async function getDependencyInfo(
   );
 
   const pomUrl = getMavenUrl(dependency, repoUrl, path);
-  const { xml: pomContent } = await downloadMavenXml(http, pomUrl);
+  const pomXmlResult = await downloadMavenXml(http, pomUrl);
-  // istanbul ignore if
+  const dependencyInfoResult = await pomXmlResult.transform(
-  if (!pomContent) {
+    async ({ data: pomContent }) => {
-    return result;
+      const result: DependencyInfo = {};
-  }
 
       const homepage = pomContent.valueWithPath('url');
       if (homepage && !containsPlaceholder(homepage)) {
@@ -549,7 +471,11 @@ export async function getDependencyInfo(
       }
 
       const parent = pomContent.childNamed('parent');
-  if (recursionLimit > 0 && parent && (!result.sourceUrl || !result.homepage)) {
+      if (
+        recursionLimit > 0 &&
+        parent &&
+        (!result.sourceUrl || !result.homepage)
+      ) {
         // if we found a parent and are missing some information
         // trying to get the scm/homepage information from it
         const [parentGroupId, parentArtifactId, parentVersion] = [
@@ -577,4 +503,8 @@ export async function getDependencyInfo(
       }
 
       return result;
+    },
+  );
+
+  return dependencyInfoResult.unwrapOr({});
 }

package.json

@@ -269,6 +269,7 @@
     "@openpgp/web-stream-tools": "0.1.3",
     "@renovate/eslint-plugin": "file:tools/eslint",
     "@semantic-release/exec": "6.0.3",
+    "@smithy/util-stream": "3.3.4",
     "@swc/core": "1.10.4",
     "@types/auth-header": "1.0.6",
     "@types/aws4": "1.11.6",

pnpm-lock.yaml

@@ -379,6 +379,9 @@ importers:
       '@semantic-release/exec':
         specifier: 6.0.3
         version: 6.0.3(semantic-release@24.2.0(typescript@5.7.2))
+      '@smithy/util-stream':
+        specifier: 3.3.4
+        version: 3.3.4
       '@swc/core':
         specifier: 1.10.4
         version: 1.10.4