fix(maven): don't cache private (#9038)

lib/datasource/maven/__snapshots__/index.spec.ts.snap

@@ -0,0 +1,34 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`datasource/maven getReleases should return all versions from a custom repository 1`] = `
+Object {
+  "display": "mysql:mysql-connector-java",
+  "group": "mysql",
+  "homepage": "http://dev.mysql.com/doc/connector-j/en/",
+  "name": "mysql-connector-java",
+  "releases": Array [
+    Object {
+      "version": "6.0.5",
+    },
+    Object {
+      "version": "6.0.6",
+    },
+    Object {
+      "version": "8.0.7",
+    },
+    Object {
+      "version": "8.0.8",
+    },
+    Object {
+      "version": "8.0.9",
+    },
+    Object {
+      "version": "8.0.11",
+    },
+    Object {
+      "version": "8.0.12",
+    },
+  ],
+  "sourceUrl": "https://github.com/mysql/mysql-connector-j",
+}
+`;

lib/datasource/maven/index.spec.ts

@@ -54,6 +54,11 @@ describe('datasource/maven', () => {
       password: 'password',
       timeout: 20000,
     });
+    hostRules.add({
+      hostType: datasource,
+      hostName: 'custom.registry.renovatebot.com',
+      token: 'abc123',
+    });
     jest.resetAllMocks();
     nock.cleanAll();
     nock.disableNetConnect();
@@ -65,6 +70,12 @@ describe('datasource/maven', () => {
         '/maven2/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom'
       )
       .reply(200, MYSQL_MAVEN_MYSQL_POM);
+    nock('https://custom.registry.renovatebot.com')
+      .get('/mysql/mysql-connector-java/maven-metadata.xml')
+      .reply(200, MYSQL_MAVEN_METADATA);
+    nock('https://custom.registry.renovatebot.com')
+      .get('/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom')
+      .reply(200, MYSQL_MAVEN_MYSQL_POM);
     nock('http://failed_repo')
       .get('/mysql/mysql-connector-java/maven-metadata.xml')
       .reply(404, null);
@@ -119,6 +130,7 @@ describe('datasource/maven', () => {
 
   afterEach(() => {
     nock.enableNetConnect();
+    delete process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK;
   });
 
   describe('getReleases', () => {
@@ -180,6 +192,16 @@ describe('datasource/maven', () => {
       expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS, true));
     });
 
+    it('should return all versions from a custom repository', async () => {
+      process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK = 'true';
+      const releases = await getPkgReleases({
+        ...config,
+        depName: 'mysql:mysql-connector-java',
+        registryUrls: ['https://custom.registry.renovatebot.com'],
+      });
+      expect(releases).toMatchSnapshot();
+    });
+
     it('should return all versions of a specific library if a repository fails', async () => {
       const releases = await getPkgReleases({
         ...config,

lib/datasource/maven/index.ts

@@ -37,36 +37,44 @@ function getMavenUrl(
   return new url.URL(`${dependency.dependencyUrl}/${path}`, repoUrl);
 }
 
+interface MavenXml {
+  authorization?: boolean;
+  xml?: XmlDocument;
+}
+
 async function downloadMavenXml(
   pkgUrl: url.URL | null
-): Promise<XmlDocument | null> {
+): Promise<MavenXml | null> {
   /* istanbul ignore if */
   if (!pkgUrl) {
-    return null;
+    return {};
   }
   let rawContent: string;
+  let authorization: boolean;
   switch (pkgUrl.protocol) {
     case 'file:':
       rawContent = await downloadFileProtocol(pkgUrl);
       break;
     case 'http:':
     case 'https:':
-      rawContent = await downloadHttpProtocol(pkgUrl);
+      ({ authorization, body: rawContent } = await downloadHttpProtocol(
+        pkgUrl
+      ));
       break;
     case 's3:':
       logger.debug('Skipping s3 dependency');
-      return null;
+      return {};
     default:
       logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`);
-      return null;
+      return {};
   }
 
   if (!rawContent) {
     logger.debug(`Content is not found for Maven url: ${pkgUrl.toString()}`);
-    return null;
+    return {};
   }
 
-  return new XmlDocument(rawContent);
+  return { authorization, xml: new XmlDocument(rawContent) };
 }
 
 async function getDependencyInfo(
@@ -78,7 +86,7 @@ async function getDependencyInfo(
   const path = `${version}/${dependency.name}-${version}.pom`;
 
   const pomUrl = getMavenUrl(dependency, repoUrl, path);
-  const pomContent = await downloadMavenXml(pomUrl);
+  const { xml: pomContent } = await downloadMavenXml(pomUrl);
   if (!pomContent) {
     return result;
   }
@@ -156,13 +164,17 @@ async function getVersionsFromMetadata(
     return cachedVersions;
   }
 
-  const mavenMetadata = await downloadMavenXml(metadataUrl);
+  const { authorization, xml: mavenMetadata } = await downloadMavenXml(
+    metadataUrl
+  );
   if (!mavenMetadata) {
     return null;
   }
 
   const versions = extractVersions(mavenMetadata);
-  await packageCache.set(cacheNamespace, cacheKey, versions, 30);
+  if (!authorization) {
+    await packageCache.set(cacheNamespace, cacheKey, versions, 30);
+  }
   return versions;
 }
 

lib/datasource/maven/util.ts

@@ -2,7 +2,7 @@ import url from 'url';
 import { HOST_DISABLED } from '../../constants/error-messages';
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
-import { Http } from '../../util/http';
+import { Http, HttpResponse } from '../../util/http';
 
 import { MAVEN_REPO, id } from './common';
 
@@ -57,12 +57,12 @@ function isUnsupportedHostError(err: { name: string }): boolean {
 export async function downloadHttpProtocol(
   pkgUrl: url.URL | string,
   hostType = id
-): Promise<string | null> {
+): Promise<Partial<HttpResponse>> {
-  let raw: { body: string };
+  let raw: HttpResponse;
   try {
     const httpClient = httpByHostType(hostType);
     raw = await httpClient.get(pkgUrl.toString());
-    return raw.body;
+    return raw;
   } catch (err) {
     const failedUrl = pkgUrl.toString();
     if (err.message === HOST_DISABLED) {
@@ -92,7 +92,7 @@ export async function downloadHttpProtocol(
     } else {
       logger.info({ failedUrl, err }, 'Unknown error');
     }
-    return null;
+    return {};
   }
 }
 

lib/datasource/sbt-package/index.ts

@@ -19,7 +19,7 @@ export async function getArtifactSubdirs(
   artifact: string,
   scalaVersion: string
 ): Promise<string[]> {
-  const indexContent = await downloadHttpProtocol(
+  const { body: indexContent } = await downloadHttpProtocol(
     ensureTrailingSlash(searchRoot),
     'sbt'
   );
@@ -59,7 +59,7 @@ export async function getPackageReleases(
     const parseReleases = (content: string): string[] =>
       parseIndexDir(content, (x) => !/^\.+$/.test(x));
     for (const searchSubdir of artifactSubdirs) {
-      const content = await downloadHttpProtocol(
+      const { body: content } = await downloadHttpProtocol(
         ensureTrailingSlash(`${searchRoot}/${searchSubdir}`),
         'sbt'
       );
@@ -109,7 +109,7 @@ export async function getUrls(
 
     for (const pomFileName of pomFileNames) {
       const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`;
-      const content = await downloadHttpProtocol(pomUrl, 'sbt');
+      const { body: content } = await downloadHttpProtocol(pomUrl, 'sbt');
 
       if (content) {
         const pomXml = new XmlDocument(content);

lib/datasource/sbt-plugin/index.ts

@@ -26,7 +26,7 @@ async function resolvePluginReleases(
   const searchRoot = `${rootUrl}/${artifact}`;
   const parse = (content: string): string[] =>
     parseIndexDir(content, (x) => !/^\.+$/.test(x));
-  const indexContent = await downloadHttpProtocol(
+  const { body: indexContent } = await downloadHttpProtocol(
     ensureTrailingSlash(searchRoot),
     'sbt'
   );
@@ -41,7 +41,7 @@ async function resolvePluginReleases(
       : scalaVersions;
     for (const searchVersion of searchVersions) {
       const searchSubRoot = `${searchRoot}/scala_${searchVersion}`;
-      const subRootContent = await downloadHttpProtocol(
+      const { body: subRootContent } = await downloadHttpProtocol(
         ensureTrailingSlash(searchSubRoot),
         'sbt'
       );
@@ -49,7 +49,7 @@ async function resolvePluginReleases(
         const sbtVersionItems = parse(subRootContent);
         for (const sbtItem of sbtVersionItems) {
           const releasesRoot = `${searchSubRoot}/${sbtItem}`;
-          const releasesIndexContent = await downloadHttpProtocol(
+          const { body: releasesIndexContent } = await downloadHttpProtocol(
             ensureTrailingSlash(releasesRoot),
             'sbt'
           );