feat(maven): additional package info fetching (#3146)

lib/datasource/maven/index.js

@@ -3,6 +3,7 @@ const url = require('url');
 const fs = require('fs-extra');
 const { XmlDocument } = require('xmldoc');
 const is = require('@sindresorhus/is');
+const { compare } = require('../../versioning/maven/compare');
 
 module.exports = {
   getPkgReleases,
@@ -22,16 +23,25 @@ async function getPkgReleases(purl) {
   logger.debug(
     `Found ${repositories.length} repositories for ${dependency.display}`
   );
+  const repoForVersions = {};
   for (let i = 0; i < repositories.length; i += 1) {
     const repoUrl = repositories[i];
     logger.debug(
       `Looking up ${dependency.display} in repository #${i} - ${repoUrl}`
     );
-    const mavenMetadata = await downloadMavenMetadata(dependency, repoUrl);
+    const mavenMetadata = await downloadMavenXml(
+      dependency,
+      repoUrl,
+      'maven-metadata.xml'
+    );
     if (mavenMetadata) {
       const newVersions = extractVersions(mavenMetadata).filter(
         version => !versions.includes(version)
       );
+      const latestVersion = getLatestVersion(newVersions);
+      if (latestVersion) {
+        repoForVersions[latestVersion] = repoUrl;
+      }
       versions.push(...newVersions);
       logger.debug(`Found ${newVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
     }
@@ -42,9 +52,17 @@ async function getPkgReleases(purl) {
     return null;
   }
   logger.debug(`Found ${versions.length} versions for ${dependency.display}`);
+  const latestVersion = getLatestVersion(versions);
+  const repoUrl = repoForVersions[latestVersion];
+  const dependencyInfo = await getDependencyInfo(
+    dependency,
+    repoUrl,
+    latestVersion
+  );
 
   return {
     ...dependency,
+    ...dependencyInfo,
     releases: versions.map(v => ({ version: v })),
   };
 }
@@ -59,19 +77,20 @@ function getDependencyParts(purl) {
   };
 }
 
-async function downloadMavenMetadata(dependency, repoUrl) {
+async function downloadMavenXml(dependency, repoUrl, dependencyFilePath) {
   const pkgUrl = new url.URL(
-    `${dependency.dependencyUrl}/maven-metadata.xml`,
+    `${dependency.dependencyUrl}/${dependencyFilePath}`,
     repoUrl
   );
-  let mavenMetadata;
+
+  let rawContent;
   switch (pkgUrl.protocol) {
     case 'file:':
-      mavenMetadata = await downloadFileProtocol(pkgUrl);
+      rawContent = await downloadFileProtocol(pkgUrl);
       break;
     case 'http:':
     case 'https:':
-      mavenMetadata = await downloadHttpProtocol(pkgUrl);
+      rawContent = await downloadHttpProtocol(pkgUrl);
       break;
     default:
       logger.error(
@@ -79,15 +98,22 @@ async function downloadMavenMetadata(dependency, repoUrl) {
       );
       return null;
   }
-  if (!mavenMetadata) {
+
+  if (!rawContent) {
     logger.debug(`${dependency.display} not found in repository ${repoUrl}`);
+    return null;
+  }
+
+  try {
+    return new XmlDocument(rawContent);
+  } catch (e) {
+    logger.debug(`Can not parse ${pkgUrl.href} for ${dependency.display}`);
+    return null;
   }
-  return mavenMetadata;
 }
 
-function extractVersions(mavenMetadata) {
+function extractVersions(metadata) {
-  const doc = new XmlDocument(mavenMetadata);
+  const versions = metadata.descendantWithPath('versioning.versions');
-  const versions = doc.descendantWithPath('versioning.versions');
   const elements = versions && versions.childrenNamed('version');
   if (!elements) return [];
   return elements.map(el => el.val);
@@ -141,3 +167,34 @@ function isTemporalError(err) {
 function isNotFoundError(err) {
   return err.statusCode === 404;
 }
+
+function getLatestVersion(versions) {
+  if (versions.length === 0) return null;
+  return versions.reduce((latestVersion, version) =>
+    compare(version, latestVersion) === 1 ? version : latestVersion
+  );
+}
+
+async function getDependencyInfo(dependency, repoUrl, version) {
+  const result = {};
+  const path = `${version}/${dependency.name}-${version}.pom`;
+
+  const pomContent = await downloadMavenXml(dependency, repoUrl, path);
+  if (!pomContent) return result;
+
+  function containsPlaceholder(str) {
+    return /\${.*?}/g.test(str);
+  }
+
+  const homepage = pomContent.valueWithPath('url');
+  if (homepage && !containsPlaceholder(homepage)) {
+    result.homepage = homepage;
+  }
+
+  const sourceUrl = pomContent.valueWithPath('scm.url');
+  if (sourceUrl && !containsPlaceholder(sourceUrl)) {
+    result.sourceUrl = sourceUrl;
+  }
+
+  return result;
+}

test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom

@@ -0,0 +1,38 @@
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>mysql</groupId>
+  <artifactId>mysql-connector-java</artifactId>
+  <version>8.0.12</version>
+  <packaging>jar</packaging>
+
+  <name>MySQL Connector/J</name>
+  <description>JDBC Type 4 driver for MySQL</description>
+
+  <licenses>
+    <license>
+      <name>The GNU General Public License, v2 with FOSS exception</name>
+      <distribution>repo</distribution>
+      <comments>For detailed license information see the LICENSE file in this distribution.</comments>
+    </license>
+  </licenses>
+
+  <url>http://dev.mysql.com/doc/connector-j/en/</url>
+
+  <scm>
+    <connection>scm:git:git@github.com:mysql/mysql-connector-j.git</connection>
+    <url>https://github.com/mysql/mysql-connector-j</url>
+  </scm>
+
+  <organization>
+    <name>Oracle Corporation</name>
+    <url>http://www.oracle.com</url>
+  </organization>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>2.6.0</version>
+    </dependency>
+  </dependencies>
+</project>

test/datasource/maven.spec.js

@@ -21,6 +21,11 @@ const MYSQL_MAVEN_METADATA = fs.readFileSync(
   'utf8'
 );
 
+const MYSQL_MAVEN_MYSQL_POM = fs.readFileSync(
+  'test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom',
+  'utf8'
+);
+
 const config = {
   versionScheme: 'loose',
 };
@@ -30,9 +35,17 @@ describe('datasource/maven', () => {
     nock('http://central.maven.org')
       .get('/maven2/mysql/mysql-connector-java/maven-metadata.xml')
       .reply(200, MYSQL_MAVEN_METADATA);
+    nock('http://central.maven.org')
+      .get(
+        '/maven2/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom'
+      )
+      .reply(200, MYSQL_MAVEN_MYSQL_POM);
     nock('http://failed_repo')
       .get('/mysql/mysql-connector-java/maven-metadata.xml')
       .reply(404, null);
+    nock('http://empty_repo')
+      .get('/mysql/mysql-connector-java/maven-metadata.xml')
+      .reply(200, 'non-sense');
   });
 
   describe('getPkgReleases', () => {
@@ -88,7 +101,7 @@ describe('datasource/maven', () => {
       const releases = await datasource.getPkgReleases({
         ...config,
         purl:
-          'pkg:maven/mysql/mysql-connector-java?repository_url=http://central.maven.org/maven2/,http://failed_repo/,http://dns_error_repo',
+          'pkg:maven/mysql/mysql-connector-java?repository_url=http://central.maven.org/maven2/,http://failed_repo/,http://dns_error_repo,http://empty_repo',
       });
       expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
     });