feat(nuget): support changelogs

Refactors nuget lookups to use generatic logic and fill in repositoryUrl to enable changelogs.

lib/datasource/index.js

@@ -2,12 +2,14 @@ const { parse } = require('../util/purl');
 
 const github = require('./github');
 const npm = require('./npm');
+const nuget = require('./nuget');
 const packagist = require('./packagist');
 const pypi = require('./pypi');
 
 const datasources = {
   github,
   npm,
+  nuget,
   packagist,
   pypi,
 };

lib/datasource/nuget.js

@@ -1,52 +1,55 @@
 const got = require('got');
 const xmlParser = require('fast-xml-parser');
+const { isVersion, sortVersions } = require('../versioning/semver');
+const parse = require('github-url-from-git');
 
 module.exports = {
-  getVersions,
-  getNuspec,
+  getDependency,
 };
 
-const map = new Map();
-const headers = {};
-
-async function getVersions(name, retries = 5) {
-  logger.trace(`getVersions(${name})`);
-
-  const url = `https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/index.json`;
-
+async function getDependency(purl) {
+  const { fullname: name } = purl;
+  logger.trace(`nuget.getDependency(${name})`);
+  const pkgUrl = `https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/index.json`;
   try {
-    const result = (await got(url, {
-      cache: process.env.RENOVATE_SKIP_CACHE ? undefined : map,
+    const res = (await got(pkgUrl, {
       json: true,
-      retries,
-      headers,
+      retries: 5,
     })).body;
-
-    return result.versions;
+    const dep = {
+      name,
+    };
+    dep.releases = res.versions
+      .filter(isVersion)
+      .sort(sortVersions)
+      .map(version => ({ version }));
+    // look up nuspec for latest release to get repository
+    const url = `https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/${res.versions.pop()}/${name.toLowerCase()}.nuspec`;
+    try {
+      const result = await got(url);
+      const nuspec = xmlParser.parse(result.body, { ignoreAttributes: false });
+      if (nuspec) {
+        const repositoryUrl = parse(
+          nuspec.package.metadata.repository['@_url']
+        );
+        if (repositoryUrl) {
+          dep.repositoryUrl = repositoryUrl;
+        }
+      }
+    } catch (err) /* istanbul ignore next */ {
+      logger.debug({ depName: name }, 'Error looking up nuspec');
+    }
+    logger.trace({ dep }, 'dep');
+    return dep;
   } catch (err) {
-    logger.warn({ err, name }, 'nuget getVersions failures: Unknown error');
-    return null;
-  }
-}
-
-async function getNuspec(name, version, retries = 5) {
-  logger.trace(`getNuspec(${name} - ${version})`);
-
-  const url = `https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/${version}/${name.toLowerCase()}.nuspec`;
-
-  try {
-    const result = await got(url, {
-      cache: process.env.RENOVATE_SKIP_CACHE ? undefined : map,
-      json: false,
-      retries,
-      headers,
-    });
-
-    const nuspec = xmlParser.parse(result.body, { ignoreAttributes: false });
-
-    return nuspec.package;
-  } catch (err) {
-    logger.warn({ err, name }, 'nuget getNuspec failures: Unknown error');
+    if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
+      logger.info({ name }, `Dependency lookup failure: not found`);
+      logger.debug({
+        err,
+      });
+      return null;
+    }
+    logger.warn({ err, name }, 'nuget registry failure: Unknown error');
     return null;
   }
 }

lib/manager/nuget/extract.js

@@ -1,3 +1,5 @@
+const { isVersion } = require('../../versioning')('semver');
+
 module.exports = {
   extractDependencies,
 };
@@ -13,14 +15,19 @@ function extractDependencies(content) {
     );
     if (match) {
       const depName = match[1];
-      const currentVersion = match[2];
-
-      deps.push({
+      const currentValue = match[2];
+      const dep = {
         depType: 'nuget',
         depName,
-        currentVersion,
+        currentValue,
         lineNumber,
-      });
+        purl: 'pkg:nuget/' + depName,
+        versionScheme: 'semver',
+      };
+      if (!isVersion(currentValue)) {
+        dep.skipReason = 'not-version';
+      }
+      deps.push(dep);
     }
     lineNumber += 1;
   }

lib/workers/repository/process/lookup/index.js

@@ -45,7 +45,7 @@ async function lookupUpdates(config) {
   // istanbul ignore if
   if (allVersions.length === 0) {
     const message = `No versions returned from registry for this package`;
-    logger.warn({ dependency }, message);
+    logger.warn({ dependency: depName, result: dependency }, message);
     // TODO: return an object
     updates.push([
       {

test/_fixtures/nuget/nunit.json

@@ -0,0 +1,44 @@
+{
+  "versions": [
+    "2.5.7.10213",
+    "2.5.9.10348",
+    "2.5.10.11092",
+    "2.6.0.12051",
+    "2.6.0.12054",
+    "2.6.1",
+    "2.6.2",
+    "2.6.3",
+    "2.6.4",
+    "2.6.5",
+    "2.6.6",
+    "3.0.0-alpha",
+    "3.0.0-alpha-2",
+    "3.0.0-alpha-3",
+    "3.0.0-alpha-4",
+    "3.0.0-alpha-5",
+    "3.0.0-beta-1",
+    "3.0.0-beta-2",
+    "3.0.0-beta-3",
+    "3.0.0-beta-4",
+    "3.0.0-beta-5",
+    "3.0.0-rc",
+    "3.0.0-rc-2",
+    "3.0.0-rc-3",
+    "3.0.0",
+    "3.0.1",
+    "3.2.0",
+    "3.2.1",
+    "3.4.0",
+    "3.4.1",
+    "3.5.0",
+    "3.6.0",
+    "3.6.1",
+    "3.7.0",
+    "3.7.1",
+    "3.8.0",
+    "3.8.1",
+    "3.9.0",
+    "3.10.0",
+    "3.10.1"
+  ]
+}

test/_fixtures/nuget/sample.csproj

@@ -21,7 +21,7 @@
     <PackageReference Include="Serilog" Version="2.4.0" />
     <PackageReference Include="Serilog.Extensions.Logging" Version="1.4.0" />
     <PackageReference Include="Serilog.Sinks.Literate" Version="2.1.0" />
-    <PackageReference Include="Stateless" Version="3.1.0" />
+    <PackageReference Include="Stateless" Version="3.1.0.5" />
   </ItemGroup>
   <ItemGroup>
     <DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="1.0.0" />

test/datasource/__snapshots__/nuget.spec.js.snap

@@ -0,0 +1,115 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`datasource/nuget getDependency processes real data 1`] = `
+Object {
+  "name": "nunit",
+  "releases": Array [
+    Object {
+      "version": "2.6.1",
+    },
+    Object {
+      "version": "2.6.2",
+    },
+    Object {
+      "version": "2.6.3",
+    },
+    Object {
+      "version": "2.6.4",
+    },
+    Object {
+      "version": "2.6.5",
+    },
+    Object {
+      "version": "2.6.6",
+    },
+    Object {
+      "version": "3.0.0-alpha",
+    },
+    Object {
+      "version": "3.0.0-alpha-2",
+    },
+    Object {
+      "version": "3.0.0-alpha-3",
+    },
+    Object {
+      "version": "3.0.0-alpha-4",
+    },
+    Object {
+      "version": "3.0.0-alpha-5",
+    },
+    Object {
+      "version": "3.0.0-beta-1",
+    },
+    Object {
+      "version": "3.0.0-beta-2",
+    },
+    Object {
+      "version": "3.0.0-beta-3",
+    },
+    Object {
+      "version": "3.0.0-beta-4",
+    },
+    Object {
+      "version": "3.0.0-beta-5",
+    },
+    Object {
+      "version": "3.0.0-rc",
+    },
+    Object {
+      "version": "3.0.0-rc-2",
+    },
+    Object {
+      "version": "3.0.0-rc-3",
+    },
+    Object {
+      "version": "3.0.0",
+    },
+    Object {
+      "version": "3.0.1",
+    },
+    Object {
+      "version": "3.2.0",
+    },
+    Object {
+      "version": "3.2.1",
+    },
+    Object {
+      "version": "3.4.0",
+    },
+    Object {
+      "version": "3.4.1",
+    },
+    Object {
+      "version": "3.5.0",
+    },
+    Object {
+      "version": "3.6.0",
+    },
+    Object {
+      "version": "3.6.1",
+    },
+    Object {
+      "version": "3.7.0",
+    },
+    Object {
+      "version": "3.7.1",
+    },
+    Object {
+      "version": "3.8.0",
+    },
+    Object {
+      "version": "3.8.1",
+    },
+    Object {
+      "version": "3.9.0",
+    },
+    Object {
+      "version": "3.10.0",
+    },
+    Object {
+      "version": "3.10.1",
+    },
+  ],
+  "repositoryUrl": "https://github.com/JamesNK/Newtonsoft.Json",
+}
+`;

test/datasource/nuget.spec.js

@@ -1,44 +1,43 @@
 const fs = require('fs');
-const nuget = require('../../lib/datasource/nuget');
+const datasource = require('../../lib/datasource');
 const got = require('got');
 
-const withRepositoryInNuspec = fs.readFileSync(
-  'test/_fixtures/nuget/sample.nuspec',
-  'utf8'
-);
 jest.mock('got');
 
-describe('api/nuget', () => {
-  describe('getVersions', () => {
-    it('returns null if errored', async () => {
-      got.mockReturnValueOnce({});
-      const nuspec = await nuget.getVersions('MyPackage');
-      expect(nuspec).toBe(null);
-    });
-    it('returns versions list', async () => {
-      got.mockReturnValueOnce({
-        body: { versions: ['1.0.0', '2.0.0', '2.1.0', '2.1.1-alpha'] },
-      });
-      const versions = await nuget.getVersions('MyPackage');
-      expect(versions).toHaveLength(4);
-    });
-  });
+const res1 = fs.readFileSync('test/_fixtures/nuget/nunit.json', 'utf8');
+const res2 = fs.readFileSync('test/_fixtures/nuget/sample.nuspec', 'utf8');
 
-  describe('getNuspec', () => {
-    it('returns null if errored', async () => {
+describe('datasource/nuget', () => {
+  describe('getDependency', () => {
+    it('returns null for empty result', async () => {
       got.mockReturnValueOnce({});
-      const nuspec = await nuget.getNuspec('MyPackage', '1.0.0.0');
-      expect(nuspec).toBe(null);
+      expect(await datasource.getDependency('pkg:nuget/something')).toBeNull();
     });
-    it('returns json-ified nuspec with attributes', async () => {
-      got.mockReturnValueOnce({ headers: {}, body: withRepositoryInNuspec });
-      const nuspec = await nuget.getNuspec('MyPackage', '1.0.0.0');
-
-      expect(nuspec.metadata.id).toBe('Newtonsoft.Json');
-      expect(nuspec.metadata.version).toBe('11.0.2');
-      expect(nuspec.metadata.repository['@_url']).toBe(
-        'https://github.com/JamesNK/Newtonsoft.Json.git'
+    it('returns null for 404', async () => {
+      got.mockImplementationOnce(() =>
+        Promise.reject({
+          statusCode: 404,
+        })
       );
+      expect(await datasource.getDependency('pkg:nuget/something')).toBeNull();
+    });
+    it('returns null for unknown error', async () => {
+      got.mockImplementationOnce(() => {
+        throw new Error();
+      });
+      expect(await datasource.getDependency('pkg:nuget/something')).toBeNull();
+    });
+    it('processes real data', async () => {
+      got.mockReturnValueOnce({
+        body: JSON.parse(res1),
+      });
+      got.mockReturnValueOnce({
+        body: res2,
+      });
+      const res = await datasource.getDependency('pkg:nuget/nunit');
+      expect(res).not.toBeNull();
+      expect(res).toMatchSnapshot();
+      expect(res.repositoryUrl).toBeDefined();
     });
   });
 });

test/manager/nuget/__snapshots__/extract.spec.js.snap

@@ -3,76 +3,101 @@
 exports[`lib/manager/nuget/extract extractDependencies() extracts all dependencies 1`] = `
 Array [
   Object {
-    "currentVersion": "4.5.0",
+    "currentValue": "4.5.0",
     "depName": "Autofac",
     "depType": "nuget",
     "lineNumber": 12,
+    "purl": "pkg:nuget/Autofac",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "4.1.0",
+    "currentValue": "4.1.0",
     "depName": "Autofac.Extensions.DependencyInjection",
     "depType": "nuget",
     "lineNumber": 13,
+    "purl": "pkg:nuget/Autofac.Extensions.DependencyInjection",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.1.2",
+    "currentValue": "1.1.2",
     "depName": "Microsoft.AspNetCore.Hosting",
     "depType": "nuget",
     "lineNumber": 14,
+    "purl": "pkg:nuget/Microsoft.AspNetCore.Hosting",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.1.3",
+    "currentValue": "1.1.3",
     "depName": "Microsoft.AspNetCore.Mvc.Core",
     "depType": "nuget",
     "lineNumber": 15,
+    "purl": "pkg:nuget/Microsoft.AspNetCore.Mvc.Core",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.1.2",
+    "currentValue": "1.1.2",
     "depName": "Microsoft.AspNetCore.Server.Kestrel",
     "depType": "nuget",
     "lineNumber": 16,
+    "purl": "pkg:nuget/Microsoft.AspNetCore.Server.Kestrel",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.1.2",
+    "currentValue": "1.1.2",
     "depName": "Microsoft.Extensions.Configuration.Json",
     "depType": "nuget",
     "lineNumber": 17,
+    "purl": "pkg:nuget/Microsoft.Extensions.Configuration.Json",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.1.2",
+    "currentValue": "1.1.2",
     "depName": "Microsoft.Extensions.Logging.Debug",
     "depType": "nuget",
     "lineNumber": 18,
+    "purl": "pkg:nuget/Microsoft.Extensions.Logging.Debug",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "10.0.2",
+    "currentValue": "10.0.2",
     "depName": "Newtonsoft.Json",
     "depType": "nuget",
     "lineNumber": 19,
+    "purl": "pkg:nuget/Newtonsoft.Json",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "2.4.0",
+    "currentValue": "2.4.0",
     "depName": "Serilog",
     "depType": "nuget",
     "lineNumber": 20,
+    "purl": "pkg:nuget/Serilog",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "1.4.0",
+    "currentValue": "1.4.0",
     "depName": "Serilog.Extensions.Logging",
     "depType": "nuget",
     "lineNumber": 21,
+    "purl": "pkg:nuget/Serilog.Extensions.Logging",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "2.1.0",
+    "currentValue": "2.1.0",
     "depName": "Serilog.Sinks.Literate",
     "depType": "nuget",
     "lineNumber": 22,
+    "purl": "pkg:nuget/Serilog.Sinks.Literate",
+    "versionScheme": "semver",
   },
   Object {
-    "currentVersion": "3.1.0",
+    "currentValue": "3.1.0.5",
     "depName": "Stateless",
     "depType": "nuget",
     "lineNumber": 23,
+    "purl": "pkg:nuget/Stateless",
+    "skipReason": "not-version",
+    "versionScheme": "semver",
   },
 ]
 `;