feat: new 'lockfile-update' rangeStrategy (#3108)

Adds a new `rangeStrategy` value `lockfile-update'. Enabling this means you will get PRs that contain updates for the versions in your lock file (currently npm and yarn only) even if the range in the `package.json` file hasn't changed because the update is "in range".

Closes #1382

lib/config/definitions.js

@@ -593,7 +593,14 @@ const options = [
     description: 'Policy for how to modify/update existing ranges.',
     type: 'string',
     default: 'replace',
-    allowedValues: ['auto', 'pin', 'bump', 'replace', 'widen'],
+    allowedValues: [
+      'auto',
+      'pin',
+      'bump',
+      'replace',
+      'widen',
+      'update-lockfile',
+    ],
     cli: false,
     env: false,
   },
@@ -825,7 +832,7 @@ const options = [
     description: 'Branch topic',
     type: 'string',
     default:
-      '{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x',
+      '{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}',
     cli: false,
   },
   {

lib/manager/npm/post-update/index.js

@@ -39,11 +39,16 @@ function determineLockFileDirs(config, packageFiles) {
       }
       continue; // eslint-disable-line no-continue
     }
+    if (upgrade.isLockfileUpdate) {
+      yarnLockDirs.push(upgrade.yarnLock);
+      npmLockDirs.push(upgrade.npmLock);
+    }
   }
 
   if (
     config.upgrades.every(
-      upgrade => upgrade.updateType === 'lockFileMaintenance'
+      upgrade =>
+        upgrade.updateType === 'lockFileMaintenance' || upgrade.isLockfileUpdate
     )
   ) {
     return {
@@ -366,12 +371,16 @@ async function getAdditionalFiles(config, packageFiles) {
     const lockFileDir = path.dirname(lockFile);
     const fileName = path.basename(lockFile);
     logger.debug(`Generating ${fileName} for ${lockFileDir}`);
+    const upgrades = config.upgrades.filter(
+      upgrade => upgrade.npmLock === lockFile
+    );
     const res = await npm.generateLockFile(
       upath.join(config.localDir, lockFileDir),
       env,
       fileName,
       config.skipInstalls,
-      config.binarySource
+      config.binarySource,
+      upgrades
     );
     if (res.error) {
       // istanbul ignore if
@@ -415,10 +424,14 @@ async function getAdditionalFiles(config, packageFiles) {
     const lockFileDir = path.dirname(lockFile);
     logger.debug(`Generating yarn.lock for ${lockFileDir}`);
     const lockFileName = upath.join(lockFileDir, 'yarn.lock');
+    const upgrades = config.upgrades.filter(
+      upgrade => upgrade.yarnLock === lockFile
+    );
     const res = await yarn.generateLockFile(
       upath.join(config.localDir, lockFileDir),
       env,
-      config
+      config,
+      upgrades
     );
     if (res.error) {
       // istanbul ignore if

lib/manager/npm/post-update/npm.js

@@ -12,7 +12,8 @@ async function generateLockFile(
   env,
   filename,
   skipInstalls,
-  binarySource
+  binarySource,
+  upgrades = []
 ) {
   logger.debug(`Spawning npm install to create ${cwd}/${filename}`);
   let lockFile = null;
@@ -76,6 +77,19 @@ async function generateLockFile(
     }));
     logger.debug(`npm stdout:\n${stdout}`);
     logger.debug(`npm stderr:\n${stderr}`);
+    const lockUpdates = upgrades.filter(upgrade => upgrade.isLockfileUpdate);
+    if (lockUpdates.length) {
+      const updateCmd =
+        cmd +
+        lockUpdates
+          .map(update => ` ${update.depName}@${update.toVersion}`)
+          .join('');
+      ({ stdout, stderr } = await exec(updateCmd, {
+        cwd,
+        shell: true,
+        env,
+      }));
+    }
     const duration = process.hrtime(startTime);
     const seconds = Math.round(duration[0] + duration[1] / 1e9);
     lockFile = await fs.readFile(upath.join(cwd, filename), 'utf8');

lib/manager/npm/post-update/yarn.js

@@ -7,7 +7,7 @@ module.exports = {
   generateLockFile,
 };
 
-async function generateLockFile(cwd, env, config = {}) {
+async function generateLockFile(cwd, env, config = {}, upgrades = []) {
   const { binarySource, yarnIntegrity } = config;
   logger.debug(`Spawning yarn install to create ${cwd}/yarn.lock`);
   let lockFile = null;
@@ -73,22 +73,37 @@ async function generateLockFile(cwd, env, config = {}) {
       cmd = 'yarn';
     }
     logger.debug(`Using yarn: ${cmd}`);
-    cmd += ' install';
+    let cmdExtras = '';
-    cmd += ' --ignore-scripts';
+    cmdExtras += ' --ignore-scripts';
-    cmd += ' --ignore-engines';
+    cmdExtras += ' --ignore-engines';
-    cmd += ' --ignore-platform';
+    cmdExtras += ' --ignore-platform';
-    cmd += process.env.YARN_MUTEX_FILE
+    cmdExtras += process.env.YARN_MUTEX_FILE
       ? ` --mutex file:${process.env.YARN_MUTEX_FILE}`
       : ' --mutex network:31879';
-
+    const installCmd = cmd + ' install' + cmdExtras;
     // TODO: Switch to native util.promisify once using only node 8
-    ({ stdout, stderr } = await exec(cmd, {
+    ({ stdout, stderr } = await exec(installCmd, {
       cwd,
       shell: true,
       env,
     }));
     logger.debug(`yarn stdout:\n${stdout}`);
     logger.debug(`yarn stderr:\n${stderr}`);
+    const lockUpdates = upgrades
+      .filter(upgrade => upgrade.isLockfileUpdate)
+      .map(upgrade => upgrade.depName);
+    if (lockUpdates.length) {
+      const updateCmd =
+        cmd +
+        ' upgrade' +
+        lockUpdates.map(depName => ` ${depName}`).join('') +
+        cmdExtras;
+      ({ stdout, stderr } = await exec(updateCmd, {
+        cwd,
+        shell: true,
+        env,
+      }));
+    }
     const duration = process.hrtime(startTime);
     const seconds = Math.round(duration[0] + duration[1] / 1e9);
     lockFile = await fs.readFile(upath.join(cwd, 'yarn.lock'), 'utf8');

lib/versioning/npm/range.js

@@ -4,6 +4,7 @@ const {
   minor,
   patch,
   prerelease,
+  satisfies,
   valid: isVersion,
 } = require('semver');
 const { parseRange } = require('semver-utils');
@@ -16,6 +17,12 @@ function getNewValue(currentValue, rangeStrategy, fromVersion, toVersion) {
   if (rangeStrategy === 'pin' || isVersion(currentValue)) {
     return toVersion;
   }
+  if (rangeStrategy === 'lockfile-update') {
+    if (satisfies(toVersion, currentValue)) {
+      return currentValue;
+    }
+    return getNewValue(currentValue, 'replace', fromVersion, toVersion);
+  }
   const parsedRange = parseRange(currentValue);
   const element = parsedRange[parsedRange.length - 1];
   if (rangeStrategy === 'widen') {

lib/workers/repository/onboarding/pr/pr-list.js

@@ -41,7 +41,9 @@ function getPrList(config, branches) {
         } else {
           text += upgrade.depName.replace(prTitleRe, '@​$1');
         }
-        text += ` to \`${upgrade.newDigest || upgrade.newValue}\``;
+        text += upgrade.isLockfileUpdate
+          ? ` to \`${upgrade.toVersion}\``
+          : ` to \`${upgrade.newDigest || upgrade.newValue}\``;
         text += '\n';
       }
       if (!seen.includes(text)) {

lib/workers/repository/process/lookup/index.js

@@ -13,7 +13,7 @@ module.exports = {
 };
 
 async function lookupUpdates(config) {
-  const { depName, currentValue } = config;
+  const { depName, currentValue, lockedVersion } = config;
   logger.trace({ dependency: depName, currentValue }, 'lookupUpdates');
   const {
     equals,
@@ -126,10 +126,15 @@ async function lookupUpdates(config) {
         newMajor: getMajor(fromVersion),
       });
     }
+    let filterStart = fromVersion;
+    if (lockedVersion && rangeStrategy === 'lockfile-update') {
+      // Look for versions greater than the current locked version that still satisfy the package.json range
+      filterStart = lockedVersion;
+    }
     // Filter latest, unstable, etc
     const filteredVersions = filterVersions(
       config,
-      fromVersion,
+      filterStart,
       dependency.latestVersion,
       allVersions,
       releases
@@ -147,12 +152,19 @@ async function lookupUpdates(config) {
         toVersion
       );
       if (!update.newValue || update.newValue === currentValue) {
-        continue; // eslint-disable-line no-continue
+        if (!config.lockedVersion) {
+          continue; // eslint-disable-line no-continue
+        }
+        update.updateType = 'lockfileUpdate';
+        update.fromVersion = lockedVersion;
+        update.isSingleVersion = true;
       }
       update.newMajor = getMajor(toVersion);
       update.newMinor = getMinor(toVersion);
-      update.updateType = getType(config, fromVersion, toVersion);
+      update.updateType =
-      update.isSingleVersion = !!isSingleVersion(update.newValue);
+        update.updateType || getType(config, update.fromVersion, toVersion);
+      update.isSingleVersion =
+        update.isSingleVersion || !!isSingleVersion(update.newValue);
       if (!isVersion(update.newValue)) {
         update.isRange = true;
       }
@@ -186,7 +198,7 @@ async function lookupUpdates(config) {
       release =>
         filteredVersions.length &&
         (filteredVersions.includes(release.version) ||
-          release.version === fromVersion)
+          release.version === filterStart)
     );
   } else if (!currentValue) {
     res.skipReason = 'unsupported-value';
@@ -245,8 +257,8 @@ async function lookupUpdates(config) {
   }
   for (const update of res.updates) {
     const { updateType, fromVersion, toVersion } = update;
-    if (updateType === 'bump') {
+    if (['bump', 'lockfileUpdate'].includes(updateType)) {
-      update.isBump = true;
+      update[updateType === 'bump' ? 'isBump' : 'isLockfileUpdate'] = true;
       if (getMajor(toVersion) > getMajor(fromVersion)) {
         update.updateType = 'major';
       } else if (
@@ -265,6 +277,7 @@ async function lookupUpdates(config) {
     .filter(
       update =>
         update.newValue !== config.currentValue ||
+        update.isLockfileUpdate ||
         (update.newDigest && !update.newDigest.startsWith(config.currentDigest))
     );
   if (res.updates.some(update => update.updateType === 'pin')) {
@@ -301,6 +314,9 @@ function getType(config, fromVersion, toVersion) {
 function getBucket(config, update) {
   const { separateMajorMinor, separateMultipleMajor } = config;
   const { updateType, newMajor } = update;
+  if (updateType === 'lockfileUpdate') {
+    return updateType;
+  }
   if (
     !separateMajorMinor ||
     config.major.automerge === true ||

test/workers/branch/lock-files/npm.spec.js

@@ -29,6 +29,33 @@ describe('generateLockFile', () => {
     expect(res.error).not.toBeDefined();
     expect(res.lockFile).toEqual('package-lock-contents');
   });
+  it('performs lock file updates', async () => {
+    getInstalledPath.mockReturnValueOnce('node_modules/npm');
+    exec.mockReturnValueOnce({
+      stdout: '',
+      stderror: '',
+    });
+    exec.mockReturnValueOnce({
+      stdout: '',
+      stderror: '',
+    });
+    fs.readFile = jest.fn(() => 'package-lock-contents');
+    const skipInstalls = true;
+    const updates = [
+      { depName: 'some-dep', toVersion: '1.0.1', isLockfileUpdate: true },
+    ];
+    const res = await npmHelper.generateLockFile(
+      'some-dir',
+      {},
+      'package-lock.json',
+      skipInstalls,
+      null,
+      updates
+    );
+    expect(fs.readFile.mock.calls.length).toEqual(1);
+    expect(res.error).not.toBeDefined();
+    expect(res.lockFile).toEqual('package-lock-contents');
+  });
   it('performs full install', async () => {
     getInstalledPath.mockReturnValueOnce('node_modules/npm');
     exec.mockReturnValueOnce({

test/workers/branch/lock-files/yarn.spec.js

@@ -22,6 +22,22 @@ describe('generateLockFile', () => {
     expect(fs.readFile.mock.calls.length).toEqual(1);
     expect(res.lockFile).toEqual('package-lock-contents');
   });
+  it('performs lock file updates', async () => {
+    getInstalledPath.mockReturnValueOnce('node_modules/yarn');
+    exec.mockReturnValueOnce({
+      stdout: '',
+      stderror: '',
+    });
+    exec.mockReturnValueOnce({
+      stdout: '',
+      stderror: '',
+    });
+    fs.readFile = jest.fn(() => 'package-lock-contents');
+    const res = await yarnHelper.generateLockFile('some-dir', {}, {}, [
+      { depName: 'some-dep', isLockfileUpdate: true },
+    ]);
+    expect(res.lockFile).toEqual('package-lock-contents');
+  });
   it('catches errors', async () => {
     getInstalledPath.mockReturnValueOnce('node_modules/yarn');
     exec.mockReturnValueOnce({

test/workers/repository/process/lookup/__snapshots__/index.spec.js.snap

@@ -1143,6 +1143,48 @@ Array [
 ]
 `;
 
+exports[`workers/repository/process/lookup .lookupUpdates() supports lock file updates mixed with regular updates 1`] = `
+Array [
+  Object {
+    "canBeUnpublished": false,
+    "fromVersion": "0.4.0",
+    "isLockfileUpdate": true,
+    "isRange": true,
+    "isSingleVersion": true,
+    "newMajor": 0,
+    "newMinor": 4,
+    "newValue": "^0.4.0",
+    "releaseTimestamp": "2011-06-10T17:20:04.719Z",
+    "toVersion": "0.4.4",
+    "updateType": "minor",
+  },
+  Object {
+    "canBeUnpublished": false,
+    "fromVersion": "0.4.4",
+    "isRange": true,
+    "isSingleVersion": false,
+    "newMajor": 0,
+    "newMinor": 9,
+    "newValue": "^0.9.0",
+    "releaseTimestamp": "2013-09-04T17:07:22.948Z",
+    "toVersion": "0.9.7",
+    "updateType": "minor",
+  },
+  Object {
+    "canBeUnpublished": false,
+    "fromVersion": "0.4.4",
+    "isRange": true,
+    "isSingleVersion": false,
+    "newMajor": 1,
+    "newMinor": 4,
+    "newValue": "^1.0.0",
+    "releaseTimestamp": "2015-05-17T04:25:07.299Z",
+    "toVersion": "1.4.1",
+    "updateType": "major",
+  },
+]
+`;
+
 exports[`workers/repository/process/lookup .lookupUpdates() supports majorgte updates 1`] = `
 Array [
   Object {

test/workers/repository/process/lookup/index.spec.js

@@ -54,6 +54,17 @@ describe('workers/repository/process/lookup', () => {
         .reply(200, qJson);
       expect((await lookup.lookupUpdates(config)).updates).toMatchSnapshot();
     });
+    it('supports lock file updates mixed with regular updates', async () => {
+      config.currentValue = '^0.4.0';
+      config.rangeStrategy = 'lockfile-update';
+      config.depName = 'q';
+      config.purl = 'pkg:npm/q';
+      config.lockedVersion = '0.4.0';
+      nock('https://registry.npmjs.org')
+        .get('/q')
+        .reply(200, qJson);
+      expect((await lookup.lookupUpdates(config)).updates).toMatchSnapshot();
+    });
     it('returns multiple updates if grouping but separateMajorMinor=true', async () => {
       config.groupName = 'somegroup';
       config.currentValue = '0.4.0';

test/workers/repository/updates/__snapshots__/flatten.spec.js.snap

@@ -10,7 +10,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",
@@ -107,7 +107,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",
@@ -301,7 +301,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",
@@ -495,7 +495,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",
@@ -592,7 +592,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",
@@ -689,7 +689,7 @@ Array [
     "binarySource": "bundled",
     "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}",
     "branchPrefix": "renovate/",
-    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x",
+    "branchTopic": "{{{depNameSanitized}}}-{{{newMajor}}}{{#if isPatch}}.{{{newMinor}}}{{/if}}.x{{#if isLockfileUpdate}}-lockfile{{/if}}",
     "bumpVersion": null,
     "commitBody": null,
     "commitMessage": "{{{commitMessagePrefix}}} {{{commitMessageAction}}} {{{commitMessageTopic}}} {{{commitMessageExtra}}} {{{commitMessageSuffix}}}",

website/docs/configuration-options.md

@@ -758,6 +758,7 @@ Behaviour:
 - `bump` = e.g. bump the range even if the new version satisifies the existing range, e.g. `^1.0.0` -> `^1.1.0`
 - `replace` = Replace the range with a newer one if the new version falls outside it, e.g. `^1.0.0` -> `^2.0.0`
 - `widen` = Widen the range with newer one, e.g. `^1.0.0` -> `^1.0.0 || ^2.0.0`
+- `update-lockfile` = Update the lock file when in-range updates are available, otherwise 'replace' for updates out of range
 
 Renovate's "auto" strategy works like this for npm: