feat: gitlabci.yml support (#1744)

Adds support for .gitlabci.yml files. Part of the logic is same as Docker Compose files, however the “services” list is new/different so requires additional logic.

Closes #1598

lib/config/definitions.js

@@ -1066,6 +1066,18 @@ const options = [
     mergeable: true,
     cli: false,
   },
+  {
+    name: 'gitlabci',
+    description:
+      'Configuration object for GitLab CI yml renovation. Also inherits settings from `docker` object.',
+    stage: 'repository',
+    type: 'json',
+    default: {
+      fileMatch: ['^.gitlab-ci.yml$'],
+    },
+    mergeable: true,
+    cli: false,
+  },
   {
     name: 'nuget',
     description: 'Configuration object for C#/Nuget',

lib/manager/gitlabci/extract.js

@@ -0,0 +1,56 @@
+const { getDep } = require('../dockerfile/extract');
+
+module.exports = {
+  extractDependencies,
+};
+
+function extractDependencies(content) {
+  const deps = [];
+  try {
+    const lines = content.split('\n');
+    for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
+      const line = lines[lineNumber];
+      const imageMatch = line.match(/^\s*image:\s*'?"?([^\s]+)'?"?\s*$/);
+      if (imageMatch) {
+        logger.trace(`Matched image on line ${lineNumber}`);
+        const currentFrom = imageMatch[1];
+        const dep = getDep(currentFrom);
+        dep.lineNumber = lineNumber;
+        dep.depType = 'image';
+        deps.push(dep);
+      }
+      const services = line.match(/^\s*services:\s*$/);
+      if (services) {
+        logger.trace(`Matched services on line ${lineNumber}`);
+        let foundImage;
+        do {
+          foundImage = false;
+          const serviceImageLine = lines[lineNumber + 1];
+          logger.trace(`serviceImageLine: "${serviceImageLine}"`);
+          const serviceImageMatch = serviceImageLine.match(
+            /^\s*-\s*'?"?([^\s'"]+)'?"?\s*$/
+          );
+          if (serviceImageMatch) {
+            logger.trace('serviceImageMatch');
+            foundImage = true;
+            const currentFrom = serviceImageMatch[1];
+            lineNumber += 1;
+            const dep = getDep(currentFrom);
+            dep.lineNumber = lineNumber;
+            dep.depType = 'service-image';
+            deps.push(dep);
+          }
+        } while (foundImage);
+      }
+    }
+  } catch (err) /* istanbul ignore next */ {
+    logger.error(
+      { err, message: err.message },
+      'Error extracting GitLab CI dependencies'
+    );
+  }
+  if (!deps.length) {
+    return null;
+  }
+  return { deps };
+}

lib/manager/gitlabci/index.js

@@ -0,0 +1,10 @@
+const { extractDependencies } = require('./extract');
+const { updateDependency } = require('./update');
+
+const language = 'docker';
+
+module.exports = {
+  extractDependencies,
+  language,
+  updateDependency,
+};

lib/manager/gitlabci/update.js

@@ -0,0 +1,42 @@
+const { getNewFrom } = require('../dockerfile/update');
+
+module.exports = {
+  updateDependency,
+};
+
+function updateDependency(currentFileContent, upgrade) {
+  try {
+    const newFrom = getNewFrom(upgrade);
+    const lines = currentFileContent.split('\n');
+    const lineToChange = lines[upgrade.lineNumber];
+    if (upgrade.depType === 'image') {
+      const imageLine = new RegExp(/^(\s*image:\s*'?"?)[^\s'"]+('?"?\s*)$/);
+      if (!lineToChange.match(imageLine)) {
+        logger.debug('No image line found');
+        return null;
+      }
+      const newLine = lineToChange.replace(imageLine, `$1${newFrom}$2`);
+      if (newLine === lineToChange) {
+        logger.debug('No changes necessary');
+        return currentFileContent;
+      }
+      lines[upgrade.lineNumber] = newLine;
+      return lines.join('\n');
+    }
+    const serviceLine = new RegExp(/^(\s*-\s*'?"?)[^\s'"]+('?"?\s*)$/);
+    if (!lineToChange.match(serviceLine)) {
+      logger.debug('No image line found');
+      return null;
+    }
+    const newLine = lineToChange.replace(serviceLine, `$1${newFrom}$2`);
+    if (newLine === lineToChange) {
+      logger.debug('No changes necessary');
+      return currentFileContent;
+    }
+    lines[upgrade.lineNumber] = newLine;
+    return lines.join('\n');
+  } catch (err) {
+    logger.info({ err }, 'Error setting new Dockerfile value');
+    return null;
+  }
+}

lib/manager/index.js

@@ -5,6 +5,7 @@ const managerList = [
   'composer',
   'docker-compose',
   'dockerfile',
+  'gitlabci',
   'meteor',
   'npm',
   'nvm',

test/_fixtures/gitlabci/gitlab-ci.yaml

@@ -0,0 +1,97 @@
+.executor-docker: &executor-docker
+    tags:
+        - docker
+
+.executor-docker-in-docker: &executor-docker-privileged
+    tags:
+        - docker-privileged
+
+.executor-docker-in-docker: &executor-docker-in-docker
+    tags:
+        - docker-in-docker
+
+.docker-login: &docker-login
+    before_script:
+        - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+
+.docker-logout: &docker-logout
+    after_script:
+        - docker logout $CI_REGISTRY
+
+.build-image: &build-image
+    export BUILD_IMAGE=$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}
+
+stages:
+    - compliance-tests
+    - build
+    - sast
+    - unit-tests
+    - release
+
+variables:
+    LATEST_IMAGE: $CI_REGISTRY_IMAGE:latest
+
+hadolint:
+    stage: compliance-tests
+    <<: *executor-docker
+    image: hadolint/hadolint:latest
+    script:
+        - hadolint Dockerfile
+
+build:
+    stage: build
+    <<: *executor-docker-privileged
+    <<: *docker-login
+    script:
+        - *build-image
+        - docker build --label "org.label-schema.build-date=$(date +%Y-%m-%dT%T%z)" --label "org.label-schema.version=$CI_COMMIT_REF_NAME" --tag $BUILD_IMAGE .
+        - docker push $BUILD_IMAGE
+    <<: *docker-logout
+
+sast:container:
+    stage: sast
+    <<: *executor-docker-in-docker
+    image: docker:latest
+    services:
+        - docker:dind
+    <<: *docker-login
+    script:
+        - *build-image
+        - docker run -d --name db arminc/clair-db:latest
+        - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
+        - apk add -U wget ca-certificates
+        - docker pull $BUILD_IMAGE
+        - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
+        - mv clair-scanner_linux_amd64 clair-scanner
+        - chmod +x clair-scanner
+        - touch clair-whitelist.yml
+        - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml $BUILD_IMAGE || true
+    <<: *docker-logout
+    artifacts:
+        paths: [gl-sast-container-report.json]
+
+constainer-structure-test:
+    stage: unit-tests
+    <<: *executor-docker-in-docker
+    image: docker:latest
+    services:
+        - docker:dind
+    <<: *docker-login
+    script:
+        - *build-image
+        - docker pull $BUILD_IMAGE
+        - echo "container-structure-test test --verbose --image $BUILD_IMAGE --config /tests/container/*.json" | docker run --rm -i --volume /var/run/docker.sock:/var/run/docker.sock --volume $CI_PROJECT_DIR/tests/container:/tests/container:ro $LATEST_IMAGE /bin/sh; exit $?
+    <<: *docker-logout
+
+release:
+    stage: release
+    <<: *executor-docker-privileged
+    <<: *docker-login
+    script:
+        - *build-image
+        - docker pull $BUILD_IMAGE
+        - docker tag $BUILD_IMAGE $LATEST_IMAGE
+        - docker push $LATEST_IMAGE
+    <<: *docker-logout
+    only:
+      - tags

test/manager/gitlabci/__snapshots__/extract.spec.js.snap

@@ -0,0 +1,81 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`lib/manager/gitlabci/extract extractDependencies() extracts multiple image lines 1`] = `
+Array [
+  Object {
+    "currentDepTag": "hadolint/hadolint:latest",
+    "currentDepTagDigest": "hadolint/hadolint:latest",
+    "currentDigest": undefined,
+    "currentFrom": "hadolint/hadolint:latest",
+    "currentTag": "latest",
+    "currentValue": "latest",
+    "depName": "hadolint/hadolint",
+    "depType": "image",
+    "dockerRegistry": undefined,
+    "lineNumber": 36,
+    "purl": "pkg:docker/hadolint/hadolint",
+    "tagSuffix": undefined,
+    "versionScheme": "docker",
+  },
+  Object {
+    "currentDepTag": "docker:latest",
+    "currentDepTagDigest": "docker:latest",
+    "currentDigest": undefined,
+    "currentFrom": "docker:latest",
+    "currentTag": "latest",
+    "currentValue": "latest",
+    "depName": "docker",
+    "depType": "image",
+    "dockerRegistry": undefined,
+    "lineNumber": 53,
+    "purl": "pkg:docker/docker",
+    "tagSuffix": undefined,
+    "versionScheme": "docker",
+  },
+  Object {
+    "currentDepTag": "docker:dind",
+    "currentDepTagDigest": "docker:dind",
+    "currentDigest": undefined,
+    "currentFrom": "docker:dind",
+    "currentTag": "dind",
+    "currentValue": "dind",
+    "depName": "docker",
+    "depType": "service-image",
+    "dockerRegistry": undefined,
+    "lineNumber": 55,
+    "purl": "pkg:docker/docker",
+    "tagSuffix": undefined,
+    "versionScheme": "docker",
+  },
+  Object {
+    "currentDepTag": "docker:latest",
+    "currentDepTagDigest": "docker:latest",
+    "currentDigest": undefined,
+    "currentFrom": "docker:latest",
+    "currentTag": "latest",
+    "currentValue": "latest",
+    "depName": "docker",
+    "depType": "image",
+    "dockerRegistry": undefined,
+    "lineNumber": 75,
+    "purl": "pkg:docker/docker",
+    "tagSuffix": undefined,
+    "versionScheme": "docker",
+  },
+  Object {
+    "currentDepTag": "docker:dind",
+    "currentDepTagDigest": "docker:dind",
+    "currentDigest": undefined,
+    "currentFrom": "docker:dind",
+    "currentTag": "dind",
+    "currentValue": "dind",
+    "depName": "docker",
+    "depType": "service-image",
+    "dockerRegistry": undefined,
+    "lineNumber": 77,
+    "purl": "pkg:docker/docker",
+    "tagSuffix": undefined,
+    "versionScheme": "docker",
+  },
+]
+`;

test/manager/gitlabci/extract.spec.js

@@ -0,0 +1,26 @@
+const fs = require('fs');
+const {
+  extractDependencies,
+} = require('../../../lib/manager/gitlabci/extract');
+
+const yamlFile = fs.readFileSync(
+  'test/_fixtures/gitlabci/gitlab-ci.yaml',
+  'utf8'
+);
+
+describe('lib/manager/gitlabci/extract', () => {
+  describe('extractDependencies()', () => {
+    let config;
+    beforeEach(() => {
+      config = {};
+    });
+    it('returns null for empty', () => {
+      expect(extractDependencies('nothing here', config)).toBe(null);
+    });
+    it('extracts multiple image lines', () => {
+      const res = extractDependencies(yamlFile, config);
+      expect(res.deps).toMatchSnapshot();
+      expect(res.deps).toHaveLength(5);
+    });
+  });
+});

test/manager/gitlabci/update.spec.js

@@ -0,0 +1,82 @@
+const fs = require('fs');
+const dcUpdate = require('../../../lib/manager/gitlabci/update');
+
+const yamlFile = fs.readFileSync(
+  'test/_fixtures/gitlabci/gitlab-ci.yaml',
+  'utf8'
+);
+
+describe('manager/gitlabci/update', () => {
+  describe('updateDependency', () => {
+    it('replaces existing value', () => {
+      const upgrade = {
+        lineNumber: 36,
+        depType: 'image',
+        depName: 'hadolint/hadolint',
+        newValue: '7.0.0',
+        newDigest: 'sha256:abcdefghijklmnop',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).not.toEqual(yamlFile);
+      expect(res.includes(upgrade.newDigest)).toBe(true);
+    });
+    it('returns same', () => {
+      const upgrade = {
+        depType: 'image',
+        lineNumber: 36,
+        depName: 'hadolint/hadolint',
+        newValue: 'latest',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).toEqual(yamlFile);
+    });
+    it('returns null if mismatch', () => {
+      const upgrade = {
+        lineNumber: 17,
+        depType: 'image',
+        depName: 'postgres',
+        newValue: '9.6.8',
+        newDigest: 'sha256:abcdefghijklmnop',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).toBe(null);
+    });
+    it('replaces service-image update', () => {
+      const upgrade = {
+        lineNumber: 55,
+        depType: 'service-image',
+        depName: 'hadolint/hadolint',
+        newValue: '7.0.0',
+        newDigest: 'sha256:abcdefghijklmnop',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).not.toEqual(yamlFile);
+      expect(res.includes(upgrade.newDigest)).toBe(true);
+    });
+    it('returns null if service-image mismatch', () => {
+      const upgrade = {
+        lineNumber: 17,
+        depType: 'service-image',
+        depName: 'postgres',
+        newValue: '9.6.8',
+        newDigest: 'sha256:abcdefghijklmnop',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).toBe(null);
+    });
+    it('returns service-image same', () => {
+      const upgrade = {
+        depType: 'serviceimage',
+        lineNumber: 55,
+        depName: 'docker',
+        newValue: 'dind',
+      };
+      const res = dcUpdate.updateDependency(yamlFile, upgrade);
+      expect(res).toEqual(yamlFile);
+    });
+    it('returns null if error', () => {
+      const res = dcUpdate.updateDependency(null, null);
+      expect(res).toBe(null);
+    });
+  });
+});

test/workers/repository/extract/__snapshots__/index.spec.js.snap

@@ -20,6 +20,9 @@ Object {
   "dockerfile": Array [
     Object {},
   ],
+  "gitlabci": Array [
+    Object {},
+  ],
   "meteor": Array [
     Object {},
   ],

website/docs/configuration-options.md

@@ -199,6 +199,10 @@ See https://renovatebot.com/docs/configuration-reference/config-presets for deta
 
 ## fileMatch
 
+## gitlabci
+
+Add to this configuration setting if you need to override any of the GitLab CI default settings. Use the `docker` config object instead if you wish for configuration to apply across all Docker-related package managers.
+
 ## group
 
 The default configuration for groups are essentially internal to Renovate and you normally shouldn't need to modify them. However, you may choose to _add_ settings to any group by defining your own `group` configuration object.