feat(bitbucket): git fs (#3168)

Adds gitFs support to Bitbucket Cloud. It is now mandatory to configure Bitbucket with username/password instead of token.

Closes #2550, Closes #3024

lib/platform/bitbucket/index.js

@@ -2,6 +2,7 @@ const parseDiff = require('parse-diff');
 const api = require('./bb-got-wrapper');
 const utils = require('./utils');
 const hostRules = require('../../util/host-rules');
+const GitStorage = require('../git/storage');
 const { appSlug } = require('../../config/app-strings');
 
 let config = {};
@@ -11,7 +12,7 @@ module.exports = {
   getRepos,
   cleanRepo,
   initRepo,
-  getRepoStatus: () => ({}),
+  getRepoStatus,
   getRepoForceRebase,
   setBaseBranch,
   // Search
@@ -76,12 +77,14 @@ async function getRepos(token, endpoint) {
 }
 
 // Initialize bitbucket by getting base branch and SHA
-async function initRepo({ repository, endpoint }) {
+async function initRepo({ repository, endpoint, localDir }) {
   logger.debug(`initRepo("${repository}")`);
   const opts = hostRules.find({ platform: 'bitbucket' }, { endpoint });
   // istanbul ignore next
-  if (!opts.token) {
+  if (!(opts.username && opts.password)) {
-    throw new Error(`No token found for Bitbucket repository ${repository}`);
+    throw new Error(
+      `No username/password found for Bitbucket repository ${repository}`
+    );
   }
   hostRules.update({ ...opts, platform: 'bitbucket', default: true });
   api.reset();
@@ -89,6 +92,22 @@ async function initRepo({ repository, endpoint }) {
   // TODO: get in touch with @rarkins about lifting up the caching into the app layer
   config.repository = repository;
   const platformConfig = {};
+
+  // Always gitFs
+  const url = GitStorage.getUrl({
+    gitFs: 'https',
+    auth: `${opts.username}:${opts.password}`,
+    hostname: 'bitbucket.org',
+    repository,
+  });
+
+  config.storage = new GitStorage();
+  await config.storage.initRepo({
+    ...config,
+    localDir,
+    url,
+  });
+
   try {
     const info = utils.repoInfoTransformer(
       (await api.get(`/2.0/repositories/${repository}`)).body
@@ -126,6 +145,7 @@ async function setBaseBranch(branchName) {
     config.baseBranch = branchName;
     delete config.baseCommitSHA;
     delete config.fileList;
+    config.storage.setBaseBranch(branchName);
     await getFileList(branchName);
   }
 }
@@ -133,76 +153,62 @@ async function setBaseBranch(branchName) {
 // Search
 
 // Get full file list
-async function getFileList(branchName) {
+function getFileList(branchName) {
-  const branch = branchName || config.baseBranch;
+  return config.storage.getFileList(branchName);
-  config.fileList = config.fileList || {};
-  if (config.fileList[branch]) {
-    return config.fileList[branch];
-  }
-  try {
-    const branchSha = await getBranchCommit(branch);
-    const filesRaw = await utils.files(
-      `/2.0/repositories/${config.repository}/src/${branchSha}/`
-    );
-    config.fileList[branch] = filesRaw.map(file => file.path);
-  } catch (err) /* istanbul ignore next */ {
-    logger.info(
-      { repository: config.repository },
-      'Error retrieving git tree - no files detected'
-    );
-    config.fileList[branch] = [];
-  }
-  return config.fileList[branch];
 }
 
 // Branch
 
 // Returns true if branch exists, otherwise false
-async function branchExists(branchName) {
+function branchExists(branchName) {
-  logger.debug(`branchExists(${branchName})`);
+  return config.storage.branchExists(branchName);
-  try {
-    const { name } = (await api.get(
-      `/2.0/repositories/${config.repository}/refs/branches/${branchName}`
-    )).body;
-    return name === branchName;
-  } catch (err) {
-    if (err.statusCode === 404) {
-      return false;
-    }
-    // istanbul ignore next
-    throw err;
-  }
 }
 
-// TODO rewrite mutating reduce to filter in other adapters
+function getAllRenovateBranches(branchPrefix) {
-async function getAllRenovateBranches(branchPrefix) {
+  return config.storage.getAllRenovateBranches(branchPrefix);
-  logger.trace('getAllRenovateBranches');
+}
-  const allBranches = await utils.accumulateValues(
+
-    `/2.0/repositories/${config.repository}/refs/branches`
+function isBranchStale(branchName) {
+  return config.storage.isBranchStale(branchName);
+}
+
+function getFile(filePath, branchName) {
+  return config.storage.getFile(filePath, branchName);
+}
+
+function deleteBranch(branchName) {
+  return config.storage.deleteBranch(branchName);
+}
+
+function getBranchLastCommitTime(branchName) {
+  return config.storage.getBranchLastCommitTime(branchName);
+}
+
+// istanbul ignore next
+function getRepoStatus() {
+  return config.storage.getRepoStatus();
+}
+
+function mergeBranch(branchName) {
+  return config.storage.mergeBranch(branchName);
+}
+
+function commitFilesToBranch(
+  branchName,
+  files,
+  message,
+  parentBranch = config.baseBranch
+) {
+  return config.storage.commitFilesToBranch(
+    branchName,
+    files,
+    message,
+    parentBranch
   );
-  return allBranches
-    .map(branch => branch.name)
-    .filter(name => name.startsWith(branchPrefix));
 }
 
-// Check if branch's parent SHA = master SHA
+function getCommitMessages() {
-async function isBranchStale(branchName) {
+  return config.storage.getCommitMessages();
-  logger.debug(`isBranchStale(${branchName})`);
-  const [branch, baseBranch] = (await Promise.all([
-    api.get(
-      `/2.0/repositories/${config.repository}/refs/branches/${branchName}`
-    ),
-    api.get(
-      `/2.0/repositories/${config.repository}/refs/branches/${
-        config.baseBranch
-      }`
-    ),
-  ])).map(res => res.body);
-
-  const branchParentCommit = branch.target.parents[0].hash;
-  const baseBranchLatestCommit = baseBranch.target.hash;
-
-  return branchParentCommit !== baseBranchLatestCommit;
 }
 
 // Returns the Pull Request for a branch. Null if not exists.
@@ -281,29 +287,6 @@ async function setBranchStatus(
   );
 }
 
-async function deleteBranch(branchName, closePr = false) {
-  // istanbul ignore if
-  if (closePr) {
-    logger.debug('Closing PR');
-    const pr = await getBranchPr(branchName);
-    if (pr) {
-      await api.post(
-        `/2.0/repositories/${config.repository}/pullrequests/${
-          pr.number
-        }/decline`
-      );
-    }
-  }
-  return api.delete(
-    `/2.0/repositories/${config.repository}/refs/branches/${branchName}`
-  );
-}
-
-function mergeBranch() {
-  // The api does not support merging branches, so any automerge must be done via PR
-  return Promise.reject(new Error('Branch automerge not supported'));
-}
-
 async function findOpenIssues(title) {
   try {
     const currentUser = (await api.get('/2.0/user')).body.username;
@@ -396,16 +379,6 @@ async function ensureIssueClosing(title) {
   }
 }
 
-async function getBranchLastCommitTime(branchName) {
-  const branches = await utils.accumulateValues(
-    `/2.0/repositories/${config.repository}/refs/branches`
-  );
-  const branch = branches.find(br => br.name === branchName) || {
-    target: {},
-  };
-  return branch.target.date ? new Date(branch.target.date) : new Date();
-}
-
 function addAssignees() {
   // Bitbucket supports "participants" and "reviewers" so does not seem to have the concept of "assignee"
   logger.warn('Cannot add assignees');
@@ -603,69 +576,6 @@ async function getBranchCommit(branchName) {
   }
 }
 
-// Add a new commit, create branch if not existing
-async function commitFilesToBranch(
-  branchName,
-  files,
-  message,
-  parentBranch = config.baseBranch
-) {
-  logger.debug(
-    `commitFilesToBranch('${branchName}', files, message, '${parentBranch})'`
-  );
-  if (branchName !== parentBranch && (await branchExists(branchName))) {
-    logger.debug('Deleting existing branch');
-    await deleteBranch(branchName);
-    delete config.fileList[branchName];
-  }
-  const parents = await getBranchCommit(parentBranch);
-
-  const form = utils.commitForm({
-    message,
-    gitAuthor: config.gitAuthor,
-    parents,
-    branchName,
-    files,
-  });
-
-  await api.post(`/2.0/repositories/${config.repository}/src`, {
-    json: false,
-    body: form,
-  });
-}
-
-// Generic File operations
-async function getFile(filePath, branchName) {
-  logger.debug(`getFile(filePath=${filePath}, branchName=${branchName})`);
-  if (!branchName || branchName === config.baseBranch) {
-    const fileList = await getFileList(branchName);
-    if (!fileList.includes(filePath)) {
-      return null;
-    }
-  }
-  try {
-    const branchSha = await getBranchCommit(branchName || config.baseBranch);
-    const file = (await api.get(
-      `/2.0/repositories/${config.repository}/src/${branchSha}/${filePath}`,
-      { json: false }
-    )).body;
-    return file;
-  } catch (err) {
-    if (err.statusCode === 404) {
-      return null;
-    }
-    throw err;
-  }
-}
-
-async function getCommitMessages() {
-  logger.debug('getCommitMessages');
-  const values = await utils.accumulateValues(
-    `/2.0/repositories/${config.repository}/commits`
-  );
-  return values.map(commit => commit.message);
-}
-
 // Pull Request
 
 async function getPrList() {
@@ -682,6 +592,10 @@ async function getPrList() {
 }
 
 function cleanRepo() {
+  // istanbul ignore if
+  if (config.storage && config.storage.cleanRepo) {
+    config.storage.cleanRepo();
+  }
   api.reset();
   config = {};
 }

lib/platform/bitbucket/utils.js

@@ -1,5 +1,4 @@
 const url = require('url');
-const FormData = require('form-data');
 const api = require('./bb-got-wrapper');
 
 const repoInfoTransformer = repoInfoBody => ({
@@ -34,31 +33,6 @@ const addMaxLength = (inputUrl, pagelen = 100) => {
   return maxedUrl;
 };
 
-const filesEndpoint = async (reqUrl, method = 'get', options) => {
-  const values = await accumulateValues(reqUrl, method, options);
-  const commitFolders = values.filter(
-    value => value.type === 'commit_directory'
-  );
-  let commitFiles = values.filter(value => value.type === 'commit_file');
-
-  if (
-    process.env.RENOVATE_DISABLE_FILE_RECURSION !== 'true' &&
-    commitFolders.length !== 0
-  ) {
-    const moreFiles = [].concat(
-      ...(await Promise.all(
-        commitFolders
-          .map(folder => folder.links.self.href)
-          .filter(Boolean)
-          .map(selfUrl => filesEndpoint(selfUrl, method, options))
-      ))
-    );
-    commitFiles = [...moreFiles, ...commitFiles];
-  }
-
-  return commitFiles;
-};
-
 const accumulateValues = async (reqUrl, method = 'get', options, pagelen) => {
   let accumulator = [];
   let nextUrl = addMaxLength(reqUrl, pagelen);
@@ -87,21 +61,6 @@ const isConflicted = files => {
   return false;
 };
 
-const commitForm = ({ message, gitAuthor, parents, branchName, files }) => {
-  const form = new FormData();
-  form.append('message', message);
-  // istanbul ignore if
-  if (gitAuthor) {
-    form.append('author', gitAuthor);
-  }
-  form.append('parents', parents);
-  form.append('branch', branchName);
-  files.forEach(({ name, contents }) => {
-    form.append(`/${name}`, contents);
-  });
-  return form;
-};
-
 const prInfo = pr => ({
   number: pr.id,
   body: pr.summary ? pr.summary.raw : undefined,
@@ -117,7 +76,5 @@ module.exports = {
   buildStates,
   prInfo,
   accumulateValues,
-  files: filesEndpoint,
   isConflicted,
-  commitForm,
 };

test/platform/bitbucket/__snapshots__/index.spec.js.snap

@@ -1,33 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`platform/bitbucket commitFilesToBranch() posts files 1`] = `
-Array [
-  "",
-  "
-Content-Disposition: form-data; name=\\"message\\"
-
-message
-",
-  "
-Content-Disposition: form-data; name=\\"parents\\"
-
-master_hash
-",
-  "
-Content-Disposition: form-data; name=\\"branch\\"
-
-branch
-",
-  "
-Content-Disposition: form-data; name=\\"/package.json\\"
-
-hello world
-",
-  "--
-",
-]
-`;
-
 exports[`platform/bitbucket createPr() posts PR 1`] = `
 Array [
   Array [
@@ -62,13 +34,6 @@ Array [
     "/2.0/repositories/some/empty/pullrequests?state=OPEN&state=MERGED&state=DECLINED&state=SUPERSEDED&pagelen=50",
     undefined,
   ],
-  Array [
-    "/2.0/repositories/some/empty/refs/branches/master",
-  ],
-  Array [
-    "/2.0/repositories/some/empty/src/null/?pagelen=100",
-    undefined,
-  ],
   Array [
     "/2.0/user",
   ],
@@ -165,31 +130,6 @@ Object {
 }
 `;
 
-exports[`platform/bitbucket getCommitMessages() works 1`] = `
-Array [
-  "Commit messsage 0",
-  "Commit messsage 1",
-  "Commit messsage 2",
-  "Commit messsage 3",
-  "Commit messsage 4",
-  "Commit messsage 5",
-  "Commit messsage 6",
-  "Commit messsage 7",
-  "Commit messsage 8",
-  "Commit messsage 9",
-  "Commit messsage 10",
-  "Commit messsage 11",
-  "Commit messsage 12",
-  "Commit messsage 13",
-  "Commit messsage 14",
-  "Commit messsage 15",
-  "Commit messsage 16",
-  "Commit messsage 17",
-  "Commit messsage 18",
-  "Commit messsage 19",
-]
-`;
-
 exports[`platform/bitbucket getPr() exists 1`] = `
 Object {
   "body": "summary",
@@ -236,21 +176,7 @@ Array [
 ]
 `;
 
-exports[`platform/bitbucket setBaseBranch() updates file list 1`] = `
+exports[`platform/bitbucket setBaseBranch() updates file list 1`] = `Array []`;
-Array [
-  Array [
-    "/2.0/repositories/some/repo/refs/branches/branch",
-  ],
-  Array [
-    "/2.0/repositories/some/repo/src/branch_hash/?pagelen=100",
-    undefined,
-  ],
-  Array [
-    "/2.0/repositories/some/repo/src/branch_hash/foo_folder/?pagelen=100",
-    undefined,
-  ],
-]
-`;
 
 exports[`platform/bitbucket setBranchStatus() posts status 1`] = `
 Array [

test/platform/bitbucket/index.spec.js

@@ -1,37 +1,44 @@
 const URL = require('url');
 const responses = require('../../_fixtures/bitbucket/responses');
 
-function streamToString(stream) {
-  // eslint-disable-next-line promise/avoid-new
-  return new Promise(resolve => {
-    const chunks = [];
-    stream.on('data', chunk => {
-      chunks.push(chunk.toString());
-    });
-    stream.on('end', () => {
-      resolve(chunks.join(''));
-    });
-    stream.resume();
-  });
-}
-
 describe('platform/bitbucket', () => {
   let bitbucket;
   let api;
   let hostRules;
+  let GitStorage;
   beforeEach(() => {
     // reset module
     jest.resetModules();
     jest.mock('../../../lib/platform/bitbucket/bb-got-wrapper');
+    jest.mock('../../../lib/platform/git/storage');
     hostRules = require('../../../lib/util/host-rules');
     api = require('../../../lib/platform/bitbucket/bb-got-wrapper');
     bitbucket = require('../../../lib/platform/bitbucket');
+    GitStorage = require('../../../lib/platform/git/storage');
+    GitStorage.mockImplementation(() => ({
+      initRepo: jest.fn(),
+      cleanRepo: jest.fn(),
+      getFileList: jest.fn(),
+      branchExists: jest.fn(() => true),
+      isBranchStale: jest.fn(() => false),
+      setBaseBranch: jest.fn(),
+      getBranchLastCommitTime: jest.fn(),
+      getAllRenovateBranches: jest.fn(),
+      getCommitMessages: jest.fn(),
+      getFile: jest.fn(),
+      commitFilesToBranch: jest.fn(),
+      mergeBranch: jest.fn(),
+      deleteBranch: jest.fn(),
+      getRepoStatus: jest.fn(),
+    }));
 
     // clean up hostRules
     hostRules.clear();
     hostRules.update({
       platform: 'bitbucket',
       token: 'token',
+      username: 'username',
+      password: 'password',
     });
   });
 
@@ -105,55 +112,31 @@ describe('platform/bitbucket', () => {
   });
 
   describe('getFileList()', () => {
-    const getFileList = wrap('getFileList');
+    it('sends to gitFs', async () => {
-    it('works', async () => {
       await initRepo();
-      expect(await getFileList('branch')).toEqual([
+      await mocked(async () => {
-        'foo_folder/foo_file',
+        await bitbucket.getFileList();
-        'bar_file',
-      ]);
       });
-    it('returns cached result', async () => {
-      await initRepo();
-      expect(await getFileList('branch')).toEqual([
-        'foo_folder/foo_file',
-        'bar_file',
-      ]);
     });
   });
 
   describe('branchExists()', () => {
-    it('returns true if branch exist in repo', async () => {
+    describe('getFileList()', () => {
-      api.get.mockImplementationOnce(() => ({ body: { name: 'branch1' } }));
+      it('sends to gitFs', async () => {
-      const actual = await bitbucket.branchExists('branch1');
+        await initRepo();
-      const expected = true;
+        await mocked(async () => {
-      expect(actual).toBe(expected);
+          await bitbucket.branchExists();
         });
-
-    it('returns false if branch does not exist in repo', async () => {
-      api.get.mockImplementationOnce(() => ({ body: { name: 'branch2' } }));
-      const actual = await bitbucket.branchExists('branch1');
-      const expected = false;
-      expect(actual).toBe(expected);
       });
-
-    it('returns false if 404', async () => {
-      api.get.mockImplementationOnce(() =>
-        Promise.reject({
-          statusCode: 404,
-        })
-      );
-      const actual = await bitbucket.branchExists('branch1');
-      const expected = false;
-      expect(actual).toBe(expected);
     });
   });
 
   describe('isBranchStale()', () => {
-    const isBranchStale = wrap('isBranchStale');
+    it('sends to gitFs', async () => {
-    it('returns false for same hash', async () => {
       await initRepo();
-      expect(await isBranchStale('branch')).toBe(false);
+      await mocked(async () => {
+        await bitbucket.isBranchStale();
+      });
     });
   });
 
@@ -207,26 +190,38 @@ describe('platform/bitbucket', () => {
   });
 
   describe('getRepoStatus()', () => {
-    it('exists', async () => {
+    it('sends to gitFs', async () => {
-      expect(await bitbucket.getRepoStatus()).toEqual({});
+      await initRepo();
+      await mocked(async () => {
+        await bitbucket.getRepoStatus();
+      });
     });
   });
 
   describe('deleteBranch()', () => {
-    it('exists', () => {
+    it('sends to gitFs', async () => {
-      expect(bitbucket.deleteBranch).toBeDefined();
+      await initRepo();
+      await mocked(async () => {
+        await bitbucket.deleteBranch();
+      });
     });
   });
 
   describe('mergeBranch()', () => {
-    it('throws', async () => {
+    it('sends to gitFs', async () => {
-      await expect(bitbucket.mergeBranch()).rejects.toBeDefined();
+      await initRepo();
+      await mocked(async () => {
+        await bitbucket.mergeBranch();
+      });
     });
   });
 
   describe('getBranchLastCommitTime()', () => {
-    it('exists', () => {
+    it('sends to gitFs', async () => {
-      expect(bitbucket.getBranchLastCommitTime).toBeDefined();
+      await initRepo();
+      await mocked(async () => {
+        await bitbucket.getBranchLastCommitTime();
+      });
     });
   });
 
@@ -378,67 +373,47 @@ describe('platform/bitbucket', () => {
   });
 
   describe('commitFilesToBranch()', () => {
-    it('posts files', async () => {
+    it('sends to gitFs', async () => {
       await initRepo();
-      const files = [
-        {
-          name: 'package.json',
-          contents: 'hello world',
-        },
-      ];
       await mocked(async () => {
-        await bitbucket.commitFilesToBranch('branch', files, 'message');
+        await bitbucket.commitFilesToBranch();
-        expect(api.post.mock.calls).toHaveLength(1);
-        const { body } = api.post.mock.calls[0][1];
-        const content = (await streamToString(body)).split(
-          '--' + body.getBoundary()
-        );
-        expect(content).toMatchSnapshot();
       });
     });
   });
 
   describe('getFile()', () => {
-    beforeEach(initRepo);
+    it('sends to gitFs', async () => {
-    const getFile = wrap('getFile');
+      await initRepo();
-    it('works', async () => {
+      await mocked(async () => {
-      expect(await getFile('bar_file', 'branch')).toBe('bar_file content');
+        await bitbucket.getFile();
       });
-    it('returns null for file not found', async () => {
-      expect(await getFile('not_found', 'master')).toBe(null);
-    });
-    it('returns null for 404', async () => {
-      expect(await getFile('not_found', 'branch')).toBe(null);
-    });
-    it('throws for non 404', async () => {
-      await expect(getFile('error', 'branch')).rejects.toBeDefined();
     });
   });
 
   describe('getCommitMessages()', () => {
-    const getCommitMessages = wrap('getCommitMessages');
+    it('sends to gitFs', async () => {
-    it('works', async () => {
       await initRepo();
-      expect(await getCommitMessages()).toMatchSnapshot();
+      await mocked(async () => {
+        await bitbucket.getCommitMessages();
+      });
     });
   });
 
   describe('getAllRenovateBranches()', () => {
-    const getAllRenovateBranches = wrap('getAllRenovateBranches');
+    it('sends to gitFs', async () => {
-    it('retuns filtered branches', async () => {
       await initRepo();
-      expect(await getAllRenovateBranches('renovate/')).toEqual([
+      await mocked(async () => {
-        'renovate/branch',
+        await bitbucket.getAllRenovateBranches();
-        'renovate/upgrade',
+      });
-      ]);
     });
   });
 
   describe('getBranchLastCommitTime()', () => {
-    const getBranchLastCommitTime = wrap('getBranchLastCommitTime');
+    it('sends to gitFs', async () => {
-    it('returns last commit time', async () => {
       await initRepo();
-      expect(await getBranchLastCommitTime('renovate/foo')).toBeDefined();
+      await mocked(async () => {
+        await bitbucket.getBranchLastCommitTime();
+      });
     });
   });