renovate/renovate
1
0
Fork 0
mirror of https://github.com/renovatebot/renovate.git synced 2025-01-12 15:06:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(github): limit pagination concurrency to 5

Browse source
This commit is contained in:
Rhys Arkins 2019-03-17 07:22:18 +01:00
parent 56a1c1548d
commit 61ce454d35
1 changed files with 9 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
lib/platform/github/gh-got-wrapper.js
View file

@ -2,6 +2,8 @@ const URL = require('url');
const ghGot = require('gh-got');
const delay = require('delay');
const parseLinkHeader = require('parse-link-header');
const pAll = require('p-all');
const hostRules = require('../../util/host-rules');
const { maskToken } = require('../../util/mask');
@ -65,14 +67,13 @@ async function get(path, options, retries = 5) {
new Array(lastPage),
(x, i) => i + 1
).slice(1);
const pages = await Promise.all(
pageNumbers.map(page => {
const url = URL.parse(linkHeader.next.url, true);
delete url.search;
url.query.page = page;
return get(URL.format(url), { ...opts, paginate: false }, retries);
})
);
const queue = pageNumbers.map(page => () => {
const url = URL.parse(linkHeader.next.url, true);
delete url.search;
url.query.page = page;
return get(URL.format(url), { ...opts, paginate: false }, retries);
});
const pages = await pAll(queue, { concurrency: 5 });
res.body = res.body.concat(
...pages.filter(Boolean).map(page => page.body)
);