From 61ce454d355816e7fdc6edc7aaa24085d0c73c15 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Sun, 17 Mar 2019 07:22:18 +0100 Subject: [PATCH] fix(github): limit pagination concurrency to 5 --- lib/platform/github/gh-got-wrapper.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/platform/github/gh-got-wrapper.js b/lib/platform/github/gh-got-wrapper.js index f881b54ef0..c868679a18 100644 --- a/lib/platform/github/gh-got-wrapper.js +++ b/lib/platform/github/gh-got-wrapper.js @@ -2,6 +2,8 @@ const URL = require('url'); const ghGot = require('gh-got'); const delay = require('delay'); const parseLinkHeader = require('parse-link-header'); +const pAll = require('p-all'); + const hostRules = require('../../util/host-rules'); const { maskToken } = require('../../util/mask'); @@ -65,14 +67,13 @@ async function get(path, options, retries = 5) { new Array(lastPage), (x, i) => i + 1 ).slice(1); - const pages = await Promise.all( - pageNumbers.map(page => { - const url = URL.parse(linkHeader.next.url, true); - delete url.search; - url.query.page = page; - return get(URL.format(url), { ...opts, paginate: false }, retries); - }) - ); + const queue = pageNumbers.map(page => () => { + const url = URL.parse(linkHeader.next.url, true); + delete url.search; + url.query.page = page; + return get(URL.format(url), { ...opts, paginate: false }, retries); + }); + const pages = await pAll(queue, { concurrency: 5 }); res.body = res.body.concat( ...pages.filter(Boolean).map(page => page.body) );