feat: automatically paginate res.body responses (#959)

lib/api/gh-got-retry.js

@@ -26,6 +26,22 @@ async function get(path, opts, retries = 5) {
       }
     }
     const res = await ghGot(path, opts);
+    // Check if result is paginated
+    const linkHeader =
+      res && res.headers && res.headers.link ? res.headers.link : '';
+    const matches = linkHeader.match(
+      /<https:\/\/api.github\.com\/(.*?)>; rel="next".*/
+    );
+    if (matches) {
+      if (Array.isArray(res.body)) {
+        res.body = res.body.concat((await get(matches[1], opts, retries)).body);
+      } else {
+        logger.warn(
+          { path },
+          'Found next in link header but res.body is not an array'
+        );
+      }
+    }
     return res;
   } catch (err) {
     if (err.statusCode >= 500 && err.statusCode < 600 && retries > 0) {

test/api/gh-got-retry.spec.js

@@ -16,6 +16,30 @@ describe('api/gh-got-retry', () => {
       'application/vnd.github.machine-man-preview+json, some-accept'
     );
   });
+  it('paginates', async () => {
+    ghGot.mockReturnValueOnce({
+      headers: {
+        link: '<https://api.github.com/something>; rel="next">',
+      },
+      body: ['a'],
+    });
+    ghGot.mockReturnValueOnce({
+      headers: {},
+      body: ['b'],
+    });
+    const res = await get('some-url');
+    expect(res.body).toHaveLength(2);
+  });
+  it('warns if body cannot be paginated', async () => {
+    ghGot.mockReturnValueOnce({
+      headers: {
+        link: '<https://api.github.com/something>; rel="next">',
+      },
+      body: {},
+    });
+    const res = await get('some-url');
+    expect(res.body).toEqual({});
+  });
   it('should retry 502s', async () => {
     ghGot.mockImplementationOnce(() =>
       Promise.reject({