const URL = require('url'); const ghGot = require('gh-got'); const delay = require('delay'); const parseLinkHeader = require('parse-link-header'); const pAll = require('p-all'); const hostRules = require('../../util/host-rules'); const { maskToken } = require('../../util/mask'); let cache = {}; let stats = {}; let endpoint = 'https://api.github.com/'; async function get(path, options, retries = 5) { const host = URL.parse(path).host || URL.parse(endpoint).host; const opts = { // TODO: Move to configurable host rules, or use utils/got timeout: 60 * 1000, ...hostRules.find({ hostType: 'github', host }), ...options, }; delete opts.endpoint; const method = opts.method || 'get'; const useCache = opts.useCache || true; let url = URL.resolve(endpoint, path); if (method.toLowerCase() === 'post' && path === 'graphql') { // GitHub Enterprise uses unversioned graphql path url = url.replace('/v3/', '/'); } if (method === 'get' && useCache && cache[path]) { logger.trace({ path }, 'Returning cached result'); return cache[path]; } logger.trace(`${method.toUpperCase()} ${path}`); stats.requests = (stats.requests || []).concat([ method.toUpperCase() + ' ' + path, ]); try { if (global.appMode) { const appAccept = 'application/vnd.github.machine-man-preview+json'; opts.headers = Object.assign( {}, { accept: appAccept, 'user-agent': process.env.RENOVATE_USER_AGENT || 'https://github.com/renovatebot/renovate', }, opts.headers ); if (opts.headers.accept !== appAccept) { opts.headers.accept = `${appAccept}, ${opts.headers.accept}`; } } const res = await ghGot(url, opts); if (res && res.headers) { stats.rateLimit = res.headers['x-ratelimit-limit']; stats.rateLimitRemaining = res.headers['x-ratelimit-remaining']; } if (opts.paginate) { // Check if result is paginated const pageLimit = opts.pageLimit || 10; const linkHeader = parseLinkHeader(res.headers.link); if (linkHeader && linkHeader.next && linkHeader.last) { let lastPage = +linkHeader.last.page; if (!process.env.RENOVATE_PAGINATE_ALL && opts.paginate !== 'all') { lastPage = Math.min(pageLimit, lastPage); } const pageNumbers = Array.from( new Array(lastPage), (x, i) => i + 1 ).slice(1); const queue = pageNumbers.map(page => () => { const nextUrl = URL.parse(linkHeader.next.url, true); delete nextUrl.search; nextUrl.query.page = page; return get( URL.format(nextUrl), { ...opts, paginate: false }, retries ); }); const pages = await pAll(queue, { concurrency: 5 }); res.body = res.body.concat( ...pages.filter(Boolean).map(page => page.body) ); } } if ( method === 'get' && (path.startsWith('repos/') || path.startsWith('https://api.github.com/repos/')) ) { cache[path] = res; } // istanbul ignore if if (method === 'POST' && path === 'graphql') { const goodResult = '{"data":{'; if (res.body.startsWith(goodResult)) { if (retries === 0) { logger.info('Recovered graphql query'); } } else if (retries > 0) { logger.info('Retrying graphql query'); opts.body = opts.body.replace('first: 100', 'first: 25'); return get(path, opts, 0); } } return res; } catch (err) { if ( err.name === 'RequestError' && (err.code === 'ENOTFOUND' || err.code === 'ETIMEDOUT') ) { throw new Error('platform-failure'); } if (err.name === 'ParseError') { throw new Error('platform-failure'); } if (err.statusCode >= 500 && err.statusCode < 600) { if (retries > 0) { logger.info( { statusCode: err.statusCode, message: err.message }, `Retrying request` ); await delay(5000 / retries); return get(path, opts, retries - 1); } logger.info({ err }, 'Giving up on GitHub 5xx error'); throw new Error('platform-failure'); } if ( err.statusCode === 403 && err.message && err.message.startsWith('You have triggered an abuse detection mechanism') ) { if (retries > 0) { logger.info( { headers: err.headers, path, statusCode: err.statusCode, message: err.message, }, `Retrying request` ); await delay(180000 / (retries * retries)); return get(path, opts, 0); } // istanbul ignore next throw new Error('platform-failure'); } if ( err.statusCode === 403 && err.message && err.message.startsWith( 'Upgrade to GitHub Pro or make this repository public to enable this feature' ) ) { // istanbul ignore next throw err; } if ( err.statusCode === 403 && err.message && err.message.includes('rate limit exceeded') ) { logger.info({ err, headers: err.headers }, 'Rate limit exceeded'); throw new Error('rate-limit-exceeded'); } else if ( err.statusCode === 403 && err.message && err.message.includes('blobs up to 1 MB in size') ) { throw err; } else if ( err.statusCode === 403 && err.message && err.message.startsWith('Resource not accessible by integration') ) { logger.debug({ err }, 'Unauthorized'); throw new Error('integration-unauthorized'); } else if (err.statusCode === 403) { if (retries > 0) { logger.info( { statusCode: err.statusCode, message: err.message }, `Retrying request` ); await delay(60000 / (retries * retries)); return get(path, opts, retries - 1); } } else if ( err.statusCode === 401 && err.message && err.message.includes('Bad credentials') ) { const rateLimit = err.headers ? err.headers['x-ratelimit-limit'] : -1; logger.info( { token: maskToken(opts.token), err, }, 'Bad credentials' ); if (rateLimit === '60') { throw new Error('platform-failure'); } throw new Error('bad-credentials'); } throw err; } } const helpers = ['get', 'post', 'put', 'patch', 'head', 'delete']; for (const x of helpers) { get[x] = (url, opts) => get(url, Object.assign({}, opts, { method: x.toUpperCase() })); } get.setAppMode = function setAppMode() { // no-op }; get.reset = function reset() { cache = null; cache = {}; // istanbul ignore if if (stats.requests && stats.requests.length > 1) { logger.info( { rateLimit: parseInt(stats.rateLimit, 10), requestCount: stats.requests.length, rateLimitRemaining: parseInt(stats.rateLimitRemaining, 10), }, 'Request stats' ); stats.requests.sort(); logger.debug({ requests: stats.requests }, 'All requests'); stats = {}; } }; get.setEndpoint = e => { endpoint = e; }; module.exports = get;