import crypto from 'crypto'; import merge from 'deepmerge'; import got, { Options, Response } from 'got'; import { HOST_DISABLED } from '../../constants/error-messages'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as memCache from '../cache/memory'; import { clone } from '../clone'; import { resolveBaseUrl } from '../url'; import { applyAuthorization, removeAuthorization } from './auth'; import { applyHostRules } from './host-rules'; import { getQueue } from './queue'; import type { GotJSONOptions, GotOptions, OutgoingHttpHeaders, RequestStats, } from './types'; // TODO: refactor code to remove this (#9651) import './legacy'; export interface HttpOptions { body?: any; username?: string; password?: string; baseUrl?: string; headers?: OutgoingHttpHeaders; /** * Do not use authentication */ noAuth?: boolean; throwHttpErrors?: boolean; useCache?: boolean; } export interface HttpPostOptions extends HttpOptions { body: unknown; } export interface InternalHttpOptions extends HttpOptions { json?: Record; responseType?: 'json' | 'buffer'; method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head'; } export interface HttpResponse { statusCode: number; body: T; headers: any; authorization?: boolean; } function cloneResponse( response: HttpResponse ): HttpResponse { const { body, statusCode, headers } = response; // clone body and headers so that the cached result doesn't get accidentally mutated // Don't use json clone for buffers return { statusCode, body: body instanceof Buffer ? (body.slice() as T) : clone(body), headers: clone(headers), authorization: !!response.authorization, }; } function applyDefaultHeaders(options: Options): void { let renovateVersion = 'unknown'; try { // eslint-disable-next-line @typescript-eslint/no-var-requires renovateVersion = require('../../../package.json').version; // eslint-disable-line global-require } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'Error getting renovate version'); } // eslint-disable-next-line no-param-reassign options.headers = { ...options.headers, 'user-agent': process.env.RENOVATE_USER_AGENT || `RenovateBot/${renovateVersion} (https://github.com/renovatebot/renovate)`, }; } // Note on types: // options.requestType can be either 'json' or 'buffer', but `T` should be // `Buffer` in the latter case. // We don't declare overload signatures because it's immediately wrapped by // `request`. async function gotRoutine( url: string, options: GotOptions, requestStats: Partial ): Promise> { logger.trace({ url, options }, 'got request'); // Cheat the TS compiler using `as` to pick a specific overload. // Otherwise it doesn't typecheck. const resp = await got(url, options as GotJSONOptions); const duration = resp.timings.phases.total || /* istanbul ignore next: can't be tested */ 0; const httpRequests = memCache.get('http-requests') || []; httpRequests.push({ ...requestStats, duration }); memCache.set('http-requests', httpRequests); return resp; } export class Http { private options?: GotOptions; constructor(private hostType: string, options?: HttpOptions) { this.options = merge(options, { context: { hostType } }); } protected async request( requestUrl: string | URL, httpOptions?: InternalHttpOptions ): Promise | null> { let url = requestUrl.toString(); if (httpOptions?.baseUrl) { url = resolveBaseUrl(httpOptions.baseUrl, url); } let options: GotOptions = merge( { method: 'get', ...this.options, hostType: this.hostType, }, httpOptions ); if (process.env.NODE_ENV === 'test') { options.retry = 0; } options.hooks = { beforeRedirect: [removeAuthorization], }; applyDefaultHeaders(options); options = applyHostRules(url, options); if (options.enabled === false) { throw new Error(HOST_DISABLED); } options = applyAuthorization(options); const cacheKey = crypto .createHash('md5') .update( 'got-' + JSON.stringify({ url, headers: options.headers, method: options.method, }) ) .digest('hex'); let resPromise; // Cache GET requests unless useCache=false if ( ['get', 'head'].includes(options.method) && options.useCache !== false ) { resPromise = memCache.get(cacheKey); } // istanbul ignore else: no cache tests if (!resPromise) { const startTime = Date.now(); const queueTask = (): Promise> => { const queueDuration = Date.now() - startTime; return gotRoutine(url, options, { method: options.method, url, queueDuration, }); }; const queue = getQueue(url); resPromise = queue?.add(queueTask) ?? queueTask(); if (options.method === 'get') { memCache.set(cacheKey, resPromise); // always set if it's a get } } try { const res = await resPromise; res.authorization = !!options?.headers?.authorization; return cloneResponse(res); } catch (err) { const { abortOnError, abortIgnoreStatusCodes } = options; if (abortOnError && !abortIgnoreStatusCodes?.includes(err.statusCode)) { throw new ExternalHostError(err); } throw err; } } get(url: string, options: HttpOptions = {}): Promise { return this.request(url, options); } head(url: string, options: HttpOptions = {}): Promise { return this.request(url, { ...options, method: 'head' }); } protected requestBuffer( url: string | URL, httpOptions?: InternalHttpOptions ): Promise | null> { return this.request(url, { ...httpOptions, responseType: 'buffer', }); } getBuffer( url: string, options: HttpOptions = {} ): Promise | null> { return this.requestBuffer(url, options); } private async requestJson( url: string, options: InternalHttpOptions ): Promise> { const { body, ...jsonOptions } = options; if (body) { jsonOptions.json = body; } const res = await this.request(url, { ...jsonOptions, responseType: 'json', }); return { ...res, body: res.body }; } getJson( url: string, options?: GetOptions ): Promise> { return this.requestJson(url, { ...options }); } headJson( url: string, options?: GetOptions ): Promise> { return this.requestJson(url, { ...options, method: 'head' }); } postJson( url: string, options?: PostOptions ): Promise> { return this.requestJson(url, { ...options, method: 'post' }); } putJson( url: string, options?: PostOptions ): Promise> { return this.requestJson(url, { ...options, method: 'put' }); } patchJson( url: string, options?: PostOptions ): Promise> { return this.requestJson(url, { ...options, method: 'patch' }); } deleteJson( url: string, options?: PostOptions ): Promise> { return this.requestJson(url, { ...options, method: 'delete' }); } stream(url: string, options?: HttpOptions): NodeJS.ReadableStream { const combinedOptions: any = { method: 'get', ...this.options, hostType: this.hostType, ...options, }; // istanbul ignore else: needs test if (options?.baseUrl) { // eslint-disable-next-line no-param-reassign url = resolveBaseUrl(options.baseUrl, url); } applyDefaultHeaders(combinedOptions); return got.stream(url, combinedOptions); } }