mirror of
https://github.com/renovatebot/renovate.git
synced 2025-01-09 21:46:26 +00:00
Merge ae76089e03
into adede1d309
This commit is contained in:
commit
f945623931
7 changed files with 220 additions and 273 deletions
|
@ -1,4 +1,6 @@
|
||||||
import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
import { Readable } from 'node:stream';
|
||||||
|
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||||
|
import { sdkStreamMixin } from '@smithy/util-stream';
|
||||||
import { mockClient } from 'aws-sdk-client-mock';
|
import { mockClient } from 'aws-sdk-client-mock';
|
||||||
import { GoogleAuth as _googleAuth } from 'google-auth-library';
|
import { GoogleAuth as _googleAuth } from 'google-auth-library';
|
||||||
import { DateTime } from 'luxon';
|
import { DateTime } from 'luxon';
|
||||||
|
@ -622,10 +624,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
|
|
||||||
describe('post-fetch release validation', () => {
|
describe('post-fetch release validation', () => {
|
||||||
it('returns null for 404', async () => {
|
it('returns null for 404', async () => {
|
||||||
httpMock
|
httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(404);
|
||||||
.scope(MAVEN_REPO)
|
|
||||||
.head('/foo/bar/1.2.3/bar-1.2.3.pom')
|
|
||||||
.reply(404);
|
|
||||||
|
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
{ datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
|
{ datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
|
||||||
|
@ -635,25 +634,23 @@ describe('modules/datasource/maven/index', () => {
|
||||||
expect(res).toBeNull();
|
expect(res).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns null for unknown error', async () => {
|
it('returns original value for unknown error', async () => {
|
||||||
httpMock
|
httpMock
|
||||||
.scope(MAVEN_REPO)
|
.scope(MAVEN_REPO)
|
||||||
.head('/foo/bar/1.2.3/bar-1.2.3.pom')
|
.get('/foo/bar/1.2.3/bar-1.2.3.pom')
|
||||||
.replyWithError('unknown error');
|
.replyWithError('unknown error');
|
||||||
|
|
||||||
|
const releaseOrig: Release = { version: '1.2.3' };
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
{ datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
|
{ datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
|
||||||
{ version: '1.2.3' },
|
releaseOrig,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(res).toBeNull();
|
expect(res).toBe(releaseOrig);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns original value for 200 response', async () => {
|
it('returns original value for 200 response', async () => {
|
||||||
httpMock
|
httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(200);
|
||||||
.scope(MAVEN_REPO)
|
|
||||||
.head('/foo/bar/1.2.3/bar-1.2.3.pom')
|
|
||||||
.reply(200);
|
|
||||||
const releaseOrig: Release = { version: '1.2.3' };
|
const releaseOrig: Release = { version: '1.2.3' };
|
||||||
|
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
|
@ -665,10 +662,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns original value for 200 response with versionOrig', async () => {
|
it('returns original value for 200 response with versionOrig', async () => {
|
||||||
httpMock
|
httpMock.scope(MAVEN_REPO).get('/foo/bar/1.2.3/bar-1.2.3.pom').reply(200);
|
||||||
.scope(MAVEN_REPO)
|
|
||||||
.head('/foo/bar/1.2.3/bar-1.2.3.pom')
|
|
||||||
.reply(200);
|
|
||||||
const releaseOrig: Release = { version: '1.2', versionOrig: '1.2.3' };
|
const releaseOrig: Release = { version: '1.2', versionOrig: '1.2.3' };
|
||||||
|
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
|
@ -683,13 +677,13 @@ describe('modules/datasource/maven/index', () => {
|
||||||
const releaseOrig: Release = { version: '1.2.3' };
|
const releaseOrig: Release = { version: '1.2.3' };
|
||||||
expect(
|
expect(
|
||||||
await postprocessRelease(
|
await postprocessRelease(
|
||||||
{ datasource, registryUrl: MAVEN_REPO },
|
{ datasource, registryUrl: MAVEN_REPO }, // packageName is missing
|
||||||
releaseOrig,
|
releaseOrig,
|
||||||
),
|
),
|
||||||
).toBe(releaseOrig);
|
).toBe(releaseOrig);
|
||||||
expect(
|
expect(
|
||||||
await postprocessRelease(
|
await postprocessRelease(
|
||||||
{ datasource, packageName: 'foo:bar' },
|
{ datasource, packageName: 'foo:bar' }, // registryUrl is missing
|
||||||
releaseOrig,
|
releaseOrig,
|
||||||
),
|
),
|
||||||
).toBe(releaseOrig);
|
).toBe(releaseOrig);
|
||||||
|
@ -698,7 +692,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
it('adds releaseTimestamp', async () => {
|
it('adds releaseTimestamp', async () => {
|
||||||
httpMock
|
httpMock
|
||||||
.scope(MAVEN_REPO)
|
.scope(MAVEN_REPO)
|
||||||
.head('/foo/bar/1.2.3/bar-1.2.3.pom')
|
.get('/foo/bar/1.2.3/bar-1.2.3.pom')
|
||||||
.reply(200, '', { 'Last-Modified': '2024-01-01T00:00:00.000Z' });
|
.reply(200, '', { 'Last-Modified': '2024-01-01T00:00:00.000Z' });
|
||||||
|
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
|
@ -719,13 +713,22 @@ describe('modules/datasource/maven/index', () => {
|
||||||
s3mock.reset();
|
s3mock.reset();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function body(input: string): ReturnType<typeof sdkStreamMixin> {
|
||||||
|
const result = new Readable();
|
||||||
|
result.push(input);
|
||||||
|
result.push(null);
|
||||||
|
return sdkStreamMixin(result);
|
||||||
|
}
|
||||||
|
|
||||||
it('checks package', async () => {
|
it('checks package', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
.resolvesOnce({});
|
.resolvesOnce({
|
||||||
|
Body: body('foo'),
|
||||||
|
});
|
||||||
|
|
||||||
const res = await postprocessRelease(
|
const res = await postprocessRelease(
|
||||||
{ datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
|
{ datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
|
||||||
|
@ -737,11 +740,12 @@ describe('modules/datasource/maven/index', () => {
|
||||||
|
|
||||||
it('supports timestamp', async () => {
|
it('supports timestamp', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
.resolvesOnce({
|
.resolvesOnce({
|
||||||
|
Body: body('foo'),
|
||||||
LastModified: DateTime.fromISO(
|
LastModified: DateTime.fromISO(
|
||||||
'2024-01-01T00:00:00.000Z',
|
'2024-01-01T00:00:00.000Z',
|
||||||
).toJSDate(),
|
).toJSDate(),
|
||||||
|
@ -760,7 +764,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
|
|
||||||
it('returns null for deleted object', async () => {
|
it('returns null for deleted object', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
|
@ -778,7 +782,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
|
|
||||||
it('returns null for NotFound response', async () => {
|
it('returns null for NotFound response', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
|
@ -796,7 +800,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
|
|
||||||
it('returns null for NoSuchKey response', async () => {
|
it('returns null for NoSuchKey response', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
|
@ -812,9 +816,9 @@ describe('modules/datasource/maven/index', () => {
|
||||||
expect(res).toBeNull();
|
expect(res).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns null for unknown error', async () => {
|
it('returns original value for any other error', async () => {
|
||||||
s3mock
|
s3mock
|
||||||
.on(HeadObjectCommand, {
|
.on(GetObjectCommand, {
|
||||||
Bucket: 'bucket',
|
Bucket: 'bucket',
|
||||||
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
|
||||||
})
|
})
|
||||||
|
@ -827,7 +831,7 @@ describe('modules/datasource/maven/index', () => {
|
||||||
releaseOrig,
|
releaseOrig,
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(res).toBeNull();
|
expect(res).toBe(releaseOrig);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
import is from '@sindresorhus/is';
|
|
||||||
import type { XmlDocument } from 'xmldoc';
|
import type { XmlDocument } from 'xmldoc';
|
||||||
import { GlobalConfig } from '../../../config/global';
|
import { GlobalConfig } from '../../../config/global';
|
||||||
import { logger } from '../../../logger';
|
import { logger } from '../../../logger';
|
||||||
import * as packageCache from '../../../util/cache/package';
|
import * as packageCache from '../../../util/cache/package';
|
||||||
import { cache } from '../../../util/cache/package/decorator';
|
import { cache } from '../../../util/cache/package/decorator';
|
||||||
|
import { Result } from '../../../util/result';
|
||||||
import { ensureTrailingSlash } from '../../../util/url';
|
import { ensureTrailingSlash } from '../../../util/url';
|
||||||
import mavenVersion from '../../versioning/maven';
|
import mavenVersion from '../../versioning/maven';
|
||||||
import * as mavenVersioning from '../../versioning/maven';
|
import * as mavenVersioning from '../../versioning/maven';
|
||||||
|
@ -18,10 +18,10 @@ import type {
|
||||||
ReleaseResult,
|
ReleaseResult,
|
||||||
} from '../types';
|
} from '../types';
|
||||||
import { MAVEN_REPO } from './common';
|
import { MAVEN_REPO } from './common';
|
||||||
import type { MavenDependency } from './types';
|
import type { MavenDependency, MavenFetchError } from './types';
|
||||||
import {
|
import {
|
||||||
checkResource,
|
|
||||||
createUrlForDependencyPom,
|
createUrlForDependencyPom,
|
||||||
|
downloadMaven,
|
||||||
downloadMavenXml,
|
downloadMavenXml,
|
||||||
getDependencyInfo,
|
getDependencyInfo,
|
||||||
getDependencyParts,
|
getDependencyParts,
|
||||||
|
@ -93,24 +93,29 @@ export class MavenDatasource extends Datasource {
|
||||||
return cachedVersions;
|
return cachedVersions;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { isCacheable, xml: mavenMetadata } = await downloadMavenXml(
|
const metadataXmlResult = await downloadMavenXml(this.http, metadataUrl);
|
||||||
this.http,
|
return metadataXmlResult
|
||||||
metadataUrl,
|
.transform(
|
||||||
);
|
async ({ isCacheable, data: mavenMetadata }): Promise<string[]> => {
|
||||||
if (!mavenMetadata) {
|
const versions = extractVersions(mavenMetadata);
|
||||||
return [];
|
const cachePrivatePackages = GlobalConfig.get(
|
||||||
}
|
'cachePrivatePackages',
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
|
||||||
const versions = extractVersions(mavenMetadata);
|
if (cachePrivatePackages || isCacheable) {
|
||||||
const cachePrivatePackages = GlobalConfig.get(
|
await packageCache.set(cacheNamespace, cacheKey, versions, 30);
|
||||||
'cachePrivatePackages',
|
}
|
||||||
false,
|
|
||||||
);
|
|
||||||
if (cachePrivatePackages || isCacheable) {
|
|
||||||
await packageCache.set(cacheNamespace, cacheKey, versions, 30);
|
|
||||||
}
|
|
||||||
|
|
||||||
return versions;
|
return versions;
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.onError((err) => {
|
||||||
|
logger.debug(
|
||||||
|
`Maven: error fetching versions for "${dependency.display}": ${err.type}`,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.unwrapOr([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
async getReleases({
|
async getReleases({
|
||||||
|
@ -190,17 +195,22 @@ export class MavenDatasource extends Datasource {
|
||||||
);
|
);
|
||||||
|
|
||||||
const artifactUrl = getMavenUrl(dependency, registryUrl, pomUrl);
|
const artifactUrl = getMavenUrl(dependency, registryUrl, pomUrl);
|
||||||
|
const fetchResult = await downloadMaven(this.http, artifactUrl);
|
||||||
|
return fetchResult
|
||||||
|
.transform((res): PostprocessReleaseResult => {
|
||||||
|
if (res.lastModified) {
|
||||||
|
release.releaseTimestamp = res.lastModified;
|
||||||
|
}
|
||||||
|
|
||||||
const res = await checkResource(this.http, artifactUrl);
|
return release;
|
||||||
|
})
|
||||||
|
.catch((err): Result<PostprocessReleaseResult, MavenFetchError> => {
|
||||||
|
if (err.type === 'not-found') {
|
||||||
|
return Result.ok('reject');
|
||||||
|
}
|
||||||
|
|
||||||
if (res === 'not-found' || res === 'error') {
|
return Result.ok(release);
|
||||||
return 'reject';
|
})
|
||||||
}
|
.unwrapOr(release);
|
||||||
|
|
||||||
if (is.date(res)) {
|
|
||||||
release.releaseTimestamp = res.toISOString();
|
|
||||||
}
|
|
||||||
|
|
||||||
return release;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
import type { XmlDocument } from 'xmldoc';
|
|
||||||
import type { Result } from '../../../util/result';
|
import type { Result } from '../../../util/result';
|
||||||
import type { ReleaseResult } from '../types';
|
import type { ReleaseResult } from '../types';
|
||||||
|
|
||||||
|
@ -9,13 +8,6 @@ export interface MavenDependency {
|
||||||
dependencyUrl: string;
|
dependencyUrl: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MavenXml {
|
|
||||||
isCacheable?: boolean;
|
|
||||||
xml?: XmlDocument;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type HttpResourceCheckResult = 'found' | 'not-found' | 'error' | Date;
|
|
||||||
|
|
||||||
export type DependencyInfo = Pick<
|
export type DependencyInfo = Pick<
|
||||||
ReleaseResult,
|
ReleaseResult,
|
||||||
'homepage' | 'sourceUrl' | 'packageScope'
|
'homepage' | 'sourceUrl' | 'packageScope'
|
||||||
|
@ -41,6 +33,7 @@ export type MavenFetchError =
|
||||||
| { type: 'unsupported-protocol' }
|
| { type: 'unsupported-protocol' }
|
||||||
| { type: 'credentials-error' }
|
| { type: 'credentials-error' }
|
||||||
| { type: 'missing-aws-region' }
|
| { type: 'missing-aws-region' }
|
||||||
|
| { type: 'xml-parse-error'; err: Error }
|
||||||
| { type: 'unknown'; err: Error };
|
| { type: 'unknown'; err: Error };
|
||||||
|
|
||||||
export type MavenFetchResult<T = string> = Result<
|
export type MavenFetchResult<T = string> = Result<
|
||||||
|
|
|
@ -4,7 +4,6 @@ import { HOST_DISABLED } from '../../../constants/error-messages';
|
||||||
import { Http, HttpError } from '../../../util/http';
|
import { Http, HttpError } from '../../../util/http';
|
||||||
import type { MavenFetchError } from './types';
|
import type { MavenFetchError } from './types';
|
||||||
import {
|
import {
|
||||||
checkResource,
|
|
||||||
downloadHttpProtocol,
|
downloadHttpProtocol,
|
||||||
downloadMavenXml,
|
downloadMavenXml,
|
||||||
downloadS3Protocol,
|
downloadS3Protocol,
|
||||||
|
@ -46,17 +45,36 @@ function httpError({
|
||||||
|
|
||||||
describe('modules/datasource/maven/util', () => {
|
describe('modules/datasource/maven/util', () => {
|
||||||
describe('downloadMavenXml', () => {
|
describe('downloadMavenXml', () => {
|
||||||
it('returns empty object for unsupported protocols', async () => {
|
it('returns error for unsupported protocols', async () => {
|
||||||
const res = await downloadMavenXml(
|
const res = await downloadMavenXml(
|
||||||
http,
|
http,
|
||||||
new URL('unsupported://server.com/'),
|
new URL('unsupported://server.com/'),
|
||||||
);
|
);
|
||||||
expect(res).toEqual({});
|
expect(res.unwrap()).toEqual({
|
||||||
|
ok: false,
|
||||||
|
err: { type: 'unsupported-protocol' } satisfies MavenFetchError,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns error for xml parse error', async () => {
|
||||||
|
const http = partial<Http>({
|
||||||
|
get: () =>
|
||||||
|
Promise.resolve({
|
||||||
|
statusCode: 200,
|
||||||
|
body: 'invalid xml',
|
||||||
|
headers: {},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const res = await downloadMavenXml(http, 'https://example.com/');
|
||||||
|
expect(res.unwrap()).toEqual({
|
||||||
|
ok: false,
|
||||||
|
err: { type: 'xml-parse-error', err: expect.any(Error) },
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('downloadS3Protocol', () => {
|
describe('downloadS3Protocol', () => {
|
||||||
it('fails for non-S3 URLs', async () => {
|
it('returns error for non-S3 URLs', async () => {
|
||||||
const res = await downloadS3Protocol(new URL('http://not-s3.com/'));
|
const res = await downloadS3Protocol(new URL('http://not-s3.com/'));
|
||||||
expect(res.unwrap()).toEqual({
|
expect(res.unwrap()).toEqual({
|
||||||
ok: false,
|
ok: false,
|
||||||
|
@ -122,16 +140,4 @@ describe('modules/datasource/maven/util', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('checkResource', () => {
|
|
||||||
it('returns not found for unsupported protocols', async () => {
|
|
||||||
const res = await checkResource(http, 'unsupported://server.com/');
|
|
||||||
expect(res).toBe('not-found');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns error for invalid URLs', async () => {
|
|
||||||
const res = await checkResource(http, 'not-a-valid-url');
|
|
||||||
expect(res).toBe('error');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import { Readable } from 'node:stream';
|
import { Readable } from 'node:stream';
|
||||||
import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
import { GetObjectCommand } from '@aws-sdk/client-s3';
|
||||||
import { DateTime } from 'luxon';
|
|
||||||
import { XmlDocument } from 'xmldoc';
|
import { XmlDocument } from 'xmldoc';
|
||||||
import { HOST_DISABLED } from '../../../constants/error-messages';
|
import { HOST_DISABLED } from '../../../constants/error-messages';
|
||||||
import { logger } from '../../../logger';
|
import { logger } from '../../../logger';
|
||||||
|
@ -9,7 +8,6 @@ import { type Http, HttpError } from '../../../util/http';
|
||||||
import type { HttpOptions, HttpResponse } from '../../../util/http/types';
|
import type { HttpOptions, HttpResponse } from '../../../util/http/types';
|
||||||
import { regEx } from '../../../util/regex';
|
import { regEx } from '../../../util/regex';
|
||||||
import { Result } from '../../../util/result';
|
import { Result } from '../../../util/result';
|
||||||
import type { S3UrlParts } from '../../../util/s3';
|
|
||||||
import { getS3Client, parseS3Url } from '../../../util/s3';
|
import { getS3Client, parseS3Url } from '../../../util/s3';
|
||||||
import { streamToString } from '../../../util/streams';
|
import { streamToString } from '../../../util/streams';
|
||||||
import { ensureTrailingSlash, parseUrl } from '../../../util/url';
|
import { ensureTrailingSlash, parseUrl } from '../../../util/url';
|
||||||
|
@ -18,11 +16,9 @@ import { getGoogleAuthToken } from '../util';
|
||||||
import { MAVEN_REPO } from './common';
|
import { MAVEN_REPO } from './common';
|
||||||
import type {
|
import type {
|
||||||
DependencyInfo,
|
DependencyInfo,
|
||||||
HttpResourceCheckResult,
|
|
||||||
MavenDependency,
|
MavenDependency,
|
||||||
MavenFetchResult,
|
MavenFetchResult,
|
||||||
MavenFetchSuccess,
|
MavenFetchSuccess,
|
||||||
MavenXml,
|
|
||||||
} from './types';
|
} from './types';
|
||||||
|
|
||||||
function getHost(url: string): string | null {
|
function getHost(url: string): string | null {
|
||||||
|
@ -268,92 +264,6 @@ export async function downloadArtifactRegistryProtocol(
|
||||||
return downloadHttpProtocol(http, url, opts);
|
return downloadHttpProtocol(http, url, opts);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkHttpResource(
|
|
||||||
http: Http,
|
|
||||||
pkgUrl: URL,
|
|
||||||
): Promise<HttpResourceCheckResult> {
|
|
||||||
try {
|
|
||||||
const res = await http.head(pkgUrl.toString());
|
|
||||||
const timestamp = res?.headers?.['last-modified'];
|
|
||||||
if (timestamp) {
|
|
||||||
const isoTimestamp = normalizeDate(timestamp);
|
|
||||||
if (isoTimestamp) {
|
|
||||||
const releaseDate = DateTime.fromISO(isoTimestamp, {
|
|
||||||
zone: 'UTC',
|
|
||||||
}).toJSDate();
|
|
||||||
return releaseDate;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 'found';
|
|
||||||
} catch (err) {
|
|
||||||
if (isNotFoundError(err)) {
|
|
||||||
return 'not-found';
|
|
||||||
}
|
|
||||||
|
|
||||||
const failedUrl = pkgUrl.toString();
|
|
||||||
logger.debug(
|
|
||||||
{ failedUrl, statusCode: err.statusCode },
|
|
||||||
`Can't check HTTP resource existence`,
|
|
||||||
);
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function checkS3Resource(
|
|
||||||
s3Url: S3UrlParts,
|
|
||||||
): Promise<HttpResourceCheckResult> {
|
|
||||||
try {
|
|
||||||
const response = await getS3Client().send(new HeadObjectCommand(s3Url));
|
|
||||||
if (response.DeleteMarker) {
|
|
||||||
return 'not-found';
|
|
||||||
}
|
|
||||||
if (response.LastModified) {
|
|
||||||
return response.LastModified;
|
|
||||||
}
|
|
||||||
return 'found';
|
|
||||||
} catch (err) {
|
|
||||||
if (isS3NotFound(err)) {
|
|
||||||
return 'not-found';
|
|
||||||
} else {
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
bucket: s3Url.Bucket,
|
|
||||||
key: s3Url.Key,
|
|
||||||
name: err.name,
|
|
||||||
message: err.message,
|
|
||||||
},
|
|
||||||
`Can't check S3 resource existence`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function checkResource(
|
|
||||||
http: Http,
|
|
||||||
pkgUrl: URL | string,
|
|
||||||
): Promise<HttpResourceCheckResult> {
|
|
||||||
const parsedUrl = typeof pkgUrl === 'string' ? parseUrl(pkgUrl) : pkgUrl;
|
|
||||||
if (parsedUrl === null) {
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
|
|
||||||
const s3Url = parseS3Url(parsedUrl);
|
|
||||||
if (s3Url) {
|
|
||||||
return await checkS3Resource(s3Url);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parsedUrl.protocol === 'http:' || parsedUrl.protocol === 'https:') {
|
|
||||||
return await checkHttpResource(http, parsedUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
{ url: pkgUrl.toString() },
|
|
||||||
`Unsupported Maven protocol in check resource`,
|
|
||||||
);
|
|
||||||
return 'not-found';
|
|
||||||
}
|
|
||||||
|
|
||||||
function containsPlaceholder(str: string): boolean {
|
function containsPlaceholder(str: string): boolean {
|
||||||
return regEx(/\${.*?}/g).test(str);
|
return regEx(/\${.*?}/g).test(str);
|
||||||
}
|
}
|
||||||
|
@ -369,44 +279,57 @@ export function getMavenUrl(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function downloadMaven(
|
||||||
|
http: Http,
|
||||||
|
url: URL | string,
|
||||||
|
): Promise<MavenFetchResult> {
|
||||||
|
const pkgUrl = url instanceof URL ? url : parseUrl(url);
|
||||||
|
// istanbul ignore if
|
||||||
|
if (!pkgUrl) {
|
||||||
|
return Result.err({ type: 'invalid-url' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const protocol = pkgUrl.protocol.slice(0, -1);
|
||||||
|
|
||||||
|
let result: MavenFetchResult = Result.err({ type: 'unsupported-protocol' });
|
||||||
|
|
||||||
|
if (protocol === 'http' || protocol === 'https') {
|
||||||
|
result = await downloadHttpProtocol(http, pkgUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (protocol === 'artifactregistry') {
|
||||||
|
result = await downloadArtifactRegistryProtocol(http, pkgUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (protocol === 's3') {
|
||||||
|
result = await downloadS3Protocol(pkgUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.onError((err) => {
|
||||||
|
if (err.type === 'unsupported-protocol') {
|
||||||
|
logger.debug(
|
||||||
|
{ url: pkgUrl.toString() },
|
||||||
|
`Maven lookup error: unsupported protocol (${protocol})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function downloadMavenXml(
|
export async function downloadMavenXml(
|
||||||
http: Http,
|
http: Http,
|
||||||
pkgUrl: URL,
|
url: URL | string,
|
||||||
): Promise<MavenXml> {
|
): Promise<MavenFetchResult<XmlDocument>> {
|
||||||
const protocol = pkgUrl.protocol;
|
const rawResult = await downloadMaven(http, url);
|
||||||
|
return rawResult.transform((result): MavenFetchResult<XmlDocument> => {
|
||||||
if (protocol === 'http:' || protocol === 'https:') {
|
try {
|
||||||
const rawResult = await downloadHttpProtocol(http, pkgUrl);
|
return Result.ok({
|
||||||
const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
|
...result,
|
||||||
const xml = new XmlDocument(data);
|
data: new XmlDocument(result.data),
|
||||||
return { isCacheable, xml };
|
});
|
||||||
});
|
} catch (err) {
|
||||||
return xmlResult.unwrapOr({});
|
return Result.err({ type: 'xml-parse-error', err });
|
||||||
}
|
}
|
||||||
|
});
|
||||||
if (protocol === 'artifactregistry:') {
|
|
||||||
const rawResult = await downloadArtifactRegistryProtocol(http, pkgUrl);
|
|
||||||
const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
|
|
||||||
const xml = new XmlDocument(data);
|
|
||||||
return { isCacheable, xml };
|
|
||||||
});
|
|
||||||
return xmlResult.unwrapOr({});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (protocol === 's3:') {
|
|
||||||
const rawResult = await downloadS3Protocol(pkgUrl);
|
|
||||||
const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
|
|
||||||
const xml = new XmlDocument(data);
|
|
||||||
return { xml };
|
|
||||||
});
|
|
||||||
return xmlResult.unwrapOr({});
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
{ url: pkgUrl.toString() },
|
|
||||||
`Content is not found for Maven url`,
|
|
||||||
);
|
|
||||||
return {};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getDependencyParts(packageName: string): MavenDependency {
|
export function getDependencyParts(packageName: string): MavenDependency {
|
||||||
|
@ -458,13 +381,14 @@ async function getSnapshotFullVersion(
|
||||||
`${version}/maven-metadata.xml`,
|
`${version}/maven-metadata.xml`,
|
||||||
);
|
);
|
||||||
|
|
||||||
const { xml: mavenMetadata } = await downloadMavenXml(http, metadataUrl);
|
const metadataXmlResult = await downloadMavenXml(http, metadataUrl);
|
||||||
// istanbul ignore if: hard to test
|
|
||||||
if (!mavenMetadata) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return extractSnapshotVersion(mavenMetadata);
|
return metadataXmlResult
|
||||||
|
.transform(({ data }) => {
|
||||||
|
const nullErr = { type: 'snapshot-extract-error' };
|
||||||
|
return Result.wrapNullable(extractSnapshotVersion(data), nullErr);
|
||||||
|
})
|
||||||
|
.unwrapOrNull();
|
||||||
}
|
}
|
||||||
|
|
||||||
function isSnapshotVersion(version: string): boolean {
|
function isSnapshotVersion(version: string): boolean {
|
||||||
|
@ -508,7 +432,6 @@ export async function getDependencyInfo(
|
||||||
version: string,
|
version: string,
|
||||||
recursionLimit = 5,
|
recursionLimit = 5,
|
||||||
): Promise<DependencyInfo> {
|
): Promise<DependencyInfo> {
|
||||||
const result: DependencyInfo = {};
|
|
||||||
const path = await createUrlForDependencyPom(
|
const path = await createUrlForDependencyPom(
|
||||||
http,
|
http,
|
||||||
version,
|
version,
|
||||||
|
@ -517,64 +440,71 @@ export async function getDependencyInfo(
|
||||||
);
|
);
|
||||||
|
|
||||||
const pomUrl = getMavenUrl(dependency, repoUrl, path);
|
const pomUrl = getMavenUrl(dependency, repoUrl, path);
|
||||||
const { xml: pomContent } = await downloadMavenXml(http, pomUrl);
|
const pomXmlResult = await downloadMavenXml(http, pomUrl);
|
||||||
// istanbul ignore if
|
const dependencyInfoResult = await pomXmlResult.transform(
|
||||||
if (!pomContent) {
|
async ({ data: pomContent }) => {
|
||||||
return result;
|
const result: DependencyInfo = {};
|
||||||
}
|
|
||||||
|
|
||||||
const homepage = pomContent.valueWithPath('url');
|
const homepage = pomContent.valueWithPath('url');
|
||||||
if (homepage && !containsPlaceholder(homepage)) {
|
if (homepage && !containsPlaceholder(homepage)) {
|
||||||
result.homepage = homepage;
|
result.homepage = homepage;
|
||||||
}
|
|
||||||
|
|
||||||
const sourceUrl = pomContent.valueWithPath('scm.url');
|
|
||||||
if (sourceUrl && !containsPlaceholder(sourceUrl)) {
|
|
||||||
result.sourceUrl = sourceUrl
|
|
||||||
.replace(regEx(/^scm:/), '')
|
|
||||||
.replace(regEx(/^git:/), '')
|
|
||||||
.replace(regEx(/^git@github.com:/), 'https://github.com/')
|
|
||||||
.replace(regEx(/^git@github.com\//), 'https://github.com/');
|
|
||||||
|
|
||||||
if (result.sourceUrl.startsWith('//')) {
|
|
||||||
// most likely the result of us stripping scm:, git: etc
|
|
||||||
// going with prepending https: here which should result in potential information retrival
|
|
||||||
result.sourceUrl = `https:${result.sourceUrl}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const groupId = pomContent.valueWithPath('groupId');
|
|
||||||
if (groupId) {
|
|
||||||
result.packageScope = groupId;
|
|
||||||
}
|
|
||||||
|
|
||||||
const parent = pomContent.childNamed('parent');
|
|
||||||
if (recursionLimit > 0 && parent && (!result.sourceUrl || !result.homepage)) {
|
|
||||||
// if we found a parent and are missing some information
|
|
||||||
// trying to get the scm/homepage information from it
|
|
||||||
const [parentGroupId, parentArtifactId, parentVersion] = [
|
|
||||||
'groupId',
|
|
||||||
'artifactId',
|
|
||||||
'version',
|
|
||||||
].map((k) => parent.valueWithPath(k)?.replace(/\s+/g, ''));
|
|
||||||
if (parentGroupId && parentArtifactId && parentVersion) {
|
|
||||||
const parentDisplayId = `${parentGroupId}:${parentArtifactId}`;
|
|
||||||
const parentDependency = getDependencyParts(parentDisplayId);
|
|
||||||
const parentInformation = await getDependencyInfo(
|
|
||||||
http,
|
|
||||||
parentDependency,
|
|
||||||
repoUrl,
|
|
||||||
parentVersion,
|
|
||||||
recursionLimit - 1,
|
|
||||||
);
|
|
||||||
if (!result.sourceUrl && parentInformation.sourceUrl) {
|
|
||||||
result.sourceUrl = parentInformation.sourceUrl;
|
|
||||||
}
|
}
|
||||||
if (!result.homepage && parentInformation.homepage) {
|
|
||||||
result.homepage = parentInformation.homepage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
const sourceUrl = pomContent.valueWithPath('scm.url');
|
||||||
|
if (sourceUrl && !containsPlaceholder(sourceUrl)) {
|
||||||
|
result.sourceUrl = sourceUrl
|
||||||
|
.replace(regEx(/^scm:/), '')
|
||||||
|
.replace(regEx(/^git:/), '')
|
||||||
|
.replace(regEx(/^git@github.com:/), 'https://github.com/')
|
||||||
|
.replace(regEx(/^git@github.com\//), 'https://github.com/');
|
||||||
|
|
||||||
|
if (result.sourceUrl.startsWith('//')) {
|
||||||
|
// most likely the result of us stripping scm:, git: etc
|
||||||
|
// going with prepending https: here which should result in potential information retrival
|
||||||
|
result.sourceUrl = `https:${result.sourceUrl}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const groupId = pomContent.valueWithPath('groupId');
|
||||||
|
if (groupId) {
|
||||||
|
result.packageScope = groupId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parent = pomContent.childNamed('parent');
|
||||||
|
if (
|
||||||
|
recursionLimit > 0 &&
|
||||||
|
parent &&
|
||||||
|
(!result.sourceUrl || !result.homepage)
|
||||||
|
) {
|
||||||
|
// if we found a parent and are missing some information
|
||||||
|
// trying to get the scm/homepage information from it
|
||||||
|
const [parentGroupId, parentArtifactId, parentVersion] = [
|
||||||
|
'groupId',
|
||||||
|
'artifactId',
|
||||||
|
'version',
|
||||||
|
].map((k) => parent.valueWithPath(k)?.replace(/\s+/g, ''));
|
||||||
|
if (parentGroupId && parentArtifactId && parentVersion) {
|
||||||
|
const parentDisplayId = `${parentGroupId}:${parentArtifactId}`;
|
||||||
|
const parentDependency = getDependencyParts(parentDisplayId);
|
||||||
|
const parentInformation = await getDependencyInfo(
|
||||||
|
http,
|
||||||
|
parentDependency,
|
||||||
|
repoUrl,
|
||||||
|
parentVersion,
|
||||||
|
recursionLimit - 1,
|
||||||
|
);
|
||||||
|
if (!result.sourceUrl && parentInformation.sourceUrl) {
|
||||||
|
result.sourceUrl = parentInformation.sourceUrl;
|
||||||
|
}
|
||||||
|
if (!result.homepage && parentInformation.homepage) {
|
||||||
|
result.homepage = parentInformation.homepage;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return dependencyInfoResult.unwrapOr({});
|
||||||
}
|
}
|
||||||
|
|
|
@ -269,6 +269,7 @@
|
||||||
"@openpgp/web-stream-tools": "0.1.3",
|
"@openpgp/web-stream-tools": "0.1.3",
|
||||||
"@renovate/eslint-plugin": "file:tools/eslint",
|
"@renovate/eslint-plugin": "file:tools/eslint",
|
||||||
"@semantic-release/exec": "6.0.3",
|
"@semantic-release/exec": "6.0.3",
|
||||||
|
"@smithy/util-stream": "3.3.4",
|
||||||
"@swc/core": "1.10.4",
|
"@swc/core": "1.10.4",
|
||||||
"@types/auth-header": "1.0.6",
|
"@types/auth-header": "1.0.6",
|
||||||
"@types/aws4": "1.11.6",
|
"@types/aws4": "1.11.6",
|
||||||
|
|
|
@ -379,6 +379,9 @@ importers:
|
||||||
'@semantic-release/exec':
|
'@semantic-release/exec':
|
||||||
specifier: 6.0.3
|
specifier: 6.0.3
|
||||||
version: 6.0.3(semantic-release@24.2.0(typescript@5.7.2))
|
version: 6.0.3(semantic-release@24.2.0(typescript@5.7.2))
|
||||||
|
'@smithy/util-stream':
|
||||||
|
specifier: 3.3.4
|
||||||
|
version: 3.3.4
|
||||||
'@swc/core':
|
'@swc/core':
|
||||||
specifier: 1.10.4
|
specifier: 1.10.4
|
||||||
version: 1.10.4
|
version: 1.10.4
|
||||||
|
|
Loading…
Reference in a new issue