STRLCPY/gradejs

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/packageMetadata.ts

1	1		import { Column, Entity, Index, PrimaryColumn, BaseEntity } from 'typeorm';
2	2
	3	+	type Maintainer = {
	4	+	name: string;
	5	+	email: string;
	6	+	avatar: string;
	7	+	};
	8	+	type VersionData = { dependencies: Record<string, string>; unpackedSize?: number };
	9	+
3	10		@Entity({ name: 'package_metadata' })
4	11		@Index(['name'], { unique: true })
5	12		export class PackageMetadata extends BaseEntity {
		skipped 11 lines
17	24
18	25		@Column()
19	26		description?: string;
	27	+
	28	+	@Column()
	29	+	fullDescription?: string;
	30	+
	31	+	@Column({ type: 'jsonb' })
	32	+	maintainers?: Maintainer[];
	33	+
	34	+	@Column({ type: 'jsonb' })
	35	+	keywords?: string[];
	36	+
	37	+	@Column({ type: 'jsonb' })
	38	+	versionSpecificValues?: Record<string, VersionData>;
20	39
21	40		@Column()
22	41		homepageUrl?: string;
		skipped 14 lines

■ ■ ■ ■ ■ ■

packages/shared/src/database/migrations/1662405581390-ExtendPackageMetadata.ts

1	+	import { MigrationInterface, QueryRunner } from 'typeorm';
2	+
3	+	export class ExtendPackageMetadata1662405581390 implements MigrationInterface {
4	+	public async up(queryRunner: QueryRunner): Promise<void> {
5	+	await queryRunner.query(`
6	+	alter table "package_metadata" add column "full_description" text;
7	+	alter table "package_metadata" add column "maintainers" jsonb not null default '[]';
8	+	alter table "package_metadata" add column "keywords" jsonb not null default '[]';
9	+	alter table "package_metadata" add column "version_specific_values" jsonb not null default '{}';
10	+	`);
11	+	}
12	+
13	+	public async down(queryRunner: QueryRunner): Promise<void> {
14	+	await queryRunner.query(`
15	+	alter table "package_metadata" drop column "full_description";
16	+	alter table "package_metadata" drop column "maintainers";
17	+	alter table "package_metadata" drop column "keywords";
18	+	alter table "package_metadata" drop column "version_specific_values";
19	+	`);
20	+	}
21	+	}
22	+

■ ■ ■ ■ ■ ■

packages/worker/src/npmRegistry/api.test.ts

		skipped 6 lines
7	7		// TODO: The replicate npm registry may be down sometimes and these tests are flaky
8	8		describe.skip('npmRegistry / api', () => {
9	9		it('fetchPackageMetadata', async () => {
10		-	const metadata = await fetchPackageMetadata('react');
	10	+	const metadata = await fetchPackageMetadata('react-ga');
11	11
12		-	expect(semver.gte(metadata.latestVersion, '18.0.0')).toBeTruthy();
13		-	expect(metadata.repositoryUrl).toEqual('https://github.com/facebook/react');
14		-	expect(metadata.homepageUrl).toEqual('https://reactjs.org/');
	12	+	expect(semver.gte(metadata.latestVersion, '3.0.0')).toBeTruthy();
	13	+	expect(metadata.repositoryUrl).toEqual('git+ssh://git@github.com/react-ga/react-ga.git');
	14	+	expect(metadata.homepageUrl).toEqual('https://github.com/react-ga/react-ga');
	15	+	expect(metadata.maintainers[0].name).toEqual('simeonc');
	16	+	expect(metadata.keywords).toContain('Google Analytics');
	17	+	expect(metadata.versionSpecificValues).toHaveProperty('3.2.1');
	18	+	expect(metadata.versionSpecificValues['3.2.1'].unpackedSize).toEqual(212680);
	19	+	expect(metadata.versionSpecificValues['3.2.1'].dependencies).toHaveProperty('react');
15	20		expect(metadata.description).toContain('React');
16		-	expect(metadata.license).toEqual('MIT');
17		-	expect(metadata.versionList).toContain('17.0.0');
	21	+	expect(metadata.fullDescription).toContain('React Google Analytics Module');
	22	+	expect(metadata.license).toEqual('Apache-2.0');
18	23		expect(metadata.updatedAt > new Date(2010, 1, 1)).toBeTruthy();
19	24		expect(metadata.updateSeq > 100_000).toBeTruthy();
20	25		});
		skipped 21 lines

■ ■ ■ ■ ■ ■

packages/worker/src/npmRegistry/api.ts

1	1		import nano from 'nano';
	2	+	import { createHash } from 'crypto';
2	3		import fetch from 'node-fetch';
3	4
4	5		// We use the relicate API since the `registry.npmjs.com`
5	6		// does not support `local_seq` parameter and the `/changes` endpoint.
6	7		const REGISTRY_URL = 'https://replicate.npmjs.com';
7	8		const NPM_API_ORIGIN = 'https://api.npmjs.org';
	9	+	const makeAvatarUrl = (email: string) =>
	10	+	`https://s.gravatar.com/avatar/${createHash('md5').update(email).digest('hex')}`;
8	11
9	12		type DocumentScope = {
10	13		description?: string;
	14	+	readme?: string;
	15	+	maintainers?: Array<{ name: string; email: string }>;
	16	+	keywords?: string[];
11	17		license?: string;
12	18		homepage?: string;
13	19		repository?:
		skipped 11 lines
25	31		'dist-tags': {
26	32		latest: string;
27	33		};
28		-	versions: Record<string, Record<string, unknown>>;
	34	+	versions?: Record<
	35	+	string,
	36	+	{
	37	+	dependencies?: Record<string, string>;
	38	+	peerDependencies?: Record<string, string>;
	39	+	dist: {
	40	+	unpackedSize?: number;
	41	+	};
	42	+	}
	43	+	>;
29	44		};
30	45
31	46		const registry = nano(REGISTRY_URL).scope<DocumentScope>('registry');
32	47
33	48		export async function fetchPackageMetadata(name: string) {
34	49		const document = await registry.get(name, { local_seq: true });
35		-	const versionList = Object.keys(document.versions);
	50	+	const versionList = Object.keys(document.versions ?? {});
36	51
37	52		let homepageUrl, repositoryUrl;
38	53
		skipped 16 lines
55	70		// );
56	71
57	72		return {
58		-	description: document.description ?? undefined,
59		-	license: document.license ?? undefined,
	73	+	description: document.description,
	74	+	fullDescription: document.readme,
	75	+	maintainers: (document.maintainers ?? []).map((author) => ({
	76	+	name: author.name,
	77	+	email: author.email,
	78	+	avatar: makeAvatarUrl(author.email),
	79	+	})),
	80	+	keywords: document.keywords ?? [],
	81	+	license: document.license,
60	82		homepageUrl,
61	83		repositoryUrl,
62	84		updateSeq: Number(document._local_seq),
63	85		updatedAt: new Date(document.time.modified),
64	86		latestVersion: document['dist-tags'].latest,
65		-	versionList,
	87	+	versionSpecificValues: versionList.reduce((acc, el) => {
	88	+	acc[el] = {
	89	+	dependencies: {
	90	+	...(document.versions?.[el].dependencies ?? {}),
	91	+	...(document.versions?.[el].peerDependencies ?? {}),
	92	+	},
	93	+	unpackedSize: document.versions?.[el].dist.unpackedSize,
	94	+	};
	95	+	return acc;
	96	+	}, {} as Record<string, { dependencies: Record<string, string>; unpackedSize?: number }>),
66	97		};
67	98		}
68	99
		skipped 76 lines

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndexBatch.test.ts

		skipped 21 lines
22	22		Promise.resolve({
23	23		updateSeq: 123,
24	24		updatedAt,
25		-	versionList: ['0.1.0', '1.0.0', '1.0.1', '1.1.0'],
	25	+	versionSpecificValues: { '0.1.0': {}, '1.0.0': {}, '1.0.1': {}, '1.1.0': {} },
26	26		latestVersion: '1.1.0',
27	27		downloads: 500,
28	28		} as any)
		skipped 35 lines
64	64		Promise.resolve({
65	65		updateSeq: savedPackage.updateSeq,
66	66		updatedAt: savedPackage.updatedAt,
67		-	versionList: ['0.1.0', '1.0.0'],
	67	+	versionSpecificValues: { '0.1.0': {}, '1.0.0': {} },
68	68		latestVersion: savedPackage.latestVersion,
69	69		downloads: 100,
70	70		} as any)
		skipped 19 lines

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndexBatch.ts

		skipped 23 lines
24	24		await Promise.all([
25	25		internalApi.requestPackageIndexing({
26	26		name,
27		-	versions: statsAndMetadata.versionList,
	27	+	versions: Object.keys(statsAndMetadata.versionSpecificValues),
28	28		}),
29	29		PackageMetadata.upsert(
30	30		{
31	31		name: name,
32	32		latestVersion: statsAndMetadata.latestVersion,
33	33		description: statsAndMetadata.description,
	34	+	fullDescription: statsAndMetadata.fullDescription,
	35	+	maintainers: statsAndMetadata.maintainers,
	36	+	keywords: statsAndMetadata.keywords,
34	37		homepageUrl: statsAndMetadata.homepageUrl,
35	38		repositoryUrl: statsAndMetadata.repositoryUrl,
36	39		license: statsAndMetadata.license,
	40	+	versionSpecificValues: statsAndMetadata.versionSpecificValues,
37	41		monthlyDownloads: statsAndMetadata.downloads,
38	42		updateSeq: statsAndMetadata.updateSeq,
39	43		updatedAt: statsAndMetadata.updatedAt,
		skipped 6 lines

chore: get more data from npm registry (#55)