STRLCPY/gradejs

■ ■ ■ ■ ■ ■

packages/public-api/src/clientApiRouter.test.ts

		skipped 1 lines
2	2		import {
3	3		getDatabaseConnection,
4	4		Hostname,
5		-	internalApi,
	5	+	systemApi,
6	6		WebPage,
7	7		WebPageScan,
8	8		} from '@gradejs-public/shared';
		skipped 35 lines
44	44		it('should initiate a webpage scan', async () => {
45	45		const siteUrl = new URL('https://example.com/' + Math.random().toString());
46	46
47		-	const requestWebPageScanMock = jest.spyOn(internalApi, 'requestWebPageScan');
	47	+	const requestWebPageScanMock = jest.spyOn(systemApi, 'requestWebPageScan');
48	48		requestWebPageScanMock.mockImplementation(async () => ({}));
49	49
50	50		const response = await api
		skipped 46 lines
97	97		it('should return a cached scan if applicable', async () => {
98	98		const siteUrl = new URL(`https://${Math.random().toString()}.example.com/`);
99	99
100		-	const requestWebPageScanMock = jest.spyOn(internalApi, 'requestWebPageScan');
	100	+	const requestWebPageScanMock = jest.spyOn(systemApi, 'requestWebPageScan');
101	101		requestWebPageScanMock.mockImplementation(async () => ({}));
102	102
103	103		const db = await getDatabaseConnection();
		skipped 4 lines
108	108		webPage,
109	109		status: WebPageScan.Status.Processed,
110	110		scanResult: {
111		-	packages: [
	111	+	identifiedModuleMap: {},
	112	+	identifiedPackages: [
112	113		{
113	114		name: 'react',
114	115		versionSet: ['17.0.0'],
115		-	versionRange: '17.0.0',
116		-	approximateByteSize: null,
	116	+	moduleIds: [],
117	117		},
118	118		],
119	119		},
		skipped 12 lines
132	132		id: existingScan.id.toString(),
133	133		status: WebPageScan.Status.Processed,
134	134		scanResult: {
135		-	packages: [
	135	+	identifiedModuleMap: {},
	136	+	identifiedPackages: [
136	137		{
137	138		name: 'react',
138	139		versionSet: ['17.0.0'],
139		-	versionRange: '17.0.0',
140		-	approximateByteSize: null,
	140	+	moduleIds: [],
141	141		},
142	142		],
143	143		},
		skipped 6 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/clientApiRouter.ts

		skipped 19 lines
20	20		export const createContext = (_: CreateExpressContextOptions) => ({}); // no context
21	21		type Context = trpc.inferAsyncReturnType<typeof createContext>;
22	22
23		-	type ScanResultPackageWithMetadata = WebPageScan.Package & { registryMetadata?: PackageMetadata };
	23	+	type ScanResultPackageWithMetadata = WebPageScan.IdentifiedPackage & {
	24	+	registryMetadata?: PackageMetadata;
	25	+	};
24	26
25	27		export namespace ClientApi {
26	28		export type PackageVulnerabilityResponse = SerializableEntity<PackageVulnerabilityData>;
		skipped 1 lines
28	30		}
29	31
30	32		function mergeRegistryMetadata(
31		-	packages: WebPageScan.Package[],
	33	+	packages: WebPageScan.IdentifiedPackage[],
32	34		registryMetadata: Record<string, PackageMetadata>
33	35		) {
34	36		return packages.map((it) => ({
		skipped 5 lines
40	42		type RequestWebPageScanResponse = Pick<WebPageScan, 'status' \| 'finishedAt'> & {
41	43		id: string;
42	44		scanResult?: {
43		-	packages: ScanResultPackageWithMetadata[];
	45	+	identifiedModuleMap: Record<string, WebPageScan.IdentifiedModule>;
	46	+	identifiedPackages: ScanResultPackageWithMetadata[];
44	47		vulnerabilities: Record<string, PackageVulnerabilityData[]>;
45	48		};
46	49		};
		skipped 13 lines
60	63		};
61	64
62	65		if (scan.scanResult) {
63		-	const packageNames = scan.scanResult.packages.map((it) => it.name);
	66	+	const packageNames = scan.scanResult.identifiedPackages.map((it) => it.name);
64	67
65	68		const [metadata, vulnerabilities] = await Promise.all([
66	69		getPackageMetadataByPackageNames(packageNames),
		skipped 1 lines
68	71		]);
69	72
70	73		scanResponse.scanResult = {
71		-	packages: mergeRegistryMetadata(scan.scanResult.packages, metadata),
	74	+	identifiedModuleMap: scan.scanResult.identifiedModuleMap,
	75	+	identifiedPackages: mergeRegistryMetadata(scan.scanResult.identifiedPackages, metadata),
72	76		vulnerabilities,
73	77		};
74	78		}
		skipped 21 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncPackageUsageByHostname.test.ts

		skipped 22 lines
23	23		webPage,
24	24		status: WebPageScan.Status.Processed,
25	25		scanResult: {
26		-	packages: [
	26	+	identifiedPackages: [
27	27		{
28	28		name: 'react',
29	29		versionSet: ['17.0.0'],
30		-	versionRange: '17.0.0',
31		-	approximateByteSize: null,
	30	+	moduleIds: [],
32	31		},
33	32		],
34	33		},
		skipped 29 lines
64	63		webPage,
65	64		status: WebPageScan.Status.Processed,
66	65		scanResult: {
67		-	packages: [
	66	+	identifiedPackages: [
68	67		{
69	68		name: 'react',
70	69		versionSet: ['17.0.0'],
71		-	versionRange: '17.0.0',
72		-	approximateByteSize: null,
	70	+	moduleIds: [],
73	71		},
74	72		{
75	73		name: 'react-dom',
76	74		versionSet: ['17.0.0'],
77	75		versionRange: '17.0.0',
78		-	approximateByteSize: null,
79	76		},
80	77		],
81	78		},
		skipped 26 lines
108	105		webPage,
109	106		status: WebPageScan.Status.Processed,
110	107		scanResult: {
111		-	packages: [
	108	+	identifiedPackages: [
112	109		{
113	110		name: 'react',
114	111		versionSet: ['17.0.0'],
115		-	versionRange: '17.0.0',
116		-	approximateByteSize: null,
	112	+	moduleIds: [],
117	113		},
118	114		],
119	115		},
		skipped 21 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncPackageUsageByHostname.ts

		skipped 32 lines
33	33		.leftJoinAndSelect('webpage.hostname', 'hostname')
34	34		.getOneOrFail();
35	35
36		-	const packageUsageEntities = newScan.scanResult.packages.map((sourcePackage) =>
	36	+	const packageUsageEntities = newScan.scanResult.identifiedPackages.map((sourcePackage) =>
37	37		packageUsageByHostRepo.create({
38	38		hostname: relatedWebPage.hostname,
39	39		sourceScan: newScan,
		skipped 10 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncScansWithVulnerabilities.test.ts

		skipped 23 lines
24	24		webPage,
25	25		status: WebPageScan.Status.Processed,
26	26		scanResult: {
27		-	packages: [
	27	+	identifiedPackages: [
28	28		{
29	29		name: 'react',
30	30		versionSet: ['17.0.0'],
31		-	versionRange: '17.0.0',
32		-	approximateByteSize: null,
	31	+	moduleIds: [],
33	32		},
34	33		],
35	34		},
		skipped 41 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/systemApiRouter.test.ts

		skipped 3 lines
4	4		useTransactionalTesting,
5	5		} from '@gradejs-public/test-utils';
6	6		import { createApp } from './app';
7		-	import { getGradeJsApiKey, internalApi, WebPageScan } from '@gradejs-public/shared';
	7	+	import { getGradeJsApiKey, systemApi, WebPageScan } from '@gradejs-public/shared';
8	8		import * as WebsiteService from './website/service';
9		-	import { SystemApi } from './systemApiRouter';
10	9		import { getRepository } from 'typeorm';
11	10
12	11		useDatabaseConnection();
		skipped 20 lines
33	32		});
34	33
35	34		it('should process reported scans', async () => {
36		-	const payload: SystemApi.ScanReport = {
37		-	id: 'test',
	35	+	const payload: systemApi.ScanReport = {
	36	+	requestId: 'test',
38	37		url: 'http://test.com',
39		-	status: internalApi.WebPageScan.Status.Ready,
40		-	scan: {
41		-	packages: [
42		-	{
43		-	name: 'react',
44		-	versionSet: ['17.0.0'],
45		-	versionRange: '17.0.0',
46		-	approximateByteSize: null,
47		-	},
48		-	],
49		-	},
	38	+	status: systemApi.ScanReport.Status.Ready,
	39	+	identifiedModuleMap: {},
	40	+	identifiedPackages: [
	41	+	{
	42	+	name: 'react',
	43	+	versionSet: ['17.0.0'],
	44	+	moduleIds: [],
	45	+	},
	46	+	],
50	47		};
51	48
52	49		const sencWebPageScanResultMock = jest.spyOn(WebsiteService, 'syncWebPageScanResult');
		skipped 12 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/systemApiRouter.ts

	1	+	import { systemApi } from '@gradejs-public/shared';
1	2		import { Router } from 'express';
2		-	import { internalApi } from '@gradejs-public/shared';
3		-	import { z } from 'zod';
4	3		import { syncWebPageScanResult } from './website/service';
5	4
6		-	const apiReportedPackageSchema = z.object({
7		-	name: z.string(),
8		-	versionSet: z.array(z.string()),
9		-	versionRange: z.string(),
10		-	approximateByteSize: z.nullable(z.number()),
11		-	});
12		-
13		-	const apiScanReportSchema = z.object({
14		-	id: z.optional(z.string()),
15		-	url: z.string().url(),
16		-	status: z.nativeEnum(internalApi.WebPageScan.Status),
17		-	scan: z.object({
18		-	packages: z.array(apiReportedPackageSchema),
19		-	}),
20		-	});
21		-
22		-	export namespace SystemApi {
23		-	export type ScanReport = z.infer<typeof apiScanReportSchema>;
24		-	export type ReportedPackage = z.infer<typeof apiReportedPackageSchema>;
25		-	}
26		-
27	5		const systemApiRouter = Router();
28	6
29	7		systemApiRouter.post('/scan', async (req, res, next) => {
30	8		try {
31		-	const scanReport = apiScanReportSchema.parse(req.body);
	9	+	const scanReport = systemApi.apiScanReportSchema.parse(req.body);
32	10		await syncWebPageScanResult(scanReport);
33	11		} catch (e) {
34	12		next(e);
		skipped 8 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/vulnerabilities/vulnerabilities.ts

		skipped 21 lines
22	22		export async function getAffectingVulnerabilities(scanResult: WebPageScan.Result) {
23	23		const affectingVulnerabilitiesByPackage: Record<string, PackageVulnerabilityData[]> = {};
24	24
25		-	const packages = scanResult.packages;
	25	+	const packages = scanResult.identifiedPackages;
26	26		if (!packages.length) {
27	27		return affectingVulnerabilitiesByPackage;
28	28		}
		skipped 9 lines
38	38
39	39		for (const vulnerability of vulnerabilitiesByPackage) {
40	40		const relatedPackage = packagesByNames[vulnerability.packageName]!;
	41	+	const relatedPackageVersionRange = relatedPackage.versionSet.join(' \|\| ');
41	42		const affectsReportedRange = semver.subset(
42		-	relatedPackage.versionRange,
	43	+	relatedPackageVersionRange,
43	44		vulnerability.packageVersionRange,
44	45		{ loose: true }
45	46		);
		skipped 25 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/website/service.test.ts

1	1		import { useDatabaseConnection, useTransactionalTesting } from '@gradejs-public/test-utils';
2	2		import { findOrCreateWebPage, syncWebPageScanResult } from './service';
3		-	import { getDatabaseConnection, internalApi, WebPageScan } from '@gradejs-public/shared';
	3	+	import { getDatabaseConnection, systemApi, WebPageScan } from '@gradejs-public/shared';
4	4
5	5		useDatabaseConnection();
6	6		useTransactionalTesting();
		skipped 13 lines
20	20		});
21	21
22	22		await syncWebPageScanResult({
23		-	id: scan.id.toString(),
24		-	status: internalApi.WebPageScan.Status.Ready,
	23	+	requestId: scan.id.toString(),
	24	+	status: systemApi.ScanReport.Status.Ready,
25	25		url: url.toString(),
26		-	scan: {
27		-	packages: [
28		-	{
29		-	name: 'react',
30		-	versionSet: ['17.0.0'],
31		-	versionRange: '17.0.0',
32		-	approximateByteSize: null,
33		-	},
34		-	],
	26	+	identifiedModuleMap: {
	27	+	moduleId: {
	28	+	packageName: 'react',
	29	+	packageVersionSet: ['17.0.0'],
	30	+	packageFile: 'index.js',
	31	+	approximateByteSize: 100,
	32	+	},
35	33		},
	34	+	identifiedPackages: [
	35	+	{
	36	+	name: 'react',
	37	+	versionSet: ['17.0.0'],
	38	+	moduleIds: ['moduleId'],
	39	+	},
	40	+	],
36	41		});
37	42
38	43		const updatedScan = await em.getRepository(WebPageScan).findOneOrFail({ id: scan.id });
39	44		expect(updatedScan).toMatchObject({
40	45		status: WebPageScan.Status.Processed,
41	46		scanResult: {
42		-	packages: [
	47	+	identifiedModuleMap: {
	48	+	moduleId: {
	49	+	packageName: 'react',
	50	+	packageVersionSet: ['17.0.0'],
	51	+	packageFile: 'index.js',
	52	+	approximateByteSize: 100,
	53	+	},
	54	+	},
	55	+	identifiedPackages: [
43	56		{
44	57		name: 'react',
45	58		versionSet: ['17.0.0'],
46		-	versionRange: '17.0.0',
47		-	approximateByteSize: null,
	59	+	moduleIds: ['moduleId'],
48	60		},
49	61		],
50	62		},
		skipped 5 lines
56	68		const url = new URL('https://example.com/test2');
57	69
58	70		const scan = await syncWebPageScanResult({
59		-	status: internalApi.WebPageScan.Status.Ready,
	71	+	status: systemApi.ScanReport.Status.Ready,
60	72		url: url.toString(),
61		-	scan: {
62		-	packages: [
63		-	{
64		-	name: 'react',
65		-	versionSet: ['17.0.0'],
66		-	versionRange: '17.0.0',
67		-	approximateByteSize: null,
68		-	},
69		-	],
70		-	},
	73	+	identifiedModuleMap: {},
	74	+	identifiedPackages: [
	75	+	{
	76	+	name: 'react',
	77	+	versionSet: ['17.0.0'],
	78	+	moduleIds: [],
	79	+	},
	80	+	],
71	81		});
72	82
73	83		expect(scan).toMatchObject({
74	84		status: WebPageScan.Status.Processed,
75	85		scanResult: {
76		-	packages: [
	86	+	identifiedModuleMap: {},
	87	+	identifiedPackages: [
77	88		{
78	89		name: 'react',
79	90		versionSet: ['17.0.0'],
80		-	versionRange: '17.0.0',
81		-	approximateByteSize: null,
	91	+	moduleIds: [],
82	92		},
83	93		],
84	94		},
		skipped 4 lines
89	99		const url = new URL('https://example.com/test3');
90	100
91	101		const syncPromise = syncWebPageScanResult({
92		-	id: '9999999',
93		-	status: internalApi.WebPageScan.Status.Ready,
	102	+	requestId: '9999999',
	103	+	status: systemApi.ScanReport.Status.Ready,
94	104		url: url.toString(),
95		-	scan: {
96		-	packages: [
97		-	{
98		-	name: 'react',
99		-	versionSet: ['17.0.0'],
100		-	versionRange: '17.0.0',
101		-	approximateByteSize: null,
102		-	},
103		-	],
104		-	},
	105	+	identifiedModuleMap: {},
	106	+	identifiedPackages: [
	107	+	{
	108	+	name: 'react',
	109	+	versionSet: ['17.0.0'],
	110	+	moduleIds: [],
	111	+	},
	112	+	],
105	113		});
106	114
107	115		await expect(syncPromise).rejects.toThrow();
		skipped 3 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/website/service.ts

1	1		import {
2	2		WebPageScan,
3	3		getDatabaseConnection,
4		-	internalApi,
	4	+	systemApi,
5	5		Hostname,
6	6		WebPage,
7	7		} from '@gradejs-public/shared';
8	8		import { EntityManager } from 'typeorm';
9	9		import { syncPackageUsageByHostname } from '../projections/syncPackageUsageByHostname';
10	10		import { syncScansWithVulnerabilities } from '../projections/syncScansWithVulnerabilities';
11		-	import { SystemApi } from '../systemApiRouter';
12	11
13	12		const RESCAN_TIMEOUT_MS = 1000 * 60 * 60 * 24; // 1 day in ms
14	13
		skipped 52 lines
67	66		status: WebPageScan.Status.Pending,
68	67		});
69	68
70		-	await internalApi.requestWebPageScan(parsedUrl.toString(), webPageScanEntity.id.toString());
	69	+	await systemApi.requestWebPageScan(parsedUrl.toString(), webPageScanEntity.id.toString());
71	70
72	71		return webPageScanEntity;
73	72		});
		skipped 1 lines
75	74		return result;
76	75		}
77	76
78		-	export async function syncWebPageScanResult(scanReport: SystemApi.ScanReport) {
	77	+	export async function syncWebPageScanResult(scanReport: systemApi.ScanReport) {
79	78		const db = await getDatabaseConnection();
80	79
81	80		return await db.transaction(async (em) => {
82	81		const webPageScanRepo = em.getRepository(WebPageScan);
83	82
84	83		let scanEntity: WebPageScan;
85		-	if (scanReport.id) {
86		-	scanEntity = await webPageScanRepo.findOneOrFail({ id: parseInt(scanReport.id, 10) });
	84	+	if (scanReport.requestId) {
	85	+	scanEntity = await webPageScanRepo.findOneOrFail({ id: parseInt(scanReport.requestId, 10) });
87	86		} else {
88	87		const webPageEntity = await findOrCreateWebPage(new URL(scanReport.url), em);
89	88		scanEntity = webPageScanRepo.create({
		skipped 1 lines
91	90		});
92	91		}
93	92
94		-	scanEntity.status = mapInternalWebsiteStatus(scanReport.status);
	93	+	scanEntity.status = mapScanReportStatus(scanReport.status);
95	94		scanEntity.finishedAt = new Date();
96		-	scanEntity.scanResult = scanReport.scan;
	95	+
	96	+	if (scanReport.status === 'ready') {
	97	+	scanEntity.scanResult = {
	98	+	identifiedModuleMap: scanReport.identifiedModuleMap,
	99	+	identifiedPackages: scanReport.identifiedPackages,
	100	+	};
	101	+	}
97	102
98	103		await webPageScanRepo.save(scanEntity);
99	104
		skipped 6 lines
106	111		});
107	112		}
108	113
109		-	function mapInternalWebsiteStatus(status: internalApi.WebPageScan.Status) {
	114	+	// TODO: add protected website use-case
	115	+	function mapScanReportStatus(status: string) {
110	116		switch (status) {
111		-	case internalApi.WebPageScan.Status.Invalid:
112		-	return WebPageScan.Status.Unsupported;
113		-	case internalApi.WebPageScan.Status.InProgress:
114		-	return WebPageScan.Status.Pending;
115		-	case internalApi.WebPageScan.Status.Protected:
116		-	return WebPageScan.Status.Protected;
	117	+	case systemApi.ScanReport.Status.Ready:
	118	+	return WebPageScan.Status.Processed;
117	119		default:
118		-	return WebPageScan.Status.Processed;
	120	+	return WebPageScan.Status.Failed;
119	121		}
120	122		}
121	123

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/webPageScan.ts

	1	+	import { systemApi } from '@gradejs-public/shared';
1	2		import {
2	3		Column,
3	4		Entity,
		skipped 6 lines
10	11		PrimaryGeneratedColumn,
11	12		} from 'typeorm';
12	13		import { WebPage } from './webPage';
13		-	import { DetectedPackage } from '../../systemApi/api';
14	14
15	15		@Entity({ name: 'web_page_scan' })
16	16		@Index(['webPage', 'createdAt'])
		skipped 30 lines
47	47		Failed = 'failed',
48	48		}
49	49
50		-	export type Package = DetectedPackage;
	50	+	export type IdentifiedModule = systemApi.ScanReport.IdentifiedModule;
	51	+	export type IdentifiedPackage = systemApi.ScanReport.IdentifiedPackage;
	52	+
51	53		export type Result = {
52		-	packages: Package[];
	54	+	identifiedModuleMap: Record<string, IdentifiedModule>;
	55	+	identifiedPackages: IdentifiedPackage[];
53	56		};
54	57		}
55	58

■ ■ ■ ■ ■ ■

packages/shared/src/index.ts

		skipped 10 lines
11	11		export * from './utils/env';
12	12		export * from './utils/types';
13	13
14		-	export * as internalApi from './systemApi/api';
	14	+	// lower case for methods and upper case for typings and zod schemas
	15	+	export * as systemApi from './systemApi/api';
15	16
16	17		export * from './worker/types';
17	18

■ ■ ■ ■ ■ ■

packages/shared/src/systemApi/api.ts

1	1		import fetch, { RequestInit } from 'node-fetch';
2	2		import { getGradeJsApiKey, getInternalApiRootUrl } from '../utils/env';
	3	+	import { z } from 'zod';
	4	+
	5	+	export const identifiedModuleSchema = z.object({
	6	+	packageName: z.string(),
	7	+	packageVersionSet: z.array(z.string()),
	8	+	packageFile: z.string(),
	9	+	approximateByteSize: z.number(),
	10	+	});
	11	+
	12	+	export const identifiedPackageSchema = z.object({
	13	+	name: z.string(),
	14	+	versionSet: z.array(z.string()),
	15	+	moduleIds: z.array(z.string()),
	16	+	});
	17	+
	18	+	// TODO: add processed scripts
	19	+	// TODO: add identified bundler
3	20
4		-	export type DetectedPackage = {
5		-	name: string;
6		-	versionSet: string[];
7		-	versionRange: string;
8		-	approximateByteSize: number \| null;
9		-	};
	21	+	export const apiScanReportSchema = z.union([
	22	+	z.object({
	23	+	requestId: z.optional(z.string()),
	24	+	url: z.string().url(),
	25	+	status: z.literal('ready'),
	26	+	identifiedModuleMap: z.record(identifiedModuleSchema),
	27	+	identifiedPackages: z.array(identifiedPackageSchema),
	28	+	}),
	29	+	z.object({
	30	+	requestId: z.optional(z.string()),
	31	+	url: z.string().url(),
	32	+	status: z.literal('error'),
	33	+	}),
	34	+	]);
10	35
11		-	export namespace WebPageScan {
	36	+	export namespace ScanReport {
	37	+	export type IdentifiedPackage = z.infer<typeof identifiedPackageSchema>;
	38	+	export type IdentifiedModule = z.infer<typeof identifiedModuleSchema>;
12	39		export enum Status {
13		-	Created = 'created',
14		-	InProgress = 'in-progress',
15	40		Ready = 'ready',
16		-	Failed = 'failed',
17		-	Invalid = 'invalid',
18		-	Protected = 'protected',
	41	+	Error = 'error',
19	42		}
20	43		}
21	44
22		-	export interface Package {
	45	+	export type ScanReport = z.infer<typeof apiScanReportSchema>;
	46	+
	47	+	export interface PackageRequest {
23	48		name: string;
24	49		latestVersion: string;
25	50		}
		skipped 4 lines
30	55		[key: string]: unknown;
31	56		}
32	57
33		-	type Paginaton = {
	58	+	export type PaginatonRequest = {
34	59		offset: number;
35	60		limit: number;
36	61		total: number;
		skipped 4 lines
41	66		}
42	67
43	68		export async function fetchPackageIndex(offset = 0, limit = 0) {
44		-	return fetchEndpoint<{ pagination: Paginaton; packages: Package[] }>('GET', '/package/index', {
45		-	offset,
46		-	limit,
47		-	});
	69	+	return fetchEndpoint<{ pagination: PaginatonRequest; packages: PackageRequest[] }>(
	70	+	'GET',
	71	+	'/package/index',
	72	+	{
	73	+	offset,
	74	+	limit,
	75	+	}
	76	+	);
48	77		}
49	78
50	79		export async function requestPackageIndexing(payload: PackageIndexRequest) {
		skipped 40 lines

■ ■ ■ ■ ■ ■

packages/shared/src/worker/types.ts

1		-	import * as internalApi from '../systemApi/api';
	1	+	import { PackageRequest } from '../systemApi/api';
2	2
3	3		export type WorkerTask = {
4	4		[Key in WorkerTaskType]: {
		skipped 6 lines
11	11
12	12		export type WorkerTaskPayloadMap = {
13	13		syncPackageIndex: undefined;
14		-	syncPackageIndexBatch: internalApi.Package[];
	14	+	syncPackageIndexBatch: PackageRequest[];
15	15		syncPackageVulnerabilities: undefined;
16	16		};
17	17

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndex.test.ts

1		-	import { internalApi } from '@gradejs-public/shared';
	1	+	import { systemApi } from '@gradejs-public/shared';
2	2		import { expectQueuedTasks } from '@gradejs-public/test-utils';
3	3		import { syncPackageIndex } from './syncPackageIndex';
4	4
		skipped 7 lines
12	12		}));
13	13
14	14		const fetchPackageIndexMock = jest
15		-	.spyOn(internalApi, 'fetchPackageIndex')
	15	+	.spyOn(systemApi, 'fetchPackageIndex')
16	16		.mockImplementation((offset = 0, limit = 0) =>
17	17		Promise.resolve({
18	18		pagination: { offset, limit, total: mockPackages.length },
		skipped 15 lines

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndex.ts

1	1		import concurrently from 'tiny-async-pool';
2		-	import { internalApi, queueWorkerTask } from '@gradejs-public/shared';
	2	+	import { systemApi, queueWorkerTask } from '@gradejs-public/shared';
3	3
4	4		const PKG_SYNC_BATCH_LIMIT = 50;
5	5		const PKG_SYNC_PARALLEL_LIMIT = 10;
		skipped 3 lines
9	9		* Look into `syncPackageIndexBatch` for the implementation.
10	10		*/
11	11		export async function syncPackageIndex() {
12		-	const { pagination } = await internalApi.fetchPackageIndex();
	12	+	const { pagination } = await systemApi.fetchPackageIndex();
13	13
14	14		let processed = 0;
15	15		const batchesTotal = Math.ceil(pagination.total / PKG_SYNC_BATCH_LIMIT);
16	16		const offsets = new Array(batchesTotal).fill(0).map((_, i) => i * PKG_SYNC_BATCH_LIMIT);
17	17
18	18		async function processBatch(offset: number) {
19		-	const { packages } = await internalApi.fetchPackageIndex(offset, PKG_SYNC_BATCH_LIMIT);
	19	+	const { packages } = await systemApi.fetchPackageIndex(offset, PKG_SYNC_BATCH_LIMIT);
20	20		await queueWorkerTask('syncPackageIndexBatch', packages);
21	21		processed++;
22	22		}
		skipped 9 lines

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndexBatch.test.ts

1	1		import { getRepository } from 'typeorm';
2	2		import { syncPackageIndexBatch } from './syncPackageIndexBatch';
3	3		import * as npmRegistry from '../npmRegistry/api';
4		-	import { internalApi, PackageMetadata } from '@gradejs-public/shared';
	4	+	import { systemApi, PackageMetadata } from '@gradejs-public/shared';
5	5		import { useDatabaseConnection, useTransactionalTesting } from '@gradejs-public/test-utils';
6	6
7	7		useDatabaseConnection();
		skipped 21 lines
29	29		);
30	30
31	31		const requestPackageIndexingMock = jest
32		-	.spyOn(internalApi, 'requestPackageIndexing')
	32	+	.spyOn(systemApi, 'requestPackageIndexing')
33	33		.mockImplementation(() => Promise.resolve(true));
34	34
35	35		await syncPackageIndexBatch([mockedPackage]);
		skipped 34 lines
70	70		} as any)
71	71		);
72	72
73		-	const requestPackageIndexingMock = jest.spyOn(internalApi, 'requestPackageIndexing');
	73	+	const requestPackageIndexingMock = jest.spyOn(systemApi, 'requestPackageIndexing');
74	74
75	75		await syncPackageIndexBatch([
76	76		{ name: savedPackage.name, latestVersion: savedPackage.latestVersion },
		skipped 13 lines

■ ■ ■ ■ ■ ■

packages/worker/src/tasks/syncPackageIndexBatch.ts

1		-	import { internalApi, PackageMetadata } from '@gradejs-public/shared';
	1	+	import { systemApi, PackageMetadata } from '@gradejs-public/shared';
2	2		import { fetchPackageStatsAndMetadata } from '../npmRegistry/api';
3	3
4		-	export async function syncPackageIndexBatch(payload: internalApi.Package[]) {
	4	+	export async function syncPackageIndexBatch(payload: systemApi.PackageRequest[]) {
5	5		// We do not need any concurrent limits here since the batch size
6	6		// is controlled by the `syncPackageIndex` task.
7	7		await Promise.all(payload.map((item) => syncPackage(item.name)));
		skipped 14 lines
22	22		const statsAndMetadata = await fetchPackageStatsAndMetadata(name);
23	23
24	24		await Promise.all([
25		-	internalApi.requestPackageIndexing({
	25	+	systemApi.requestPackageIndexing({
26	26		name,
27	27		versions: Object.keys(statsAndMetadata.versionSpecificValues),
28	28		}),
		skipped 21 lines

feat: extendable scan report format (#62)