STRLCPY/gradejs

■ ■ ■ ■ ■ ■

.gitignore

		skipped 21 lines
22	22		.idea
23	23		.vscode
24	24
	25	+	./docker-compose.override.yml

■ ■ ■ ■ ■ ■

cli/local_start.sh | 100644 /~icons-ver-BEF942F0F42935333EFA072090F4E956.svg#arrow3 100755

		skipped 14 lines
15	15
16	16		# Save current env and values passed in CLI
17	17		CURENV=$(declare -p -x)
18		-	# Set local development envvars
	18	+	# Set local development env vars
19	19		set -o allexport
20	20		source cli/development.env
21	21		set +o allexport
		skipped 13 lines
35	35
36	36		echo "Starting worker package"
37	37		AWS_REGION=test PORT=8084 DB_URL=postgres://gradejs:gradejs@localhost:5432/gradejs-public \
38		-	INTERNAL_API_ORIGIN=http://localhost:8082 SQS_WORKER_QUEUE_URL=/test/frontend-queue \
	38	+	SQS_WORKER_QUEUE_URL=/test/frontend-queue \
39	39		SQS_LOCAL_PORT=29324 AWS_ACCESS_KEY_ID=secret AWS_SECRET_ACCESS_KEY=secret \
	40	+	INTERNAL_API_ROOT_URL=http://localhost:8082 \
	41	+	GRADEJS_API_KEY=TEST_API_KEY \
40	42		npm run debug --prefix packages/worker 2>&1 &
41	43		WORKER_PID=$!
42	44
		skipped 4 lines
47	49
48	50		echo "Starting public api package"
49	51		AWS_REGION=test PORT=8083 DB_URL=postgres://gradejs:gradejs@localhost:5432/gradejs-public \
50		-	INTERNAL_API_ORIGIN=http://localhost:8082 SQS_WORKER_QUEUE_URL=/test/frontend-queue \
	52	+	SQS_WORKER_QUEUE_URL=/test/frontend-queue \
51	53		SQS_LOCAL_PORT=29324 AWS_ACCESS_KEY_ID=secret AWS_SECRET_ACCESS_KEY=secret \
52	54		CORS_ALLOWED_ORIGIN=http://localhost:3000 \
	55	+	INTERNAL_API_ROOT_URL=http://localhost:8082 \
	56	+	GRADEJS_API_KEY=TEST_API_KEY \
53	57		npm run debug --prefix packages/public-api 2>&1 &
54	58		API_PID=$!
55	59
		skipped 60 lines

■ ■ ■ ■ ■ ■

packages/public-api/package.json

		skipped 5 lines
6	6		"prettier": "prettier --check ./src",
7	7		"typecheck": "tsc --project tsconfig.build.json --noEmit",
8	8		"build": "tsc --project tsconfig.build.json",
9		-	"test": "CORS_ALLOWED_ORIGIN=http://localhost:3000 jest",
	9	+	"test": "jest",
10	10		"start": "node build/index.js",
11	11		"debug": "node --inspect=9201 build/index.js"
12	12		},
		skipped 29 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/app.ts

1	1		import express from 'express';
2	2		import { createExpressMiddleware } from '@trpc/server/adapters/express';
3	3		import { endpointMissingMiddleware, errorHandlerMiddleware, cors } from './middleware/common';
4		-	import { appRouter, createContext } from './router';
	4	+	import { appRouter, createContext } from './clientApiRouter';
	5	+	import { verifySystemApiToken } from './middleware/verifySystemApiToken';
	6	+	import systemApiRouter from './systemApiRouter';
5	7
6	8		export function createApp() {
7	9		const app = express();
8	10		app.get('/', (_, res) => res.send('gradejs-public-api')); // healthcheck path
9		-	app.use(cors);
	11	+
	12	+	app.use('/system', verifySystemApiToken, express.json(), systemApiRouter);
10	13
11	14		app.use(
12		-	'/',
	15	+	'/client',
	16	+	cors,
13	17		createExpressMiddleware({
14	18		router: appRouter,
15	19		createContext,
		skipped 10 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/clientApiRouter.test.ts

1	+	import { TRPC_ERROR_CODES_BY_KEY } from '@trpc/server/rpc';
2	+	import {
3	+	getDatabaseConnection,
4	+	Hostname,
5	+	internalApi,
6	+	WebPage,
7	+	WebPageScan,
8	+	} from '@gradejs-public/shared';
9	+	import {
10	+	createSupertestApi,
11	+	useDatabaseConnection,
12	+	useTransactionalTesting,
13	+	} from '@gradejs-public/test-utils';
14	+	import { getRepository } from 'typeorm';
15	+	import { createApp } from './app';
16	+	import { findOrCreateWebPage } from './website/service';
17	+
18	+	useDatabaseConnection();
19	+	useTransactionalTesting();
20	+
21	+	const api = createSupertestApi(createApp);
22	+
23	+	describe('routes / heathCheck', () => {
24	+	it('should return valid response for healthcheck path', async () => {
25	+	await api.get('/').send().expect(200);
26	+	});
27	+
28	+	it('should return not found client error', async () => {
29	+	const response = await api
30	+	.get('/client/any-invalid-route')
31	+	.set('Origin', 'http://localhost:3000')
32	+	.send()
33	+	.expect(404);
34	+
35	+	expect(response.body).toMatchObject({
36	+	error: {
37	+	code: TRPC_ERROR_CODES_BY_KEY.NOT_FOUND,
38	+	},
39	+	});
40	+	});
41	+	});
42	+
43	+	describe('routes / website', () => {
44	+	it('should initiate a webpage scan', async () => {
45	+	const siteUrl = new URL('https://example.com/' + Math.random().toString());
46	+
47	+	const requestWebPageScanMock = jest.spyOn(internalApi, 'requestWebPageScan');
48	+	requestWebPageScanMock.mockImplementation(async () => ({}));
49	+
50	+	const response = await api
51	+	.post('/client/requestWebPageScan')
52	+	.set('Origin', 'http://localhost:3000')
53	+	.send(JSON.stringify(siteUrl))
54	+	.expect(200);
55	+
56	+	const hostname = await getRepository(Hostname).findOneOrFail({ hostname: siteUrl.hostname });
57	+
58	+	expect(hostname).toMatchObject({
59	+	hostname: siteUrl.hostname,
60	+	});
61	+
62	+	const webPage = await getRepository(WebPage)
63	+	.createQueryBuilder('webpage')
64	+	.where('webpage.hostname_id = :hostnameId', { hostnameId: hostname.id })
65	+	.andWhere('webpage.path = :path', { path: siteUrl.pathname })
66	+	.limit(1)
67	+	.getOneOrFail();
68	+
69	+	expect(webPage).toMatchObject({
70	+	path: siteUrl.pathname,
71	+	});
72	+
73	+	const webPageScan = await getRepository(WebPageScan)
74	+	.createQueryBuilder('scan')
75	+	.where('scan.web_page_id = :webPageId', { webPageId: webPage.id })
76	+	.getOneOrFail();
77	+
78	+	expect(webPageScan).toMatchObject({
79	+	status: WebPageScan.Status.Pending,
80	+	});
81	+
82	+	expect(requestWebPageScanMock).toHaveBeenCalledTimes(1);
83	+	expect(requestWebPageScanMock).toHaveBeenCalledWith(
84	+	siteUrl.toString(),
85	+	webPageScan.id.toString()
86	+	);
87	+	expect(response.body).toMatchObject({
88	+	result: {
89	+	data: {
90	+	id: webPageScan.id.toString(),
91	+	status: WebPageScan.Status.Pending,
92	+	},
93	+	},
94	+	});
95	+	});
96	+
97	+	it('should return a cached scan if applicable', async () => {
98	+	const siteUrl = new URL(`https://${Math.random().toString()}.example.com/`);
99	+
100	+	const requestWebPageScanMock = jest.spyOn(internalApi, 'requestWebPageScan');
101	+	requestWebPageScanMock.mockImplementation(async () => ({}));
102	+
103	+	const db = await getDatabaseConnection();
104	+	const em = db.createEntityManager();
105	+
106	+	const webPage = await findOrCreateWebPage(siteUrl, em);
107	+	const existingScan = await em.getRepository(WebPageScan).save({
108	+	webPage,
109	+	status: WebPageScan.Status.Processed,
110	+	scanResult: {
111	+	packages: [
112	+	{
113	+	name: 'react',
114	+	versionSet: ['17.0.0'],
115	+	versionRange: '17.0.0',
116	+	approximateByteSize: null,
117	+	},
118	+	],
119	+	},
120	+	});
121	+
122	+	const response = await api
123	+	.post('/client/requestWebPageScan')
124	+	.set('Origin', 'http://localhost:3000')
125	+	.send(JSON.stringify(siteUrl))
126	+	.expect(200);
127	+
128	+	expect(requestWebPageScanMock).toHaveBeenCalledTimes(0);
129	+	expect(response.body).toMatchObject({
130	+	result: {
131	+	data: {
132	+	id: existingScan.id.toString(),
133	+	status: WebPageScan.Status.Processed,
134	+	scanResult: {
135	+	packages: [
136	+	{
137	+	name: 'react',
138	+	versionSet: ['17.0.0'],
139	+	versionRange: '17.0.0',
140	+	approximateByteSize: null,
141	+	},
142	+	],
143	+	},
144	+	},
145	+	},
146	+	});
147	+	});
148	+	});
149	+

■ ■ ■ ■ ■ ■

packages/public-api/src/clientApiRouter.ts

1	+	import * as trpc from '@trpc/server';
2	+	// See also: https://colinhacks.com/essays/painless-typesafety
3	+	import { CreateExpressContextOptions } from '@trpc/server/adapters/express';
4	+	import { z, ZodError } from 'zod';
5	+	import { getOrRequestWebPageScan } from './website/service';
6	+	import { getAffectingVulnerabilities } from './vulnerabilities/vulnerabilities';
7	+	import {
8	+	PackageMetadata,
9	+	PackageVulnerabilityData,
10	+	SerializableEntity,
11	+	toSerializable,
12	+	WebPageScan,
13	+	} from '@gradejs-public/shared';
14	+	import { getPackageMetadataByPackageNames } from './packageMetadata/packageMetadataService';
15	+
16	+	// const hostnameRe =
17	+	// /^(?:(?:25[0-5]\|2[0-4][0-9]\|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]\|2[0-4][0-9]\|[01]?[0-9][0-9]?)$\|^(([a-zA-Z0-9]\|[a-zA-Z0-9][a-zA-Z0-9\-][a-zA-Z0-9])\.)+([A-Za-z]\|[A-Za-z][A-Za-z0-9\-][A-Za-z0-9])$/;
18	+
19	+	// created for each request
20	+	export const createContext = (_: CreateExpressContextOptions) => ({}); // no context
21	+	type Context = trpc.inferAsyncReturnType<typeof createContext>;
22	+
23	+	type ScanResultPackageWithMetadata = WebPageScan.Package & { registryMetadata?: PackageMetadata };
24	+
25	+	export namespace ClientApi {
26	+	export type PackageVulnerabilityResponse = SerializableEntity<PackageVulnerabilityData>;
27	+	export type ScanResultPackageResponse = SerializableEntity<ScanResultPackageWithMetadata>;
28	+	}
29	+
30	+	function mergeRegistryMetadata(
31	+	packages: WebPageScan.Package[],
32	+	registryMetadata: Record<string, PackageMetadata>
33	+	) {
34	+	return packages.map((it) => ({
35	+	...it,
36	+	registryMetadata: registryMetadata[it.name],
37	+	}));
38	+	}
39	+
40	+	type RequestWebPageScanResponse = Pick<WebPageScan, 'status' \| 'finishedAt'> & {
41	+	id: string;
42	+	scanResult?: {
43	+	packages: ScanResultPackageWithMetadata[];
44	+	vulnerabilities: Record<string, PackageVulnerabilityData[]>;
45	+	};
46	+	};
47	+
48	+	export const appRouter = trpc
49	+	.router<Context>()
50	+	.mutation('requestWebPageScan', {
51	+	input: z.string().url(),
52	+	async resolve({ input: url }) {
53	+	const scan = await getOrRequestWebPageScan(url);
54	+
55	+	const scanResponse: RequestWebPageScanResponse = {
56	+	id: scan.id.toString(),
57	+	status: scan.status,
58	+	finishedAt: scan.finishedAt,
59	+	scanResult: undefined,
60	+	};
61	+
62	+	if (scan.scanResult) {
63	+	const packageNames = scan.scanResult.packages.map((it) => it.name);
64	+
65	+	const [metadata, vulnerabilities] = await Promise.all([
66	+	getPackageMetadataByPackageNames(packageNames),
67	+	getAffectingVulnerabilities(scan.scanResult),
68	+	]);
69	+
70	+	scanResponse.scanResult = {
71	+	packages: mergeRegistryMetadata(scan.scanResult.packages, metadata),
72	+	vulnerabilities,
73	+	};
74	+	}
75	+
76	+	return toSerializable(scanResponse);
77	+	},
78	+	})
79	+	.formatError(({ shape, error }) => {
80	+	// TODO: proper reporting
81	+	return {
82	+	...shape,
83	+	data: {
84	+	...shape.data,
85	+	zodError:
86	+	error.code === 'BAD_REQUEST' && error.cause instanceof ZodError
87	+	? error.cause.flatten()
88	+	: null,
89	+	},
90	+	};
91	+	});
92	+
93	+	// export type definition of API
94	+	export type ClientApiRouter = typeof appRouter;
95	+

■ ■ ■ ■ ■ ■

packages/public-api/src/index.ts

		skipped 9 lines
10	10		Env.AwsRegion,
11	11		Env.DatabaseUrl,
12	12		Env.SqsWorkerQueueUrl,
13		-	Env.InternalApiOrigin,
	13	+	Env.InternalApiRootUrl,
	14	+	Env.GradeJsApiKey,
14	15		]);
15	16
16	17		const port = getPort(8080);
		skipped 7 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/middleware/common.ts

1	1		import { ZodError } from 'zod';
2	2		import createCorsMiddleware from 'cors';
3	3		import { Request, Response, NextFunction } from 'express';
4		-	import { NotFoundError, respondWithError } from './response';
	4	+	import { NotFoundError, respondWithError, UnauthorizedError } from './response';
5	5		import { getCorsAllowedOrigins } from '@gradejs-public/shared';
6	6
7	7		const originAllowList = getCorsAllowedOrigins();
		skipped 26 lines
34	34		_next: NextFunction
35	35		) {
36	36		// Log only useful errors
37		-	if (!(error instanceof NotFoundError) && !(error instanceof ZodError)) {
	37	+	if (
	38	+	!(error instanceof NotFoundError) &&
	39	+	!(error instanceof UnauthorizedError) &&
	40	+	!(error instanceof ZodError)
	41	+	) {
38	42		// TODO: add logger
39	43		console.error(error, req);
40	44		}
		skipped 4 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/middleware/response.ts

		skipped 14 lines
15	15		message = 'Not Found';
16	16		}
17	17
	18	+	export class UnauthorizedError extends Error {
	19	+	code = 401;
	20	+	message = 'Unauthorized';
	21	+	}
	22	+
18	23		export function respond<T>(res: Response, data: T) {
19	24		res.send({
20	25		data,
		skipped 6 lines
27	32		message: 'Internal server error, try again later',
28	33		};
29	34
30		-	if (err instanceof NotFoundError) {
	35	+	if (err instanceof NotFoundError \|\| err instanceof UnauthorizedError) {
31	36		error.code = err.code;
32	37		error.message = err.message;
33	38		}
		skipped 15 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/middleware/verifySystemApiToken.ts

1	+	import { NextFunction, Request, Response } from 'express';
2	+	import { getGradeJsApiKey } from '@gradejs-public/shared';
3	+	import { UnauthorizedError } from './response';
4	+
5	+	const API_TOKEN_HEADER = 'X-Api-Key';
6	+
7	+	export function verifySystemApiToken(req: Request, _res: Response, next: NextFunction) {
8	+	if (req.headers[API_TOKEN_HEADER.toLowerCase()] !== getGradeJsApiKey()) {
9	+	next(new UnauthorizedError());
10	+	return;
11	+	}
12	+
13	+	next();
14	+	}
15	+

■ ■ ■ ■ ■ ■

packages/public-api/src/packageMetadata/packageMetadataService.ts

1	+	import { getDatabaseConnection, PackageMetadata } from '@gradejs-public/shared';
2	+
3	+	export async function getPackageMetadataByPackageNames(packageNames: string[]) {
4	+	if (!packageNames.length) {
5	+	return {};
6	+	}
7	+
8	+	const db = await getDatabaseConnection();
9	+	const packageMetadataRepo = db.getRepository(PackageMetadata);
10	+
11	+	const packageMetadataEntities = await packageMetadataRepo
12	+	.createQueryBuilder('pm')
13	+	.where('pm.name IN (:...packageNames)', { packageNames })
14	+	.getMany();
15	+
16	+	const packageMetadataMap = packageMetadataEntities.reduce((acc, val) => {
17	+	acc[val.name] = val;
18	+
19	+	return acc;
20	+	}, {} as Record<string, PackageMetadata>);
21	+
22	+	return packageMetadataMap;
23	+	}
24	+

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncPackageUsageByHostname.test.ts

1	+	import { useDatabaseConnection, useTransactionalTesting } from '@gradejs-public/test-utils';
2	+	import {
3	+	getDatabaseConnection,
4	+	PackageUsageByHostnameProjection,
5	+	WebPageScan,
6	+	} from '@gradejs-public/shared';
7	+	import { findOrCreateWebPage } from '../website/service';
8	+	import { syncPackageUsageByHostname } from './syncPackageUsageByHostname';
9	+
10	+	useDatabaseConnection();
11	+	useTransactionalTesting();
12	+
13	+	describe('projections / packageUsageByHostname', () => {
14	+	it('should sync reported scans', async () => {
15	+	const db = await getDatabaseConnection();
16	+	const em = db.createEntityManager();
17	+
18	+	const url = new URL('https://example.com/test');
19	+
20	+	const webPage = await findOrCreateWebPage(url, em);
21	+
22	+	const scan = await em.getRepository(WebPageScan).save({
23	+	webPage,
24	+	status: WebPageScan.Status.Processed,
25	+	scanResult: {
26	+	packages: [
27	+	{
28	+	name: 'react',
29	+	versionSet: ['17.0.0'],
30	+	versionRange: '17.0.0',
31	+	approximateByteSize: null,
32	+	},
33	+	],
34	+	},
35	+	});
36	+
37	+	await syncPackageUsageByHostname(scan, em);
38	+
39	+	const packageUsageByHostEntries = await em
40	+	.getRepository(PackageUsageByHostnameProjection)
41	+	.find({
42	+	packageName: 'react',
43	+	});
44	+
45	+	expect(packageUsageByHostEntries).toMatchObject([
46	+	{
47	+	sourceScanId: scan.id,
48	+	hostnameId: scan.webPage.hostnameId,
49	+	packageName: 'react',
50	+	packageVersionSet: ['17.0.0'],
51	+	},
52	+	]);
53	+	});
54	+
55	+	it('should clean up changed packages between scans', async () => {
56	+	const db = await getDatabaseConnection();
57	+	const em = db.createEntityManager();
58	+
59	+	const url = new URL('https://example.com/test');
60	+
61	+	const webPage = await findOrCreateWebPage(url, em);
62	+
63	+	const firstScan = await em.getRepository(WebPageScan).save({
64	+	webPage,
65	+	status: WebPageScan.Status.Processed,
66	+	scanResult: {
67	+	packages: [
68	+	{
69	+	name: 'react',
70	+	versionSet: ['17.0.0'],
71	+	versionRange: '17.0.0',
72	+	approximateByteSize: null,
73	+	},
74	+	{
75	+	name: 'react-dom',
76	+	versionSet: ['17.0.0'],
77	+	versionRange: '17.0.0',
78	+	approximateByteSize: null,
79	+	},
80	+	],
81	+	},
82	+	});
83	+
84	+	await syncPackageUsageByHostname(firstScan, em);
85	+
86	+	const packageUsageByHostEntries = await em
87	+	.getRepository(PackageUsageByHostnameProjection)
88	+	.createQueryBuilder('usage')
89	+	.where('usage.hostname_id = :hostnameId', { hostnameId: firstScan.webPage.hostnameId })
90	+	.getMany();
91	+
92	+	expect(packageUsageByHostEntries).toMatchObject([
93	+	{
94	+	sourceScanId: firstScan.id,
95	+	hostnameId: firstScan.webPage.hostnameId,
96	+	packageName: 'react',
97	+	packageVersionSet: ['17.0.0'],
98	+	},
99	+	{
100	+	sourceScanId: firstScan.id,
101	+	hostnameId: firstScan.webPage.hostnameId,
102	+	packageName: 'react-dom',
103	+	packageVersionSet: ['17.0.0'],
104	+	},
105	+	]);
106	+
107	+	const secondScan = await em.getRepository(WebPageScan).save({
108	+	webPage,
109	+	status: WebPageScan.Status.Processed,
110	+	scanResult: {
111	+	packages: [
112	+	{
113	+	name: 'react',
114	+	versionSet: ['17.0.0'],
115	+	versionRange: '17.0.0',
116	+	approximateByteSize: null,
117	+	},
118	+	],
119	+	},
120	+	});
121	+
122	+	await syncPackageUsageByHostname(secondScan, em);
123	+
124	+	const secondPackageUsageByHostEntries = await em
125	+	.getRepository(PackageUsageByHostnameProjection)
126	+	.createQueryBuilder('usage')
127	+	.where('usage.hostname_id = :hostnameId', { hostnameId: firstScan.webPage.hostnameId })
128	+	.getMany();
129	+
130	+	expect(secondPackageUsageByHostEntries).toMatchObject([
131	+	{
132	+	sourceScanId: secondScan.id,
133	+	hostnameId: firstScan.webPage.hostnameId,
134	+	packageName: 'react',
135	+	packageVersionSet: ['17.0.0'],
136	+	},
137	+	]);
138	+	});
139	+	});
140	+

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncPackageUsageByHostname.ts

1	+	import { PackageUsageByHostnameProjection, WebPage, WebPageScan } from '@gradejs-public/shared';
2	+	import { EntityManager } from 'typeorm';
3	+
4	+	export async function syncPackageUsageByHostname(newScan: WebPageScan, em: EntityManager) {
5	+	if (!newScan.scanResult) {
6	+	throw new Error('Scan was not completed');
7	+	}
8	+
9	+	const packageUsageByHostRepo = em.getRepository(PackageUsageByHostnameProjection);
10	+	const webPageScanRepo = em.getRepository(WebPageScan);
11	+	const webPageRepo = em.getRepository(WebPage);
12	+
13	+	const previousScan = await webPageScanRepo
14	+	.createQueryBuilder('webpagescan')
15	+	.where('webpagescan.id < :newId', { newId: newScan.id })
16	+	.andWhere('webpagescan.web_page_id = :webPageId', { webPageId: newScan.webPageId })
17	+	.orderBy('webpagescan.id', 'DESC')
18	+	.limit(1)
19	+	.getOne();
20	+
21	+	if (previousScan) {
22	+	await em
23	+	.createQueryBuilder()
24	+	.delete()
25	+	.from(PackageUsageByHostnameProjection)
26	+	.where('source_scan_id = :scanId', { scanId: previousScan.id })
27	+	.execute();
28	+	}
29	+
30	+	const relatedWebPage = await webPageRepo
31	+	.createQueryBuilder('webpage')
32	+	.where('webpage.id = :id', { id: newScan.webPageId })
33	+	.leftJoinAndSelect('webpage.hostname', 'hostname')
34	+	.getOneOrFail();
35	+
36	+	const packageUsageEntities = newScan.scanResult.packages.map((sourcePackage) =>
37	+	packageUsageByHostRepo.create({
38	+	hostname: relatedWebPage.hostname,
39	+	sourceScan: newScan,
40	+	packageName: sourcePackage.name,
41	+	packageVersionSet: sourcePackage.versionSet,
42	+	})
43	+	);
44	+
45	+	if (packageUsageEntities.length) {
46	+	await packageUsageByHostRepo.save(packageUsageEntities);
47	+	}
48	+	}
49	+

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncScansWithVulnerabilities.test.ts

1	+	import { useDatabaseConnection, useTransactionalTesting } from '@gradejs-public/test-utils';
2	+	import {
3	+	getDatabaseConnection,
4	+	PackageVulnerability,
5	+	ScansWithVulnerabilitiesProjection,
6	+	WebPageScan,
7	+	} from '@gradejs-public/shared';
8	+	import { findOrCreateWebPage } from '../website/service';
9	+	import { syncScansWithVulnerabilities } from './syncScansWithVulnerabilities';
10	+
11	+	useDatabaseConnection();
12	+	useTransactionalTesting();
13	+
14	+	describe('projections / scansWithVulnerabilities', () => {
15	+	it('should sync reported scans with vulnerabilities', async () => {
16	+	const db = await getDatabaseConnection();
17	+	const em = db.createEntityManager();
18	+
19	+	const url = new URL('https://example.com/test');
20	+
21	+	const webPage = await findOrCreateWebPage(url, em);
22	+
23	+	const scan = await em.getRepository(WebPageScan).save({
24	+	webPage,
25	+	status: WebPageScan.Status.Processed,
26	+	scanResult: {
27	+	packages: [
28	+	{
29	+	name: 'react',
30	+	versionSet: ['17.0.0'],
31	+	versionRange: '17.0.0',
32	+	approximateByteSize: null,
33	+	},
34	+	],
35	+	},
36	+	});
37	+
38	+	await em.getRepository(PackageVulnerability).save({
39	+	packageName: 'react',
40	+	packageVersionRange: '<=17.0.0',
41	+	osvId: 'test-osv-id',
42	+	osvData: {
43	+	schema_version: '',
44	+	modified: '',
45	+	id: 'test-osv-id',
46	+	severity: [
47	+	{
48	+	type: 'CVSS_V3',
49	+	score: 'SEVERE',
50	+	},
51	+	],
52	+	database_specific: {
53	+	severity: 'SEVERE',
54	+	},
55	+	},
56	+	});
57	+
58	+	await syncScansWithVulnerabilities(scan, em);
59	+
60	+	const scanWithVulnerabilities = await em
61	+	.getRepository(ScansWithVulnerabilitiesProjection)
62	+	.createQueryBuilder('scans')
63	+	.where('scans.source_scan_id = :scanId', { scanId: scan.id })
64	+	.getOne();
65	+
66	+	expect(scanWithVulnerabilities).toMatchObject({
67	+	vulnerabilities: [
68	+	{
69	+	packageName: 'react',
70	+	severity: 'SEVERE',
71	+	},
72	+	],
73	+	});
74	+	});
75	+	});
76	+

■ ■ ■ ■ ■ ■

packages/public-api/src/projections/syncScansWithVulnerabilities.ts

1	+	import { ScansWithVulnerabilitiesProjection, WebPageScan } from '@gradejs-public/shared';
2	+	import { EntityManager } from 'typeorm';
3	+	import { getAffectingVulnerabilities } from '../vulnerabilities/vulnerabilities';
4	+
5	+	export async function syncScansWithVulnerabilities(newScan: WebPageScan, em: EntityManager) {
6	+	if (!newScan.scanResult) {
7	+	throw new Error('Scan was not completed');
8	+	}
9	+
10	+	const affectingVulnerabilities = await getAffectingVulnerabilities(newScan.scanResult);
11	+
12	+	const vulnerabilityList = Object.values(affectingVulnerabilities).flat();
13	+	if (!vulnerabilityList.length) {
14	+	return;
15	+	}
16	+
17	+	const scansWithVulnerabilitiesRepo = em.getRepository(ScansWithVulnerabilitiesProjection);
18	+
19	+	await scansWithVulnerabilitiesRepo.save({
20	+	sourceScan: newScan,
21	+	vulnerabilities: vulnerabilityList.map((vulnerability) => ({
22	+	packageName: vulnerability.affectedPackageName,
23	+	severity: vulnerability.severity,
24	+	})),
25	+	});
26	+	}
27	+

■ ■ ■ ■ ■ ■

packages/public-api/src/router.test.ts

1	-	import { TRPC_ERROR_CODES_BY_KEY } from '@trpc/server/rpc';
2	-	import {
3	-	internalApi,
4	-	PackageMetadata,
5	-	PackageVulnerability,
6	-	WebPage,
7	-	WebPagePackage,
8	-	} from '@gradejs-public/shared';
9	-	import {
10	-	createSupertestApi,
11	-	useDatabaseConnection,
12	-	useTransactionalTesting,
13	-	} from '@gradejs-public/test-utils';
14	-	import { getRepository } from 'typeorm';
15	-	import { createApp } from './app';
16	-
17	-	useDatabaseConnection();
18	-	useTransactionalTesting();
19	-
20	-	const api = createSupertestApi(createApp);
21	-
22	-	describe('routes / heathCheck', () => {
23	-	it('should return valid response for healthcheck path', async () => {
24	-	await api.get('/').send().expect(200);
25	-	});
26	-
27	-	it('should return not found error', async () => {
28	-	const response = await api
29	-	.get('/any-invalid-route')
30	-	.set('Origin', 'http://localhost:3000')
31	-	.send()
32	-	.expect(404);
33	-
34	-	expect(response.body).toMatchObject({
35	-	error: {
36	-	code: TRPC_ERROR_CODES_BY_KEY.NOT_FOUND,
37	-	},
38	-	});
39	-	});
40	-	});
41	-
42	-	describe('routes / website', () => {
43	-	it('should initiate webpage parsing', async () => {
44	-	const siteUrl = 'https://example.com/' + Math.random().toString();
45	-
46	-	const initiateUrlProcessingInternalMock = jest.spyOn(internalApi, 'initiateUrlProcessing');
47	-
48	-	initiateUrlProcessingInternalMock.mockImplementation((url) =>
49	-	Promise.resolve({
50	-	url,
51	-	status: 'in-progress',
52	-	} as internalApi.Website)
53	-	);
54	-
55	-	const response = await api
56	-	.post('/requestParseWebsite')
57	-	.set('Origin', 'http://localhost:3000')
58	-	.send(JSON.stringify(siteUrl))
59	-	.expect(200);
60	-	const webpage = await getRepository(WebPage).findOne({ url: siteUrl });
61	-
62	-	expect(initiateUrlProcessingInternalMock).toHaveBeenCalledTimes(1);
63	-	expect(initiateUrlProcessingInternalMock).toHaveBeenCalledWith(siteUrl);
64	-	expect(response.body).toMatchObject({
65	-	result: {
66	-	data: {
67	-	id: expect.anything(),
68	-	url: siteUrl,
69	-	hostname: 'example.com',
70	-	status: 'pending',
71	-	},
72	-	},
73	-	});
74	-
75	-	expect(webpage).toMatchObject({
76	-	url: siteUrl,
77	-	hostname: 'example.com',
78	-	status: 'pending',
79	-	});
80	-	});
81	-
82	-	it('should return cached website by a hostname', async () => {
83	-	const hostname = Math.random().toString() + 'example.com';
84	-	const url = `https://${hostname}/`;
85	-
86	-	const fetchUrlPackagesMock = jest.spyOn(internalApi, 'fetchUrlPackages');
87	-
88	-	// Populate
89	-	const webpageInsert = await getRepository(WebPage).insert({
90	-	url,
91	-	hostname,
92	-	status: WebPage.Status.Processed,
93	-	});
94	-
95	-	const packageInsert = await getRepository(WebPagePackage).insert({
96	-	latestUrl: url,
97	-	hostname,
98	-	packageName: 'react',
99	-	possiblePackageVersions: ['17.0.2'],
100	-	packageVersionRange: '17.0.2',
101	-	});
102	-
103	-	const packageMetadataInsert = await getRepository(PackageMetadata).insert({
104	-	name: 'react',
105	-	latestVersion: '18.0.0',
106	-	monthlyDownloads: 100,
107	-	updateSeq: 5,
108	-	license: 'MIT',
109	-	});
110	-
111	-	await getRepository(PackageVulnerability).insert({
112	-	packageName: 'react',
113	-	packageVersionRange: '>=17.0.0 <18.0.0',
114	-	osvId: 'GRJS-test-id',
115	-	osvData: {
116	-	schema_version: '1.2.0',
117	-	id: 'GRJS-test-id',
118	-	summary: 'Test summary',
119	-	database_specific: {
120	-	severity: 'HIGH',
121	-	},
122	-	},
123	-	});
124	-
125	-	const response = await api
126	-	.post('/syncWebsite')
127	-	.set('Origin', 'http://localhost:3000')
128	-	.send(JSON.stringify(hostname))
129	-	.expect(200);
130	-
131	-	expect(fetchUrlPackagesMock).toHaveBeenCalledTimes(0);
132	-	expect(response.body).toMatchObject({
133	-	result: {
134	-	data: {
135	-	webpages: webpageInsert.generatedMaps,
136	-	packages: [
137	-	{
138	-	...packageInsert.generatedMaps[0],
139	-	registryMetadata: packageMetadataInsert.generatedMaps[0],
140	-	},
141	-	],
142	-	vulnerabilities: {
143	-	react: [
144	-	{
145	-	affectedPackageName: 'react',
146	-	affectedVersionRange: '>=17.0.0 <18.0.0',
147	-	osvId: 'GRJS-test-id',
148	-	detailsUrl: `https://github.com/advisories/GRJS-test-id`,
149	-	summary: 'Test summary',
150	-	severity: 'HIGH',
151	-	},
152	-	],
153	-	},
154	-	},
155	-	},
156	-	});
157	-	});
158	-
159	-	it('should sync pending webpages', async () => {
160	-	const hostname = Math.random().toString() + 'example.com';
161	-	const siteUrl = `https://${hostname}/`;
162	-
163	-	// Populate
164	-	const webpageInsert = await getRepository(WebPage).insert({
165	-	url: siteUrl,
166	-	hostname,
167	-	status: WebPage.Status.Pending,
168	-	});
169	-
170	-	const fetchUrlPackagesMock = jest.spyOn(internalApi, 'fetchUrlPackages');
171	-
172	-	fetchUrlPackagesMock.mockImplementation((url) =>
173	-	Promise.resolve({
174	-	id: 0,
175	-	updatedAt: '1',
176	-	createdAt: '1',
177	-	url,
178	-	status: 'ready',
179	-	detectedPackages: [
180	-	{
181	-	name: 'react',
182	-	versionRange: '17.0.2',
183	-	possibleVersions: ['17.0.2'],
184	-	approximateSize: 1337,
185	-	},
186	-	{
187	-	name: 'object-assign',
188	-	versionRange: '4.1.0 - 4.1.1',
189	-	possibleVersions: ['4.1.0', '4.1.1'],
190	-	approximateSize: 42,
191	-	},
192	-	],
193	-	} as internalApi.Website)
194	-	);
195	-
196	-	const response = await api
197	-	.post('/syncWebsite')
198	-	.set('Origin', 'http://localhost:3000')
199	-	.send(JSON.stringify(hostname))
200	-	.expect(200);
201	-	expect(fetchUrlPackagesMock).toHaveBeenCalledTimes(1);
202	-	expect(fetchUrlPackagesMock).toHaveBeenCalledWith(siteUrl);
203	-
204	-	expect(response.body).toMatchObject({
205	-	result: {
206	-	data: {
207	-	webpages: [
208	-	{
209	-	...webpageInsert.generatedMaps.at(0),
210	-	status: WebPage.Status.Processed,
211	-	updatedAt: expect.anything(),
212	-	},
213	-	],
214	-	packages: [
215	-	{
216	-	latestUrl: siteUrl,
217	-	hostname,
218	-	packageName: 'react',
219	-	possiblePackageVersions: ['17.0.2'],
220	-	packageVersionRange: '17.0.2',
221	-	packageMetadata: {
222	-	approximateByteSize: 1337,
223	-	},
224	-	},
225	-	{
226	-	latestUrl: siteUrl,
227	-	hostname,
228	-	packageName: 'object-assign',
229	-	possiblePackageVersions: ['4.1.0', '4.1.1'],
230	-	packageVersionRange: '4.1.0 - 4.1.1',
231	-	packageMetadata: {
232	-	approximateByteSize: 42,
233	-	},
234	-	},
235	-	],
236	-	vulnerabilities: {},
237	-	},
238	-	},
239	-	});
240	-	});
241	-	});
242	-

■ ■ ■ ■ ■ ■

packages/public-api/src/router.ts

1	-	import * as trpc from '@trpc/server';
2	-	// See also: https://colinhacks.com/essays/painless-typesafety
3	-	import { CreateExpressContextOptions } from '@trpc/server/adapters/express';
4	-	import { z, ZodError } from 'zod';
5	-	import { NotFoundError } from './middleware/response';
6	-	import {
7	-	getPackagesByHostname,
8	-	getWebPagesByHostname,
9	-	requestWebPageParse,
10	-	syncWebPage,
11	-	} from './website/service';
12	-	import { getAffectingVulnerabilities } from './vulnerabilities/vulnerabilities';
13	-	import { SerializableEntity, toSerializable } from '@gradejs-public/shared';
14	-
15	-	const hostnameRe =
16	-	/^(?:(?:25[0-5]\|2[0-4][0-9]\|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]\|2[0-4][0-9]\|[01]?[0-9][0-9]?)$\|^(([a-zA-Z0-9]\|[a-zA-Z0-9][a-zA-Z0-9\-][a-zA-Z0-9])\.)+([A-Za-z]\|[A-Za-z][A-Za-z0-9\-][A-Za-z0-9])$/;
17	-
18	-	// created for each request
19	-	export const createContext = (_: CreateExpressContextOptions) => ({}); // no context
20	-	type Context = trpc.inferAsyncReturnType<typeof createContext>;
21	-
22	-	export namespace Api {
23	-	export type WebPage = SerializableEntity<
24	-	Awaited<ReturnType<typeof getWebPagesByHostname>>[number]
25	-	>;
26	-	export type WebPagePackage = SerializableEntity<
27	-	Awaited<ReturnType<typeof getPackagesByHostname>>[number]
28	-	>;
29	-	export type WebSiteParseResult = SerializableEntity<
30	-	Awaited<ReturnType<typeof requestWebPageParse>>
31	-	>;
32	-	export type Vulnerability = SerializableEntity<
33	-	Awaited<ReturnType<typeof getAffectingVulnerabilities>>[string][number]
34	-	>;
35	-	}
36	-
37	-	export const appRouter = trpc
38	-	.router<Context>()
39	-	.mutation('syncWebsite', {
40	-	input: z.string().regex(hostnameRe),
41	-	async resolve({ input: hostname }) {
42	-	const webpages = await getWebPagesByHostname(hostname);
43	-
44	-	if (webpages.length === 0) {
45	-	throw new NotFoundError();
46	-	}
47	-
48	-	await Promise.all(webpages.map(syncWebPage));
49	-	const packages = await getPackagesByHostname(hostname);
50	-	const vulnerabilities = await getAffectingVulnerabilities(packages);
51	-
52	-	return {
53	-	webpages: webpages.map(toSerializable),
54	-	packages: packages.map(toSerializable),
55	-	vulnerabilities,
56	-	};
57	-	},
58	-	})
59	-	.mutation('requestParseWebsite', {
60	-	input: z.string().url(),
61	-	async resolve({ input: url }) {
62	-	return toSerializable(await requestWebPageParse(url));
63	-	},
64	-	})
65	-	.formatError(({ shape, error }) => {
66	-	// TODO: proper reporting
67	-	return {
68	-	...shape,
69	-	data: {
70	-	...shape.data,
71	-	zodError:
72	-	error.code === 'BAD_REQUEST' && error.cause instanceof ZodError
73	-	? error.cause.flatten()
74	-	: null,
75	-	},
76	-	};
77	-	});
78	-
79	-	// export type definition of API
80	-	export type AppRouter = typeof appRouter;
81	-

■ ■ ■ ■ ■ ■

packages/public-api/src/systemApiRouter.test.ts

1	+	import {
2	+	createSupertestApi,
3	+	useDatabaseConnection,
4	+	useTransactionalTesting,
5	+	} from '@gradejs-public/test-utils';
6	+	import { createApp } from './app';
7	+	import { getGradeJsApiKey, internalApi, WebPageScan } from '@gradejs-public/shared';
8	+	import * as WebsiteService from './website/service';
9	+	import { SystemApi } from './systemApiRouter';
10	+	import { getRepository } from 'typeorm';
11	+
12	+	useDatabaseConnection();
13	+	useTransactionalTesting();
14	+
15	+	const api = createSupertestApi(createApp);
16	+
17	+	describe('routes / systemApi', () => {
18	+	it('should deny requests without valid api key', async () => {
19	+	await api.post('/system/scan').send().expect(401);
20	+	await api.post('/system/scan').set('X-Api-Key', 'INVALID').send().expect(401);
21	+	});
22	+
23	+	it('should deny requests with invalid body', async () => {
24	+	await api
25	+	.post('/system/scan')
26	+	.set('X-Api-Key', getGradeJsApiKey())
27	+	.send({
28	+	id: 'test',
29	+	url: 'http://test.com',
30	+	status: 'invalid',
31	+	})
32	+	.expect(400);
33	+	});
34	+
35	+	it('should process reported scans', async () => {
36	+	const payload: SystemApi.ScanReport = {
37	+	id: 'test',
38	+	url: 'http://test.com',
39	+	status: internalApi.WebPageScan.Status.Ready,
40	+	scan: {
41	+	packages: [
42	+	{
43	+	name: 'react',
44	+	versionSet: ['17.0.0'],
45	+	versionRange: '17.0.0',
46	+	approximateByteSize: null,
47	+	},
48	+	],
49	+	},
50	+	};
51	+
52	+	const sencWebPageScanResultMock = jest.spyOn(WebsiteService, 'syncWebPageScanResult');
53	+	sencWebPageScanResultMock.mockImplementation(async () => {
54	+	return getRepository(WebPageScan).create({
55	+	status: WebPageScan.Status.Processed,
56	+	});
57	+	});
58	+
59	+	await api.post('/system/scan').set('X-Api-Key', getGradeJsApiKey()).send(payload).expect(204);
60	+
61	+	expect(sencWebPageScanResultMock).toBeCalledWith(payload);
62	+	});
63	+	});
64	+

■ ■ ■ ■ ■ ■

packages/public-api/src/systemApiRouter.ts

1	+	import { Router } from 'express';
2	+	import { internalApi } from '@gradejs-public/shared';
3	+	import { z } from 'zod';
4	+	import { syncWebPageScanResult } from './website/service';
5	+
6	+	const apiReportedPackageSchema = z.object({
7	+	name: z.string(),
8	+	versionSet: z.array(z.string()),
9	+	versionRange: z.string(),
10	+	approximateByteSize: z.nullable(z.number()),
11	+	});
12	+
13	+	const apiScanReportSchema = z.object({
14	+	id: z.optional(z.string()),
15	+	url: z.string().url(),
16	+	status: z.nativeEnum(internalApi.WebPageScan.Status),
17	+	scan: z.object({
18	+	packages: z.array(apiReportedPackageSchema),
19	+	}),
20	+	});
21	+
22	+	export namespace SystemApi {
23	+	export type ScanReport = z.infer<typeof apiScanReportSchema>;
24	+	export type ReportedPackage = z.infer<typeof apiReportedPackageSchema>;
25	+	}
26	+
27	+	const systemApiRouter = Router();
28	+
29	+	systemApiRouter.post('/scan', async (req, res, next) => {
30	+	try {
31	+	const scanReport = apiScanReportSchema.parse(req.body);
32	+	await syncWebPageScanResult(scanReport);
33	+	} catch (e) {
34	+	next(e);
35	+	return;
36	+	}
37	+
38	+	res.status(204).end();
39	+	});
40	+
41	+	export default systemApiRouter;
42	+

■ ■ ■ ■ ■ ■

packages/public-api/src/vulnerabilities/vulnerabilities.ts

		skipped 1 lines
2	2		GithubAdvisoryDatabaseSpecific,
3	3		PackageVulnerabilityData,
4	4		PackageVulnerability,
5		-	WebPagePackage,
	5	+	WebPageScan,
6	6		} from '@gradejs-public/shared';
7	7		import { getRepository } from 'typeorm';
8	8		import semver from 'semver';
		skipped 10 lines
19	19		return vulnerabilitiesQuery.getMany();
20	20		}
21	21
22		-	export async function getAffectingVulnerabilities(packages: WebPagePackage[]) {
	22	+	export async function getAffectingVulnerabilities(scanResult: WebPageScan.Result) {
23	23		const affectingVulnerabilitiesByPackage: Record<string, PackageVulnerabilityData[]> = {};
	24	+
	25	+	const packages = scanResult.packages;
24	26		if (!packages.length) {
25	27		return affectingVulnerabilitiesByPackage;
26	28		}
27	29
28	30		const packagesByNames = packages.reduce((acc, pkg) => {
29		-	acc[pkg.packageName] = pkg;
	31	+	acc[pkg.name] = pkg;
30	32		return acc;
31		-	}, {} as Record<string, WebPagePackage>);
	33	+	}, {} as Record<string, typeof packages[number]>);
32	34
33	35		const vulnerabilitiesByPackage = await getVulnerabilitiesByPackageNames(
34	36		Object.keys(packagesByNames)
		skipped 2 lines
37	39		for (const vulnerability of vulnerabilitiesByPackage) {
38	40		const relatedPackage = packagesByNames[vulnerability.packageName]!;
39	41		const affectsReportedRange = semver.subset(
40		-	relatedPackage.packageVersionRange,
	42	+	relatedPackage.versionRange,
41	43		vulnerability.packageVersionRange,
42	44		{ loose: true }
43	45		);
		skipped 25 lines

■ ■ ■ ■ ■ ■

packages/public-api/src/website/service.test.ts

1	+	import { useDatabaseConnection, useTransactionalTesting } from '@gradejs-public/test-utils';
2	+	import { findOrCreateWebPage, syncWebPageScanResult } from './service';
3	+	import { getDatabaseConnection, internalApi, WebPageScan } from '@gradejs-public/shared';
4	+
5	+	useDatabaseConnection();
6	+	useTransactionalTesting();
7	+
8	+	describe('website / service', () => {
9	+	it('should persist expected scan result', async () => {
10	+	const db = await getDatabaseConnection();
11	+	const em = db.createEntityManager();
12	+
13	+	const url = new URL('https://example.com/test');
14	+
15	+	const webPage = await findOrCreateWebPage(url, em);
16	+
17	+	const scan = await em.getRepository(WebPageScan).save({
18	+	webPage: webPage,
19	+	status: WebPageScan.Status.Pending,
20	+	});
21	+
22	+	await syncWebPageScanResult({
23	+	id: scan.id.toString(),
24	+	status: internalApi.WebPageScan.Status.Ready,
25	+	url: url.toString(),
26	+	scan: {
27	+	packages: [
28	+	{
29	+	name: 'react',
30	+	versionSet: ['17.0.0'],
31	+	versionRange: '17.0.0',
32	+	approximateByteSize: null,
33	+	},
34	+	],
35	+	},
36	+	});
37	+
38	+	const updatedScan = await em.getRepository(WebPageScan).findOneOrFail({ id: scan.id });
39	+	expect(updatedScan).toMatchObject({
40	+	status: WebPageScan.Status.Processed,
41	+	scanResult: {
42	+	packages: [
43	+	{
44	+	name: 'react',
45	+	versionSet: ['17.0.0'],
46	+	versionRange: '17.0.0',
47	+	approximateByteSize: null,
48	+	},
49	+	],
50	+	},
51	+	});
52	+	expect(updatedScan.finishedAt?.getTime()).toBeGreaterThan(updatedScan.createdAt.getTime());
53	+	});
54	+
55	+	it('should persist new scan result', async () => {
56	+	const url = new URL('https://example.com/test2');
57	+
58	+	const scan = await syncWebPageScanResult({
59	+	status: internalApi.WebPageScan.Status.Ready,
60	+	url: url.toString(),
61	+	scan: {
62	+	packages: [
63	+	{
64	+	name: 'react',
65	+	versionSet: ['17.0.0'],
66	+	versionRange: '17.0.0',
67	+	approximateByteSize: null,
68	+	},
69	+	],
70	+	},
71	+	});
72	+
73	+	expect(scan).toMatchObject({
74	+	status: WebPageScan.Status.Processed,
75	+	scanResult: {
76	+	packages: [
77	+	{
78	+	name: 'react',
79	+	versionSet: ['17.0.0'],
80	+	versionRange: '17.0.0',
81	+	approximateByteSize: null,
82	+	},
83	+	],
84	+	},
85	+	});
86	+	});
87	+
88	+	it('should throw upon syncing non-existing scan', async () => {
89	+	const url = new URL('https://example.com/test3');
90	+
91	+	const syncPromise = syncWebPageScanResult({
92	+	id: '9999999',
93	+	status: internalApi.WebPageScan.Status.Ready,
94	+	url: url.toString(),
95	+	scan: {
96	+	packages: [
97	+	{
98	+	name: 'react',
99	+	versionSet: ['17.0.0'],
100	+	versionRange: '17.0.0',
101	+	approximateByteSize: null,
102	+	},
103	+	],
104	+	},
105	+	});
106	+
107	+	await expect(syncPromise).rejects.toThrow();
108	+	});
109	+	});
110	+

■ ■ ■ ■ ■ ■

packages/public-api/src/website/service.ts

1		-	import { getRepository } from 'typeorm';
2		-	import { WebPage, WebPagePackage, internalApi } from '@gradejs-public/shared';
	1	+	import {
	2	+	WebPageScan,
	3	+	getDatabaseConnection,
	4	+	internalApi,
	5	+	Hostname,
	6	+	WebPage,
	7	+	} from '@gradejs-public/shared';
	8	+	import { EntityManager } from 'typeorm';
	9	+	import { syncPackageUsageByHostname } from '../projections/syncPackageUsageByHostname';
	10	+	import { syncScansWithVulnerabilities } from '../projections/syncScansWithVulnerabilities';
	11	+	import { SystemApi } from '../systemApiRouter';
3	12
4		-	export async function requestWebPageParse(url: string) {
5		-	const [cached, internal] = await Promise.all([
6		-	getRepository(WebPage).findOne({ url }),
7		-	internalApi.initiateUrlProcessing(url),
8		-	// Clear cached results
9		-	getRepository(WebPagePackage).delete({ hostname: getHostnameFromUrl(url) }),
10		-	]);
	13	+	const RESCAN_TIMEOUT_MS = 1000 * 60 * 60 * 24; // 1 day in ms
	14	+
	15	+	export async function findOrCreateWebPage(url: URL, em: EntityManager) {
	16	+	const hostnameRepo = em.getRepository(Hostname);
	17	+	const webPageRepo = em.getRepository(WebPage);
11	18
12		-	// Insert or update database entry
13		-	return getRepository(WebPage).save({
14		-	url: internal.url,
15		-	hostname: getHostnameFromUrl(internal.url),
16		-	status: mapInternalWebsiteStatus(internal.status),
17		-	...(cached ? { id: cached.id } : {}),
	19	+	let hostnameEntity = await hostnameRepo.findOne({
	20	+	hostname: url.hostname,
18	21		});
19		-	}
20		-
21		-	export async function syncWebPage(webpage: WebPage) {
22		-	// Skip sync for processed items
23		-	if (webpage.status !== WebPage.Status.Pending) {
24		-	return;
	22	+	if (!hostnameEntity) {
	23	+	hostnameEntity = await hostnameRepo.save({
	24	+	hostname: url.hostname,
	25	+	});
25	26		}
26	27
27		-	const internal = await internalApi.fetchUrlPackages(webpage.url);
28		-	const nextStatus = mapInternalWebsiteStatus(internal.status);
29		-
30		-	// Save updated status if changed
31		-	if (webpage.status !== nextStatus) {
32		-	webpage.status = nextStatus;
33		-	await webpage.save();
	28	+	let webPageEntity = await webPageRepo
	29	+	.createQueryBuilder('web_page')
	30	+	.where('web_page.hostname_id = :hostnameId', { hostnameId: hostnameEntity.id })
	31	+	.andWhere('web_page.path = :path', { path: url.pathname })
	32	+	.limit(1)
	33	+	.getOne();
	34	+	if (!webPageEntity) {
	35	+	webPageEntity = await webPageRepo.save({
	36	+	hostname: hostnameEntity,
	37	+	path: url.pathname,
	38	+	});
34	39		}
35	40
36		-	// Save packages if ready
37		-	if (webpage.status === WebPage.Status.Processed && internal.detectedPackages.length > 0) {
38		-	await getRepository(WebPagePackage).upsert(
39		-	internal.detectedPackages.map((pkg) => ({
40		-	latestUrl: internal.url,
41		-	hostname: getHostnameFromUrl(internal.url),
42		-	packageName: pkg.name,
43		-	possiblePackageVersions: pkg.possibleVersions,
44		-	packageVersionRange: pkg.versionRange,
45		-	packageMetadata: {
46		-	approximateByteSize: pkg.approximateSize ?? undefined,
47		-	},
48		-	})),
49		-	['hostname', 'packageName']
50		-	);
51		-	}
	41	+	return webPageEntity;
52	42		}
53	43
54		-	export async function getWebPagesByHostname(hostname: string) {
55		-	return getRepository(WebPage).find({ hostname });
	44	+	export async function getOrRequestWebPageScan(url: string) {
	45	+	const parsedUrl = new URL(url);
	46	+
	47	+	const db = await getDatabaseConnection();
	48	+
	49	+	const result = await db.transaction(async (em) => {
	50	+	const webPageScanRepo = em.getRepository(WebPageScan);
	51	+
	52	+	const webPageEntity = await findOrCreateWebPage(parsedUrl, em);
	53	+
	54	+	const mostRecentScan = await webPageScanRepo
	55	+	.createQueryBuilder('scan')
	56	+	.where('scan.web_page_id = :webPageId', { webPageId: webPageEntity.id })
	57	+	.orderBy('scan.created_at', 'DESC')
	58	+	.limit(1)
	59	+	.getOne();
	60	+
	61	+	if (mostRecentScan && Date.now() - mostRecentScan.createdAt.getTime() < RESCAN_TIMEOUT_MS) {
	62	+	return mostRecentScan;
	63	+	}
	64	+
	65	+	const webPageScanEntity = await webPageScanRepo.save({
	66	+	webPage: webPageEntity,
	67	+	status: WebPageScan.Status.Pending,
	68	+	});
	69	+
	70	+	await internalApi.requestWebPageScan(parsedUrl.toString(), webPageScanEntity.id.toString());
	71	+
	72	+	return webPageScanEntity;
	73	+	});
	74	+
	75	+	return result;
56	76		}
57	77
58		-	export async function getPackagesByHostname(hostname: string) {
59		-	return getRepository(WebPagePackage).find({
60		-	relations: ['registryMetadata'],
61		-	where: { hostname },
	78	+	export async function syncWebPageScanResult(scanReport: SystemApi.ScanReport) {
	79	+	const db = await getDatabaseConnection();
	80	+
	81	+	return await db.transaction(async (em) => {
	82	+	const webPageScanRepo = em.getRepository(WebPageScan);
	83	+
	84	+	let scanEntity: WebPageScan;
	85	+	if (scanReport.id) {
	86	+	scanEntity = await webPageScanRepo.findOneOrFail({ id: parseInt(scanReport.id, 10) });
	87	+	} else {
	88	+	const webPageEntity = await findOrCreateWebPage(new URL(scanReport.url), em);
	89	+	scanEntity = webPageScanRepo.create({
	90	+	webPage: webPageEntity,
	91	+	});
	92	+	}
	93	+
	94	+	scanEntity.status = mapInternalWebsiteStatus(scanReport.status);
	95	+	scanEntity.finishedAt = new Date();
	96	+	scanEntity.scanResult = scanReport.scan;
	97	+
	98	+	await webPageScanRepo.save(scanEntity);
	99	+
	100	+	await Promise.all([
	101	+	syncPackageUsageByHostname(scanEntity, em),
	102	+	syncScansWithVulnerabilities(scanEntity, em),
	103	+	]);
	104	+
	105	+	return scanEntity;
62	106		});
63	107		}
64	108
65		-	function mapInternalWebsiteStatus(status: internalApi.WebsiteStatus) {
	109	+	function mapInternalWebsiteStatus(status: internalApi.WebPageScan.Status) {
66	110		switch (status) {
67		-	case internalApi.WebsiteStatus.Invalid:
68		-	return WebPage.Status.Unsupported;
69		-	case internalApi.WebsiteStatus.InProgress:
70		-	return WebPage.Status.Pending;
71		-	case internalApi.WebsiteStatus.Protected:
72		-	return WebPage.Status.Protected;
	111	+	case internalApi.WebPageScan.Status.Invalid:
	112	+	return WebPageScan.Status.Unsupported;
	113	+	case internalApi.WebPageScan.Status.InProgress:
	114	+	return WebPageScan.Status.Pending;
	115	+	case internalApi.WebPageScan.Status.Protected:
	116	+	return WebPageScan.Status.Protected;
73	117		default:
74		-	return WebPage.Status.Processed;
	118	+	return WebPageScan.Status.Processed;
75	119		}
76	120		}
77	121
78		-	function getHostnameFromUrl(url: string) {
79		-	return new URL(url).hostname;
80		-	}
81		-

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/hostname.ts

1	+	import { BaseEntity, Column, Entity, Index, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
2	+	import { WebPage } from './webPage';
3	+
4	+	@Entity({ name: 'hostname' })
5	+	@Index(['hostname'], { unique: true })
6	+	export class Hostname extends BaseEntity {
7	+	@PrimaryGeneratedColumn({ type: 'int' })
8	+	id!: number;
9	+
10	+	@Column()
11	+	hostname!: string;
12	+
13	+	@OneToMany(() => WebPage, (webPage) => webPage.hostname)
14	+	webPages?: WebPage[];
15	+	}
16	+

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/packageMetadata.ts

1		-	import { Column, Entity, Index, PrimaryColumn, BaseEntity } from 'typeorm';
	1	+	import { Column, Entity, Index, BaseEntity, PrimaryGeneratedColumn } from 'typeorm';
2	2
3	3		type Maintainer = {
4	4		name: string;
		skipped 5 lines
10	10		@Entity({ name: 'package_metadata' })
11	11		@Index(['name'], { unique: true })
12	12		export class PackageMetadata extends BaseEntity {
13		-	@PrimaryColumn({ type: 'int', generated: 'increment' })
	13	+	@PrimaryGeneratedColumn({ type: 'int' })
14	14		id!: number;
15	15
16	16		@Column()
		skipped 39 lines

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/packagePopularityView.ts

1	+	import { BaseEntity, Column, Index, ViewEntity } from 'typeorm';
2	+
3	+	@ViewEntity({ name: 'package_popularity_view' })
4	+	@Index(['packageName'])
5	+	@Index(['usageByHostnameCount'])
6	+	export class PackagePopularityView extends BaseEntity {
7	+	@Column()
8	+	packageName!: string;
9	+
10	+	@Column()
11	+	usageByHostnameCount!: number;
12	+
13	+	@Column({ type: 'jsonb' })
14	+	versionPopularity!: PackageVersionPopularity;
15	+	}
16	+
17	+	type PackageVersionPopularity = Array<{ package_version: string; count: number }>;
18	+

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/packageUsageByHostnameProjection.ts

1	+	import {
2	+	BaseEntity,
3	+	Column,
4	+	Entity,
5	+	Index,
6	+	ManyToOne,
7	+	PrimaryGeneratedColumn,
8	+	RelationId,
9	+	} from 'typeorm';
10	+	import { Hostname } from './hostname';
11	+	import { WebPageScan } from './webPageScan';
12	+
13	+	@Entity({ name: 'package_usage_by_hostname_projection' })
14	+	@Index(['packageName'])
15	+	export class PackageUsageByHostnameProjection extends BaseEntity {
16	+	@PrimaryGeneratedColumn({ type: 'int' })
17	+	id!: number;
18	+
19	+	@ManyToOne(() => Hostname)
20	+	hostname!: Hostname;
21	+
22	+	@RelationId((self: PackageUsageByHostnameProjection) => self.hostname)
23	+	hostnameId!: number;
24	+
25	+	@ManyToOne(() => WebPageScan)
26	+	sourceScan!: WebPageScan;
27	+
28	+	@RelationId((self: PackageUsageByHostnameProjection) => self.sourceScan)
29	+	sourceScanId!: number;
30	+
31	+	@Column()
32	+	packageName!: string;
33	+
34	+	@Column({ type: 'jsonb' })
35	+	packageVersionSet!: string[];
36	+	}
37	+

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/packageVulnerability.ts

1		-	import { Column, Entity, Index, PrimaryColumn, BaseEntity } from 'typeorm';
	1	+	import { Column, Entity, Index, BaseEntity, PrimaryGeneratedColumn } from 'typeorm';
2	2
3	3		export enum OSVAffectedRangeType {
4	4		GIT = 'GIT',
		skipped 81 lines
86	86
87	87		export enum GithubAdvisorySeverity {
88	88		Low = 'LOW',
89		-	Medium = 'MEDIUM',
	89	+	Moderate = 'MODERATE',
90	90		High = 'HIGH',
91	91		Critical = 'CRITICAL',
92	92		}
		skipped 10 lines
103	103		@Entity({ name: 'package_vulnerability' })
104	104		@Index(['packageName', 'osvId'], { unique: true })
105	105		export class PackageVulnerability extends BaseEntity {
106		-	@PrimaryColumn({ type: 'int', generated: 'increment' })
	106	+	@PrimaryGeneratedColumn({ type: 'int' })
107	107		id!: number; // Autogenerated PKey as one vulnerability may point to multiple packages
108	108
109	109		@Column()
		skipped 12 lines

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/scansWithVulnerabilitiesProjection.ts

1	+	import {
2	+	BaseEntity,
3	+	Column,
4	+	CreateDateColumn,
5	+	Entity,
6	+	Index,
7	+	ManyToOne,
8	+	PrimaryGeneratedColumn,
9	+	RelationId,
10	+	} from 'typeorm';
11	+	import { WebPageScan } from './webPageScan';
12	+
13	+	@Entity({ name: 'scans_with_vulnerabilities_projection' })
14	+	@Index(['createdAt'])
15	+	export class ScansWithVulnerabilitiesProjection extends BaseEntity {
16	+	@PrimaryGeneratedColumn({ type: 'int' })
17	+	id!: number;
18	+
19	+	@ManyToOne(() => WebPageScan)
20	+	sourceScan!: WebPageScan;
21	+
22	+	@RelationId((self: ScansWithVulnerabilitiesProjection) => self.sourceScan)
23	+	sourceScanId!: number;
24	+
25	+	@Column({ type: 'jsonb' })
26	+	vulnerabilities!: CompactVulnerabilityDescription[];
27	+
28	+	@CreateDateColumn()
29	+	createdAt!: Date;
30	+	}
31	+
32	+	export type CompactVulnerabilityDescription = {
33	+	affectedPackageName: string;
34	+	severity: string;
35	+	};
36	+

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/webPage.ts

1	1		import {
2	2		Column,
3	3		Entity,
4		-	Index,
5		-	PrimaryColumn,
6	4		BaseEntity,
7	5		CreateDateColumn,
8		-	UpdateDateColumn,
	6	+	PrimaryGeneratedColumn,
	7	+	ManyToOne,
	8	+	JoinColumn,
	9	+	OneToMany,
	10	+	RelationId,
	11	+	Index,
9	12		} from 'typeorm';
	13	+	import { Hostname } from './hostname';
	14	+	import { WebPageScan } from './webPageScan';
10	15
11	16		@Entity({ name: 'web_page' })
12		-	@Index(['url'], { unique: true })
	17	+	@Index(['hostname', 'path'], { unique: true })
13	18		export class WebPage extends BaseEntity {
14		-	@PrimaryColumn({ type: 'int', generated: 'increment' })
	19	+	@PrimaryGeneratedColumn({ type: 'int' })
15	20		id!: number;
16	21
17		-	@Column()
18		-	hostname!: string;
	22	+	@ManyToOne(() => Hostname, (hostname) => hostname.webPages)
	23	+	@JoinColumn({ name: 'hostname_id', referencedColumnName: 'id' })
	24	+	hostname!: Hostname;
19	25
20		-	@Column()
21		-	url!: string;
	26	+	@RelationId((self: WebPage) => self.hostname)
	27	+	hostnameId!: number;
22	28
23	29		@Column()
24		-	status!: WebPage.Status;
	30	+	path!: string;
25	31
26		-	@UpdateDateColumn()
27		-	updatedAt?: Date;
	32	+	@OneToMany(() => WebPageScan, (webPageScan) => webPageScan.webPage)
	33	+	scans!: WebPageScan[];
28	34
29	35		@CreateDateColumn()
30	36		createdAt!: Date;
31	37		}
32	38
33		-	export namespace WebPage {
34		-	export enum Status {
35		-	Pending = 'pending',
36		-	Processed = 'processed',
37		-	Unsupported = 'unsupported',
38		-	Protected = 'protected',
39		-	}
40		-	}
41		-

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/webPagePackage.ts

1	-	import {
2	-	Column,
3	-	Entity,
4	-	Index,
5	-	PrimaryColumn,
6	-	BaseEntity,
7	-	CreateDateColumn,
8	-	UpdateDateColumn,
9	-	ManyToOne,
10	-	JoinColumn,
11	-	} from 'typeorm';
12	-	import { PackageMetadata } from './packageMetadata';
13	-
14	-	export type WebPagePackageMetadata = {
15	-	approximateByteSize?: number;
16	-	};
17	-
18	-	@Entity({ name: 'web_page_package' })
19	-	@Index(['hostname', 'packageName'], { unique: true })
20	-	export class WebPagePackage extends BaseEntity {
21	-	@PrimaryColumn({ type: 'int', generated: 'increment' })
22	-	id!: number;
23	-
24	-	@Column()
25	-	hostname!: string;
26	-
27	-	@Column()
28	-	latestUrl!: string;
29	-
30	-	@Column()
31	-	packageName!: string;
32	-
33	-	@Column({ type: 'jsonb' })
34	-	possiblePackageVersions!: string[];
35	-
36	-	@Column()
37	-	packageVersionRange!: string;
38	-
39	-	@Column({ type: 'jsonb' })
40	-	packageMetadata?: WebPagePackageMetadata;
41	-
42	-	@UpdateDateColumn()
43	-	updatedAt?: Date;
44	-
45	-	@CreateDateColumn()
46	-	createdAt!: Date;
47	-
48	-	@ManyToOne(() => PackageMetadata)
49	-	@JoinColumn({ name: 'package_name', referencedColumnName: 'name' })
50	-	registryMetadata?: PackageMetadata;
51	-	}
52	-

■ ■ ■ ■ ■ ■

packages/shared/src/database/entities/webPageScan.ts

1	+	import {
2	+	Column,
3	+	Entity,
4	+	BaseEntity,
5	+	CreateDateColumn,
6	+	ManyToOne,
7	+	JoinColumn,
8	+	RelationId,
9	+	Index,
10	+	PrimaryGeneratedColumn,
11	+	} from 'typeorm';
12	+	import { WebPage } from './webPage';
13	+	import { DetectedPackage } from '../../systemApi/api';
14	+
15	+	@Entity({ name: 'web_page_scan' })
16	+	@Index(['webPage', 'createdAt'])
17	+	export class WebPageScan extends BaseEntity {
18	+	@PrimaryGeneratedColumn({ type: 'int' })
19	+	id!: number;
20	+
21	+	@ManyToOne(() => WebPage, (webPage) => webPage.scans)
22	+	@JoinColumn({ name: 'web_page_id', referencedColumnName: 'id' })
23	+	webPage!: WebPage;
24	+
25	+	@RelationId((self: WebPageScan) => self.webPage)
26	+	webPageId!: number;
27	+
28	+	@Column()
29	+	status!: WebPageScan.Status;
30	+
31	+	@Column({ type: 'jsonb' })
32	+	scanResult?: WebPageScan.Result;
33	+
34	+	@CreateDateColumn()
35	+	createdAt!: Date;
36	+
37	+	@Column()
38	+	finishedAt?: Date;
39	+	}
40	+
41	+	export namespace WebPageScan {
42	+	export enum Status {
43	+	Pending = 'pending',
44	+	Processed = 'processed',
45	+	Unsupported = 'unsupported',
46	+	Protected = 'protected',
47	+	Failed = 'failed',
48	+	}
49	+
50	+	export type Package = DetectedPackage;
51	+	export type Result = {
52	+	packages: Package[];
53	+	};
54	+	}
55	+

■ ■ ■ ■ ■ ■

packages/shared/src/database/migrations/1662077882511-DataLayerRework.ts

1	+	import { MigrationInterface, QueryRunner } from 'typeorm';
2	+
3	+	export class DataLayerRework1662077882511 implements MigrationInterface {
4	+	public async up(queryRunner: QueryRunner): Promise<void> {
5	+	await queryRunner.query(`
6	+	DROP TABLE "web_page_package";
7	+	`);
8	+
9	+	await queryRunner.query(`
10	+	DROP TABLE "web_page";
11	+	`);
12	+
13	+	await queryRunner.query(`
14	+	CREATE TABLE "hostname" (
15	+	"id" serial primary key,
16	+	"hostname" text not null
17	+	);
18	+
19	+	CREATE UNIQUE INDEX "hostname_hostname" ON "hostname" ("hostname");
20	+	`);
21	+
22	+	await queryRunner.query(`
23	+	CREATE TABLE "web_page" (
24	+	"id" serial primary key,
25	+	"hostname_id" integer not null references "hostname"("id"),
26	+	"path" text not null,
27	+	"created_at" timestamp not null default now()
28	+	);
29	+
30	+	CREATE UNIQUE INDEX "web_page_hostname_id_path" ON "web_page" ("hostname_id", "path");
31	+	`);
32	+
33	+	await queryRunner.query(`
34	+	CREATE TABLE "web_page_scan" (
35	+	"id" serial primary key,
36	+	"web_page_id" integer not null references "web_page"("id"),
37	+	"status" text not null,
38	+	"scan_result" jsonb default null,
39	+	"created_at" timestamp not null default now(),
40	+	"finished_at" timestamp
41	+	);
42	+
43	+	CREATE INDEX "web_page_scan_web_page_id_created_at" ON "web_page_scan" ("web_page_id", "created_at" DESC);
44	+	`);
45	+
46	+	await queryRunner.query(`
47	+	CREATE TABLE "package_usage_by_hostname_projection" (
48	+	"id" serial primary key,
49	+	"hostname_id" integer not null references "hostname"("id"),
50	+	"source_scan_id" integer not null references "web_page_scan"("id"),
51	+	"package_name" text not null,
52	+	"package_version_set" jsonb not null default '[]'
53	+	);
54	+
55	+	CREATE INDEX "package_usage_by_hostname_projection_package_name" ON "package_usage_by_hostname_projection" ("package_name");
56	+	`);
57	+
58	+	await queryRunner.query(`
59	+	CREATE TABLE "scans_with_vulnerabilities_projection" (
60	+	"id" serial primary key,
61	+	"source_scan_id" integer not null references "web_page_scan"("id"),
62	+	"vulnerabilities" jsonb not null,
63	+	"created_at" timestamp not null default now()
64	+	);
65	+
66	+	CREATE INDEX "scans_with_vulnerabilities_projection_created_at" ON "scans_with_vulnerabilities_projection" ("created_at" DESC);
67	+	`);
68	+
69	+	await queryRunner.query(`
70	+	CREATE MATERIALIZED VIEW "package_popularity_view" AS
71	+	SELECT
72	+	"package_usage"."package_name" AS "package_name",
73	+	count(DISTINCT "package_usage"."hostname_id") as "usage_by_hostname_count",
74	+	(SELECT jsonb_agg(r)
75	+	FROM (
76	+	SELECT "package_version", count("package_version")
77	+	FROM
78	+	"package_usage_by_hostname_projection" as "package_usage_subquery",
79	+	jsonb_array_elements_text("package_usage_subquery"."package_version_set") AS "package_version"
80	+	WHERE "package_usage_subquery"."package_name" = "package_usage"."package_name"
81	+	GROUP BY "package_version"
82	+	) as r
83	+	) as version_popularity
84	+	FROM
85	+	"package_usage_by_hostname_projection" as "package_usage"
86	+	GROUP BY "package_usage"."package_name";
87	+
88	+	CREATE INDEX "package_popularity_view_usage_by_hostname_count" ON "package_popularity_view" ("usage_by_hostname_count" DESC);
89	+	CREATE INDEX "package_popularity_view_package_name" ON "package_popularity_view" ("package_name" DESC);
90	+	`);
91	+	}
92	+
93	+	public async down(queryRunner: QueryRunner): Promise<void> {
94	+	await queryRunner.query(`
95	+	DROP MATERIALIZED VIEW "package_popularity_view";
96	+	`);
97	+
98	+	await queryRunner.query(`
99	+	DROP TABLE "hostname", "web_page", "web_page_scan", "package_usage_by_hostname_projection", "scans_with_vulnerabilities_projection" cascade;
100	+	`);
101	+
102	+	await queryRunner.query(`
103	+	CREATE TABLE "web_page" (
104	+	"id" serial primary key,
105	+	"hostname" text not null,
106	+	"url" text not null,
107	+	"status" text not null,
108	+	"updated_at" timestamp not null default now(),
109	+	"created_at" timestamp not null default now()
110	+	);
111	+
112	+	CREATE UNIQUE INDEX "web_page_url_index" ON "web_page" ("url");
113	+	`);
114	+
115	+	await queryRunner.query(`
116	+	create table "web_page_package" (
117	+	"id" serial primary key,
118	+	"hostname" text not null,
119	+	"latest_url" text not null,
120	+	"package_name" text not null,
121	+	"possible_package_versions" jsonb not null default '[]',
122	+	"package_version_range" text not null,
123	+	"package_metadata" jsonb,
124	+	"updated_at" timestamp not null default now(),
125	+	"created_at" timestamp not null default now()
126	+	);
127	+
128	+	CREATE UNIQUE INDEX "web_page_package_hostname_package_name_index" ON "web_page_package" ("hostname", "package_name");
129	+	`);
130	+	}
131	+	}
132	+

■ ■ ■ ■ ■ ■

packages/shared/src/index.ts

1	1		export * from './database/connection';
	2	+	export * from './database/entities/hostname';
2	3		export * from './database/entities/webPage';
3		-	export * from './database/entities/webPagePackage';
	4	+	export * from './database/entities/webPageScan';
4	5		export * from './database/entities/packageMetadata';
5	6		export * from './database/entities/packageVulnerability';
	7	+	export * from './database/entities/packageUsageByHostnameProjection';
	8	+	export * from './database/entities/scansWithVulnerabilitiesProjection';
6	9
7	10		export * from './utils/aws';
8	11		export * from './utils/env';
9	12		export * from './utils/types';
10	13
11		-	export * as internalApi from './internalApi/api';
	14	+	export * as internalApi from './systemApi/api';
12	15
13	16		export * from './worker/types';
14	17

■ ■ ■ ■ ■ ■

packages/shared/src/internalApi/api.test.ts

1	-	import fetch from 'node-fetch';
2	-	import { initiateUrlProcessing, fetchUrlPackages } from './api';
3	-
4	-	jest.mock('node-fetch');
5	-	process.env.INTERNAL_API_ORIGIN = 'https://mocked-domain.com/';
6	-
7	-	const fetchMode = fetch as any as jest.Mock;
8	-	const { Response } = jest.requireActual('node-fetch');
9	-
10	-	beforeEach(() => {
11	-	fetchMode.mockClear();
12	-	});
13	-
14	-	describe('internalApi', () => {
15	-	it('initiateUrlProcessingInternal', async () => {
16	-	const url = 'http://example.com/' + Math.random().toString();
17	-	const response = {
18	-	data: {
19	-	id: 1,
20	-	url,
21	-	status: 'in-progress',
22	-	packages: [],
23	-	updatedAt: new Date().toISOString(),
24	-	createdAt: new Date().toISOString(),
25	-	},
26	-	};
27	-
28	-	fetchMode.mockImplementation(() => Promise.resolve(new Response(JSON.stringify(response))));
29	-
30	-	const result = await initiateUrlProcessing(url);
31	-
32	-	expect(fetchMode).toBeCalledWith('https://mocked-domain.com/website/parse', {
33	-	method: 'POST',
34	-	body: `{"url":"${url}"}`,
35	-	headers: {
36	-	'Content-Type': 'application/json',
37	-	},
38	-	});
39	-
40	-	expect(fetchMode).toHaveBeenCalledTimes(1);
41	-	expect(result).toMatchObject(response.data);
42	-	});
43	-
44	-	it('fetchUrlPackages', async () => {
45	-	const url = 'http://example.com/parsed';
46	-	const response = {
47	-	data: {
48	-	id: 1,
49	-	url,
50	-	status: 'ready',
51	-	detectedPackages: [
52	-	{
53	-	name: 'react',
54	-	possibleVersions: ['17.0.2'],
55	-	versionRange: '17.0.2',
56	-	approximateSize: 1337,
57	-	},
58	-	],
59	-	updatedAt: new Date().toISOString(),
60	-	createdAt: new Date().toISOString(),
61	-	},
62	-	};
63	-
64	-	fetchMode.mockImplementation(() => Promise.resolve(new Response(JSON.stringify(response))));
65	-
66	-	const result = await fetchUrlPackages(url);
67	-
68	-	expect(fetchMode).toBeCalledWith(
69	-	'https://mocked-domain.com/website?url=http%3A%2F%2Fexample.com%2Fparsed',
70	-	{
71	-	method: 'GET',
72	-	}
73	-	);
74	-
75	-	expect(fetchMode).toHaveBeenCalledTimes(1);
76	-	expect(result).toMatchObject(response.data);
77	-	});
78	-	});
79	-

■ ■ ■ ■ ■ ■

packages/shared/src/systemApi/api.test.ts

1	+	import fetch from 'node-fetch';
2	+	import { requestWebPageScan } from './api';
3	+
4	+	jest.mock('node-fetch');
5	+	process.env.INTERNAL_API_ORIGIN = 'https://mocked-domain.com/';
6	+
7	+	const fetchMode = fetch as any as jest.Mock;
8	+	const { Response } = jest.requireActual('node-fetch');
9	+
10	+	beforeEach(() => {
11	+	fetchMode.mockClear();
12	+	});
13	+
14	+	describe('systemApi', () => {
15	+	it('initiateUrlProcessingInternal', async () => {
16	+	const url = 'http://example.com/' + Math.random().toString();
17	+	const requestId = 'test-req-id';
18	+
19	+	fetchMode.mockImplementation(() => Promise.resolve(new Response('', { status: 204 })));
20	+
21	+	const result = await requestWebPageScan(url, requestId);
22	+
23	+	expect(fetchMode).toBeCalledWith('https://api.test.gradejs.com/website/scan', {
24	+	method: 'POST',
25	+	body: `{"url":"${url}","requestId":"${requestId}"}`,
26	+	headers: {
27	+	'Content-Type': 'application/json',
28	+	'X-Api-Key': 'TEST_API_KEY',
29	+	},
30	+	});
31	+
32	+	expect(fetchMode).toHaveBeenCalledTimes(1);
33	+	expect(result).toMatchObject({});
34	+	});
35	+	});
36	+

■ ■ ■ ■ ■ ■

packages/shared/src/internalApi/api.ts packages/shared/src/systemApi/api.ts

1	1		import fetch, { RequestInit } from 'node-fetch';
2		-	import { getInternalApiOrigin } from '../utils/env';
	2	+	import { getGradeJsApiKey, getInternalApiRootUrl } from '../utils/env';
3	3
4	4		export type DetectedPackage = {
5	5		name: string;
6		-	possibleVersions: string[];
	6	+	versionSet: string[];
7	7		versionRange: string;
8		-	approximateSize: number \| null;
	8	+	approximateByteSize: number \| null;
9	9		};
10	10
11		-	export interface Website {
12		-	id: number;
13		-	url: string;
14		-	status: WebsiteStatus;
15		-	detectedPackages: DetectedPackage[];
16		-	updatedAt: string;
17		-	createdAt: string;
18		-	}
19		-
20		-	export enum WebsiteStatus {
21		-	Created = 'created',
22		-	InProgress = 'in-progress',
23		-	Ready = 'ready',
24		-	Failed = 'failed',
25		-	Invalid = 'invalid',
26		-	Protected = 'protected',
	11	+	export namespace WebPageScan {
	12	+	export enum Status {
	13	+	Created = 'created',
	14	+	InProgress = 'in-progress',
	15	+	Ready = 'ready',
	16	+	Failed = 'failed',
	17	+	Invalid = 'invalid',
	18	+	Protected = 'protected',
	19	+	}
27	20		}
28	21
29	22		export interface Package {
		skipped 13 lines
43	36		total: number;
44	37		};
45	38
46		-	export async function initiateUrlProcessing(url: string) {
47		-	return fetchEndpoint<Website>('POST', '/website/parse', { url });
48		-	}
49		-
50		-	export async function fetchUrlPackages(url: string) {
51		-	return fetchEndpoint<Website>('GET', '/website', { url });
	39	+	export async function requestWebPageScan(url: string, requestId: string) {
	40	+	return fetchEndpoint<{}>('POST', '/website/scan', { url, requestId });
52	41		}
53	42
54	43		export async function fetchPackageIndex(offset = 0, limit = 0) {
		skipped 12 lines
67	56		endpoint: string,
68	57		data?: Record<string, unknown>
69	58		) {
70		-	const requestUrl = new URL(endpoint, getInternalApiOrigin());
	59	+	const requestUrl = new URL(endpoint, getInternalApiRootUrl());
71	60		const requestInit: RequestInit = { method };
72	61
73	62		if (method === 'POST' \|\| method === 'PATCH') {
74		-	requestInit.headers = { 'Content-Type': 'application/json' };
	63	+	requestInit.headers = { 'Content-Type': 'application/json', 'X-Api-Key': getGradeJsApiKey() };
75	64		requestInit.body = JSON.stringify(data);
76	65		} else if (method === 'GET' && data) {
77	66		for (const key of Object.keys(data)) {
		skipped 1 lines
79	68		}
80	69		}
81	70
82		-	// console.log('Request to internal API: ', requestUrl.toString(), requestInit);
	71	+	return fetch(requestUrl.toString(), requestInit)
	72	+	.then((response) => {
	73	+	if (response.status !== 204) {
	74	+	return response.json();
	75	+	}
83	76
84		-	return fetch(requestUrl.toString(), requestInit)
85		-	.then((response) => response.json())
	77	+	return { data: {} };
	78	+	})
86	79		.then((json: any) => {
87	80		if (!json.data) {
88	81		throw new Error('Invalid response format');
		skipped 9 lines

■ ■ ■ ■ ■ ■

packages/shared/src/utils/env.ts

		skipped 20 lines
21	21		GitHubAccessToken = 'GITHUB_ACCESS_TOKEN',
22	22
23	23		// Internal
24		-	InternalApiOrigin = 'INTERNAL_API_ORIGIN',
	24	+	InternalApiRootUrl = 'INTERNAL_API_ROOT_URL',
	25	+	GradeJsApiKey = 'GRADEJS_API_KEY',
25	26
26	27		CorsAllowedOrigin = 'CORS_ALLOWED_ORIGIN',
27	28		}
		skipped 12 lines
40	41		export const isDevelopment = () => getNodeEnv() === 'development';
41	42		export const isTest = () => getNodeEnv() === 'test';
42	43
43		-	export const getInternalApiOrigin = () => getEnv(Env.InternalApiOrigin);
	44	+	export const getInternalApiRootUrl = () => {
	45	+	if (isTest()) {
	46	+	return getEnvUnsafe(Env.InternalApiRootUrl) ?? 'https://api.test.gradejs.com/';
	47	+	}
	48	+
	49	+	return getEnv(Env.InternalApiRootUrl);
	50	+	};
	51	+	export const getGradeJsApiKey = () => {
	52	+	if (isTest()) {
	53	+	return getEnvUnsafe(Env.GradeJsApiKey) ?? 'TEST_API_KEY';
	54	+	}
	55	+
	56	+	return getEnv(Env.GradeJsApiKey);
	57	+	};
	58	+
44	59		export const getSqsLocalPort = () => Number(getEnv(Env.SqsLocalPort, '0'));
45	60
46	61		export const getGitHubAccessToken = () => getEnv(Env.GitHubAccessToken);
		skipped 44 lines

■ ■ ■ ■ ■ ■

packages/shared/src/worker/types.ts

1		-	import * as internalApi from '../internalApi/api';
	1	+	import * as internalApi from '../systemApi/api';
2	2
3	3		export type WorkerTask = {
4	4		[Key in WorkerTaskType]: {
		skipped 13 lines

■ ■ ■ ■ ■ ■

packages/test-utils/src/index.ts

		skipped 51 lines
52	52		async (runInTransaction: (entityManager: EntityManager) => Promise<unknown>) => {
53	53		const em = connection.createEntityManager();
54	54		if (connection.createQueryRunner().isTransactionActive) {
55		-	await runInTransaction(em);
	55	+	return await runInTransaction(em);
56	56		} else {
57		-	await connection.transaction(runInTransaction);
	57	+	return await connection.transaction(runInTransaction);
58	58		}
59	59		}
60	60		);
		skipped 28 lines

■ ■ ■ ■ ■ ■

packages/web/src/components/layouts/Website/Website.stories.tsx

		skipped 11 lines
12	12		export const Ready = (args: Props) => <Website {...args} />;
13	13		Ready.args = {
14	14		host: 'gradejs.com',
15		-	webpages: [],
16	15		packages: [
17	16		{
18		-	packageName: 'react',
19		-	possiblePackageVersions: ['17.0.2'],
20		-	packageVersionRange: '17.0.2',
	17	+	name: 'react',
	18	+	versionSet: ['17.0.2'],
	19	+	versionRange: '17.0.2',
21	20		registryMetadata: {
22	21		latestVersion: '18.0.0',
23	22		description: 'Test description',
		skipped 3 lines
27	26		},
28	27		},
29	28		{
30		-	packageName: 'react-dom',
31		-	possiblePackageVersions: ['17.0.2'],
32		-	packageVersionRange: '17.0.2',
	29	+	name: 'react-dom',
	30	+	versionSet: ['17.0.2'],
	31	+	versionRange: '17.0.2',
33	32		},
34	33		{
35		-	packageName: 'react',
36		-	possiblePackageVersions: ['17.0.2'],
37		-	packageVersionRange: '17.0.2',
	34	+	name: 'react',
	35	+	versionSet: ['17.0.2'],
	36	+	versionRange: '17.0.2',
38	37		},
39	38		{
40		-	packageName: 'react-dom',
41		-	possiblePackageVersions: ['17.0.2'],
42		-	packageVersionRange: '17.0.2',
	39	+	name: 'react-dom',
	40	+	versionSet: ['17.0.2'],
	41	+	versionRange: '17.0.2',
43	42		},
44	43		{
45		-	packageName: 'react',
46		-	possiblePackageVersions: ['17.0.2'],
47		-	packageVersionRange: '17.0.2',
	44	+	name: 'react',
	45	+	versionSet: ['17.0.2'],
	46	+	versionRange: '17.0.2',
48	47		},
49	48		{
50		-	packageName: 'react-dom',
51		-	possiblePackageVersions: ['17.0.2'],
52		-	packageVersionRange: '17.0.2',
	49	+	name: 'react-dom',
	50	+	versionSet: ['17.0.2'],
	51	+	versionRange: '17.0.2',
53	52		},
54	53		{
55		-	packageName: 'react',
56		-	possiblePackageVersions: ['17.0.2'],
57		-	packageVersionRange: '17.0.2',
	54	+	name: 'react',
	55	+	versionSet: ['17.0.2'],
	56	+	versionRange: '17.0.2',
58	57		},
59	58		{
60		-	packageName: 'react-dom',
61		-	possiblePackageVersions: ['17.0.2'],
62		-	packageVersionRange: '17.0.2',
	59	+	name: 'react-dom',
	60	+	versionSet: ['17.0.2'],
	61	+	versionRange: '17.0.2',
63	62		},
64	63		{
65		-	packageName: 'react',
66		-	possiblePackageVersions: ['17.0.2'],
67		-	packageVersionRange: '17.0.2',
	64	+	name: 'react',
	65	+	versionSet: ['17.0.2'],
	66	+	versionRange: '17.0.2',
68	67		},
69	68		{
70		-	packageName: 'react-dom',
71		-	possiblePackageVersions: ['17.0.2'],
72		-	packageVersionRange: '17.0.2',
	69	+	name: 'react-dom',
	70	+	versionSet: ['17.0.2'],
	71	+	versionRange: '17.0.2',
73	72		},
74	73		],
75	74		vulnerabilities: {
		skipped 21 lines
97	96		export const Pending = (args: Props) => <Website {...args} />;
98	97		Pending.args = {
99	98		host: 'gradejs.com',
100		-	webpages: [
101		-	{
102		-	status: 'pending',
103		-	},
104		-	],
105	99		packages: [],
	100	+	isLoading: false,
	101	+	isPending: true,
106	102		};
107	103
108	104		export const Loading = (args: Props) => <Website {...args} />;
109	105		Loading.args = {
110	106		host: 'gradejs.com',
111		-	webpages: [],
112	107		packages: [],
	108	+	isLoading: true,
	109	+	isPending: false,
113	110		};
114	111

■ ■ ■ ■ ■ ■

packages/web/src/components/layouts/Website/Website.tsx

		skipped 6 lines
7	7		import Filters, { FiltersState } from '../Filters/Filters';
8	8		import TagBadge from '../../ui/TagBadge/TagBadge';
9	9		import { trackCustomEvent } from '../../../services/analytics';
10		-	import { Api } from '../../../services/apiClient';
	10	+	import { ClientApi } from '../../../services/apiClient';
11	11		import { Icon } from '../../ui/Icon/Icon';
12	12
13	13		// TODO: Add plashechka
		skipped 6 lines
20	20		// }>;
21	21		isLoading: boolean;
22	22		isPending: boolean;
23		-	packages: Api.WebPagePackage[];
24		-	vulnerabilities: Record<string, Api.Vulnerability[]>;
25		-	webpages: Api.WebPage[];
	23	+	packages: ClientApi.ScanResultPackageResponse[];
	24	+	vulnerabilities: Record<string, ClientApi.PackageVulnerabilityResponse[]>;
26	25		onFiltersApply: SubmitHandler<FiltersState>;
27	26		};
28	27
		skipped 2 lines
31	30		isLoading,
32	31		isPending,
33	32		packages,
34		-	webpages,
35	33		vulnerabilities,
36	34		onFiltersApply,
37	35		}: Props) {
		skipped 11 lines
49	47		It may take a few minutes and depends on the number of JavaScript files and their size.
50	48		</div>
51	49		) : (
52		-	webpages.length > 0 && (
	50	+	packages.length > 0 && (
53	51		<div className={styles.disclaimer}>
54	52		The <strong>beta</strong> version of GradeJS is able to detect only 1,826 popular
55	53		packages with up to 85% accuracy.
		skipped 64 lines
120	118		variant={view}
121	119		className={styles.package}
122	120		pkg={data}
123		-	vulnerabilities={vulnerabilities[data.packageName] \|\| []}
	121	+	vulnerabilities={vulnerabilities[data.name] \|\| []}
124	122		/>
125	123		))}
126	124		{isLoading && (
		skipped 21 lines

■ ■ ■ ■ ■ ■

packages/web/src/components/pages/WebsiteResults.tsx

		skipped 15 lines
16	16		const { hostname } = useParams();
17	17		const navigate = useNavigate();
18	18		const dispatch = useAppDispatch();
19		-	const { webpages, vulnerabilities } = useAppSelector(selectors.default);
	19	+	const { vulnerabilities } = useAppSelector(selectors.default);
20	20		const packagesFiltered = useAppSelector(selectors.packagesSortedAndFiltered);
21	21		const packagesStats = useAppSelector(selectors.packagesStats);
22	22		const { isProtected, isPending, isLoading, isFailed, isInvalid } = useAppSelector(
		skipped 8 lines
31	31		}
32	32
33	33		useEffect(() => {
34		-	if (hostname && !isLoading && isPending) {
	34	+	if (hostname && isPending && !isFailed) {
35	35		const promise = dispatch(getWebsite({ hostname }));
36	36		return function cleanup() {
37	37		promise.abort();
38	38		};
39	39		}
40	40		return () => {};
41		-	}, [hostname]);
	41	+	}, [hostname, isPending, isFailed]);
42	42
43	43		// TODO: properly handle history/routing
44	44		useEffect(() => {
		skipped 59 lines
104	104		<Website
105	105		isLoading={isLoading}
106	106		isPending={isPending}
107		-	webpages={webpages}
108		-	packages={packagesFiltered}
109		-	host={hostname ?? ''}
110		-	vulnerabilities={vulnerabilities}
	107	+	packages={packagesFiltered ?? []}
	108	+	host={hostname ?? ''}
	109	+	vulnerabilities={vulnerabilities ?? {}}
111	110		onFiltersApply={setFilters}
112	111		/>
113	112		</>
		skipped 3 lines

■ ■ ■ ■ ■ ■

packages/web/src/components/ui/Package/Package.tsx

		skipped 6 lines
7	7		import Vulnerability from '../Vulnerability/Vulnerability';
8	8		import TagBadge from '../TagBadge/TagBadge';
9	9		import { trackCustomEvent } from '../../../services/analytics';
10		-	import { Api } from '../../../services/apiClient';
	10	+	import { ClientApi } from '../../../services/apiClient';
11	11		import { Icon } from '../Icon/Icon';
12	12
13	13		export type Props = {
14	14		className?: string;
15	15		variant?: 'grid' \| 'lines';
16		-	pkg: Api.WebPagePackage;
17		-	vulnerabilities: Api.Vulnerability[];
	16	+	pkg: ClientApi.ScanResultPackageResponse;
	17	+	vulnerabilities: ClientApi.PackageVulnerabilityResponse[];
18	18		};
19	19
20	20		export default function Package({ className, variant = 'grid', pkg, vulnerabilities }: Props) {
21	21		const repositoryUrl = pkg.registryMetadata?.repositoryUrl;
22	22		const homepageUrl = pkg.registryMetadata?.homepageUrl;
23	23		const isOutdated =
24		-	pkg.registryMetadata && semver.gtr(pkg.registryMetadata.latestVersion, pkg.packageVersionRange);
	24	+	pkg.registryMetadata && semver.gtr(pkg.registryMetadata.latestVersion, pkg.versionRange);
25	25		const isVulnerable = !!vulnerabilities?.length;
26	26
27	27		return (
		skipped 53 lines
81	81		</div>
82	82		<a
83	83		className={styles.name}
84		-	href={`https://www.npmjs.com/package/${pkg.packageName}`}
	84	+	href={`https://www.npmjs.com/package/${pkg.name}`}
85	85		target='_blank'
86	86		rel='noopener noreferrer'
87		-	aria-label={pkg.packageName}
	87	+	aria-label={pkg.name}
88	88		// Browser won't break a line for '/' symbol, so we add the <wbr> specificaly
89	89		// eslint-disable-next-line react/no-danger
90		-	dangerouslySetInnerHTML={{ __html: pkg.packageName.replace('/', '/<wbr>') }}
	90	+	dangerouslySetInnerHTML={{ __html: pkg.name.replace('/', '/<wbr>') }}
91	91		onClick={() => trackCustomEvent('Package', 'ClickPackageUrl')}
92	92		/>
93	93		<div className={styles.meta}>
94		-	<div className={styles.version}>{toReadableVersion(pkg.packageVersionRange)}</div>
95		-	{!!pkg.packageMetadata?.approximateByteSize && (
96		-	<span className={styles.size}>
97		-	{toReadableSize(pkg.packageMetadata.approximateByteSize)}
98		-	</span>
	94	+	<div className={styles.version}>{toReadableVersion(pkg.versionRange)}</div>
	95	+	{!!pkg.approximateByteSize && (
	96	+	<span className={styles.size}>{toReadableSize(pkg.approximateByteSize)}</span>
99	97		)}
100	98		</div>
101	99		</div>
		skipped 25 lines

■ ■ ■ ■ ■ ■

packages/web/src/components/ui/Vulnerability/Vulnerability.tsx

		skipped 1 lines
2	2		import clsx from 'clsx';
3	3		import TagBadge from '../TagBadge/TagBadge';
4	4		import styles from './Vulnerability.module.scss';
5		-	import { Api } from '../../../services/apiClient';
	5	+	import { ClientApi } from '../../../services/apiClient';
6	6		import { Icon } from '../Icon/Icon';
7	7
8	8		export type Props = {
9		-	vulnerability: Api.Vulnerability;
	9	+	vulnerability: ClientApi.PackageVulnerabilityResponse;
10	10		};
11	11
12	12		export default function Vulnerability({ vulnerability }: Props) {
		skipped 55 lines

■ ■ ■ ■ ■ ■

packages/web/src/services/apiClient.ts

1	1		import { createTRPCClient } from '@trpc/client';
2	2		import type { inferProcedureOutput } from '@trpc/server';
3		-	import type { AppRouter, Api } from '../../../public-api/src/router';
	3	+	import type { ClientApiRouter, ClientApi } from '@gradejs-public/public-api/src/clientApiRouter';
4	4
5	5		// polyfill fetch, trpc needs it
6	6		import f from 'node-fetch';
7	7
8	8		// Helper types
9		-	export type TQuery = keyof AppRouter['_def']['queries'];
10		-	export type TMutation = keyof AppRouter['_def']['mutations'];
	9	+	export type TQuery = keyof ClientApiRouter['_def']['queries'];
	10	+	export type TMutation = keyof ClientApiRouter['_def']['mutations'];
11	11		export type InferMutationOutput<TRouteKey extends TMutation> = inferProcedureOutput<
12		-	AppRouter['_def']['mutations'][TRouteKey]
	12	+	ClientApiRouter['_def']['mutations'][TRouteKey]
13	13		>;
14	14		export type InferQueryOutput<TRouteKey extends TQuery> = inferProcedureOutput<
15		-	AppRouter['_def']['queries'][TRouteKey]
	15	+	ClientApiRouter['_def']['queries'][TRouteKey]
16	16		>;
17	17
18	18		if (!process.env.API_ORIGIN) {
19	19		throw new Error('API_ORIGIN must be defined');
20	20		}
21	21
22		-	export const client = createTRPCClient<AppRouter>({
23		-	url: process.env.API_ORIGIN,
	22	+	export const client = createTRPCClient<ClientApiRouter>({
	23	+	url: new URL('/client', process.env.API_ORIGIN).toString(),
24	24		fetch: typeof window === 'undefined' ? (f as any) : window.fetch.bind(window),
25	25		headers: {
26	26		Origin: process.env.CORS_ORIGIN,
27	27		},
28	28		});
29	29
30		-	export type SyncWebsiteOutput = InferMutationOutput<'syncWebsite'>;
31		-	export type RequestParseWebsiteOutput = InferMutationOutput<'requestParseWebsite'>;
32		-	export type { Api };
	30	+	export type RequestWebPageScanOutput = InferMutationOutput<'requestWebPageScan'>;
	31	+	export type { ClientApi };
33	32		export type ApiClient = typeof client;
34	33

■ ■ ■ ■ ■ ■

packages/web/src/store/selectors/websiteResults.ts

		skipped 4 lines
5	5		import { RootState } from '../';
6	6		import { FiltersState } from '../../components/layouts/Filters/Filters';
7	7		import { SeverityWeightMap } from '../../components/ui/Vulnerability/Vulnerability';
8		-	import type { Api } from '../../services/apiClient';
	8	+	import type { ClientApi } from '../../services/apiClient';
9	9
10	10		const getFlags = (state: RootState) => ({
11	11		isLoading: state.webpageResults.isLoading,
12	12		isFailed: state.webpageResults.isFailed,
13	13		});
14		-	const getPackages = (state: RootState) => state.webpageResults.detectionResult.packages;
15		-	const getWebpages = (state: RootState) => state.webpageResults.detectionResult.webpages;
	14	+	const getScanStatus = (state: RootState) => state.webpageResults.detectionResult?.status;
	15	+	const getPackages = (state: RootState) =>
	16	+	state.webpageResults.detectionResult?.scanResult?.packages;
16	17		const getVulnerabilities = (state: RootState) =>
17		-	state.webpageResults.detectionResult.vulnerabilities;
	18	+	state.webpageResults.detectionResult?.scanResult?.vulnerabilities;
18	19		const getSorting = (state: RootState) => state.webpageResults.filters.sort;
19	20		const getFilter = (state: RootState) => state.webpageResults.filters.filter;
20	21		const getPackageNameFilter = (state: RootState) => state.webpageResults.filters.filterPackageName;
21	22
22		-	const compareByPopularity = (left: Api.WebPagePackage, right: Api.WebPagePackage) =>
	23	+	const compareByPopularity = (
	24	+	left: ClientApi.ScanResultPackageResponse,
	25	+	right: ClientApi.ScanResultPackageResponse
	26	+	) =>
23	27		(right.registryMetadata?.monthlyDownloads ?? 0) - (left.registryMetadata?.monthlyDownloads ?? 0);
24	28
25	29		const pickHighestSeverity = memoize(
26		-	(packageName: string, vulnerabilities: Record<string, Api.Vulnerability[]>) =>
	30	+	(
	31	+	packageName: string,
	32	+	vulnerabilities: Record<string, ClientApi.PackageVulnerabilityResponse[]>
	33	+	) =>
27	34		(vulnerabilities[packageName] ?? [])
28	35		.map((it) => it.severity)
29	36		.filter((it): it is GithubAdvisorySeverity => !!it)
		skipped 6 lines
36	43		const sortingModes: Record<
37	44		FiltersState['sort'],
38	45		(
39		-	packages: Api.WebPagePackage[],
40		-	vulnerabilities: Record<string, Api.Vulnerability[]>
41		-	) => Api.WebPagePackage[]
	46	+	packages: ClientApi.ScanResultPackageResponse[],
	47	+	vulnerabilities: Record<string, ClientApi.PackageVulnerabilityResponse[]>
	48	+	) => ClientApi.ScanResultPackageResponse[]
42	49		> = {
43	50		// TODO
44	51		confidenceScore: (packages) => packages,
		skipped 1 lines
46	53		importDepth: (packages) => packages,
47	54		severity: (packages, vulnerabilities) =>
48	55		[...packages].sort((left, right) => {
49		-	const leftSeverity = pickHighestSeverity(left.packageName, vulnerabilities);
50		-	const rightSeverity = pickHighestSeverity(right.packageName, vulnerabilities);
	56	+	const leftSeverity = pickHighestSeverity(left.name, vulnerabilities);
	57	+	const rightSeverity = pickHighestSeverity(right.name, vulnerabilities);
51	58
52	59		if (leftSeverity !== rightSeverity) {
53	60		return SeverityWeightMap[rightSeverity] - SeverityWeightMap[leftSeverity];
		skipped 3 lines
57	64		}),
58	65		size: (packages) =>
59	66		[...packages].sort(
60		-	(left, right) =>
61		-	(right.packageMetadata?.approximateByteSize ?? 0) -
62		-	(left.packageMetadata?.approximateByteSize ?? 0)
	67	+	(left, right) => (right.approximateByteSize ?? 0) - (left.approximateByteSize ?? 0)
63	68		),
64		-	name: (packages) =>
65		-	[...packages].sort((left, right) => left.packageName.localeCompare(right.packageName)),
	69	+	name: (packages) => [...packages].sort((left, right) => left.name.localeCompare(right.name)),
66	70		packagePopularity: (packages) => [...packages].sort(compareByPopularity),
67	71		};
68	72
69	73		const filterModes: Record<
70	74		FiltersState['filter'],
71	75		(
72		-	packages: Api.WebPagePackage[],
73		-	vulnerabilities: Record<string, Api.Vulnerability[]>,
	76	+	packages: ClientApi.ScanResultPackageResponse[],
	77	+	vulnerabilities: Record<string, ClientApi.PackageVulnerabilityResponse[]>,
74	78		packageName?: string
75		-	) => Api.WebPagePackage[]
	79	+	) => ClientApi.ScanResultPackageResponse[]
76	80		> = {
77	81		name: (packages, vulnerabilities, packageName) => {
78	82		if (!packageName) {
79	83		return packages;
80	84		}
81		-	return packages.filter((pkg) => pkg.packageName.includes(packageName));
	85	+	return packages.filter((pkg) => pkg.name.includes(packageName));
82	86		},
83	87		outdated: (packages) =>
84	88		packages.filter(
85	89		(pkg) =>
86		-	pkg.registryMetadata &&
87		-	semver.gtr(pkg.registryMetadata.latestVersion, pkg.packageVersionRange)
	90	+	pkg.registryMetadata && semver.gtr(pkg.registryMetadata.latestVersion, pkg.versionRange)
88	91		),
89		-	vulnerable: (packages, vulnerabilities) =>
90		-	packages.filter((pkg) => !!vulnerabilities[pkg.packageName]),
	92	+	vulnerable: (packages, vulnerabilities) => packages.filter((pkg) => !!vulnerabilities[pkg.name]),
91	93		all: (packages) => packages,
92	94		};
93	95
94	96		export const selectors = {
95		-	default: createSelector([getWebpages, getVulnerabilities], (webpages, vulnerabilities) => ({
96		-	webpages,
	97	+	default: createSelector([getScanStatus, getVulnerabilities], (scanStatus, vulnerabilities) => ({
	98	+	status: scanStatus,
97	99		vulnerabilities,
98	100		})),
99		-	stateFlags: createSelector([getWebpages, getPackages, getFlags], (webpages, packages, flags) => ({
100		-	...flags,
101		-	isInvalid:
102		-	packages.length === 0 &&
103		-	webpages.length > 0 &&
104		-	webpages.some((item) => item.status === 'pending'),
105		-	isPending: webpages.length === 0 \|\| webpages.some((item) => item.status === 'pending'),
106		-	isProtected: webpages.some((item) => item.status === 'protected'),
107		-	})),
108		-	packagesStats: createSelector([getPackages, getVulnerabilities], (packages, vulnerabilities) => ({
	101	+	stateFlags: createSelector(
	102	+	[getScanStatus, getPackages, getFlags],
	103	+	(scanStatus, packages, flags) => ({
	104	+	...flags,
	105	+	isInvalid: packages && packages.length === 0,
	106	+	isPending: !scanStatus \|\| scanStatus === 'pending',
	107	+	isProtected: scanStatus === 'protected',
	108	+	})
	109	+	),
	110	+	packagesStats: createSelector([getPackages, getVulnerabilities], (packages = [], vulnerabilities = {}) => ({
109	111		total: packages.length,
110		-	vulnerable: packages.filter((pkg) => (vulnerabilities[pkg.packageName]?.length ?? 0) > 0)
	112	+	vulnerable: packages.filter((pkg) => (vulnerabilities[pkg.name]?.length ?? 0) > 0)
111	113		.length,
112	114		outdated: packages.filter(
113	115		(pkg) =>
114	116		pkg.registryMetadata &&
115		-	semver.gtr(pkg.registryMetadata.latestVersion, pkg.packageVersionRange)
	117	+	semver.gtr(pkg.registryMetadata.latestVersion, pkg.versionRange)
116	118		).length,
117	119		})),
118	120		packagesSortedAndFiltered: createSelector(
119	121		[getPackages, getVulnerabilities, getSorting, getFilter, getPackageNameFilter],
120	122		(packages, vulnerabilities, sorting, filter, packageNameFilter) =>
	123	+	packages &&
	124	+	vulnerabilities &&
121	125		filterModes[filter](
122	126		sortingModes[sorting](packages, vulnerabilities),
123	127		vulnerabilities,
		skipped 5 lines

■ ■ ■ ■ ■ ■

packages/web/src/store/slices/home.ts

		skipped 2 lines
3	3		import { RootState } from '../';
4	4
5	5		const parseWebsite = createAsyncThunk('home/submitWebsite', async (url: string) => {
6		-	await client.mutation('requestParseWebsite', url);
	6	+	if (!url.startsWith('http://') && !url.startsWith('https://')) {
	7	+	url = `https://${url}`;
	8	+	}
	9	+
	10	+	await client.mutation('requestWebPageScan', url);
7	11		});
8	12
9	13		const home = createSlice({
		skipped 34 lines

■ ■ ■ ■ ■ ■

packages/web/src/store/slices/websiteResults.ts

1	1		import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
2	2		import { DefaultFiltersAndSorters, FiltersState } from '../../components/layouts/Filters/Filters';
3		-	import { client, SyncWebsiteOutput } from '../../services/apiClient';
	3	+	import { client, RequestWebPageScanOutput } from '../../services/apiClient';
4	4		import { trackCustomEvent } from '../../services/analytics';
5	5
6		-	const defaultDetectionResult: SyncWebsiteOutput = {
7		-	packages: [],
8		-	vulnerabilities: {},
9		-	webpages: [],
	6	+	type WebsiteResultsState = {
	7	+	filters: typeof DefaultFiltersAndSorters;
	8	+	isFailed: boolean;
	9	+	isLoading: boolean;
	10	+	detectionResult?: RequestWebPageScanOutput;
10	11		};
11	12
12		-	const initialState = {
	13	+	const initialState: WebsiteResultsState = {
13	14		filters: { ...DefaultFiltersAndSorters },
14	15		isFailed: false,
15	16		isLoading: false,
16		-	detectionResult: defaultDetectionResult,
	17	+	detectionResult: undefined,
17	18		};
18	19
19	20		const sleep = (ms: number \| undefined) =>
		skipped 1 lines
21	22		setTimeout(r, ms);
22	23		});
23	24
24		-	const hasPendingPages = (result: DetectionResult) =>
25		-	!!result.webpages.find((item) => item.status === 'pending');
	25	+	const isScanPending = (result: DetectionResult) => result && result.status === 'pending';
26	26
27	27		const getWebsite = createAsyncThunk(
28	28		'websiteResults/getWebsite',
29	29		async ({ hostname, useRetry = true }: { hostname: string; useRetry?: boolean }) => {
	30	+	if (!hostname.startsWith('http://') && !hostname.startsWith('https://')) {
	31	+	hostname = `https://${hostname}`;
	32	+	}
	33	+
30	34		const loadStartTime = Date.now();
31		-	let results = await client.mutation('syncWebsite', hostname);
	35	+	let results = await client.mutation('requestWebPageScan', hostname);
32	36		if (useRetry) {
33		-	while (hasPendingPages(results)) {
	37	+	while (isScanPending(results)) {
34	38		await sleep(5000);
35		-	results = await client.mutation('syncWebsite', hostname);
	39	+	results = await client.mutation('requestWebPageScan', hostname);
36	40		}
37	41		}
38	42		// TODO: move to tracking middleware?
		skipped 20 lines
59	63		.addCase(getWebsite.pending, (state) => {
60	64		state.isLoading = true;
61	65		state.isFailed = false;
	66	+	state.detectionResult = undefined;
62	67		})
63	68		.addCase(getWebsite.fulfilled, (state, action) => {
64	69		state.isLoading = false;
		skipped 6 lines
71	76		},
72	77		});
73	78
74		-	export type DetectionResult = typeof defaultDetectionResult;
	79	+	export type DetectionResult = RequestWebPageScanOutput \| undefined;
75	80		export const { resetFilters, applyFilters } = websiteResults.actions;
76	81		export { getWebsite };
77	82		export const websiteResultsReducer = websiteResults.reducer;
		skipped 1 lines

■ ■ ■ ■ ■ ■

packages/worker/src/index.ts

		skipped 8 lines
9	9		checkRequiredEnvironmentVariables([
10	10		Env.AwsRegion,
11	11		Env.DatabaseUrl,
12		-	Env.InternalApiOrigin,
	12	+	Env.InternalApiRootUrl,
13	13		Env.SqsWorkerQueueUrl,
14	14		]);
15	15
		skipped 8 lines

feat: Data layer and flow rework (#53)