STRLCPY/opencti

[back] Improve connector screen by batching works and user fetching
Julien Richard committed 1 year ago

de2c3cc0

1 parent 3f3fdceb

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

opencti-platform/opencti-front/src/schema/relay.schema.graphql

		skipped 2847 lines
2848	2848		identity_class
2849	2849		name
2850	2850		aliases
	2851	+	x_opencti_aliases
2851	2852		confidence
2852	2853		created
2853	2854		modified
		skipped 960 lines
3814	3815		enum LocationsFilter {
3815	3816		entity_type
3816	3817		name
	3818	+	aliases
3817	3819		x_opencti_aliases
3818	3820		confidence
3819	3821		created
		skipped 4699 lines

■ ■ ■ ■ ■ ■

opencti-platform/opencti-graphql/src/domain/connector.js

		skipped 11 lines
12	12		import { ENTITY_TYPE_CONNECTOR, ENTITY_TYPE_SYNC, ENTITY_TYPE_WORK } from '../schema/internalObject';
13	13		import { FunctionalError, UnsupportedError } from '../config/errors';
14	14		import { now } from '../utils/format';
15		-	import { elLoadById } from '../database/engine';
	15	+	import { elFindByIds, elLoadById } from '../database/engine';
16	16		import { INTERNAL_SYNC_QUEUE, isEmptyField, READ_INDEX_HISTORY } from '../database/utils';
17	17		import { ABSTRACT_INTERNAL_OBJECT, CONNECTOR_INTERNAL_EXPORT_FILE } from '../schema/general';
18	18		import { SYSTEM_USER } from '../utils/access';
19		-	import { delEditContext, notify, setEditContext } from '../database/redis';
	19	+	import { delEditContext, notify, redisGetWork, setEditContext } from '../database/redis';
20	20		import { BUS_TOPICS, logApp } from '../config/conf';
21	21		import { deleteWorkForConnector } from './work';
22	22		import { listEntities } from '../database/middleware-loader';
		skipped 6 lines
29	29		const work = await elLoadById(context, user, id, ENTITY_TYPE_WORK, READ_INDEX_HISTORY);
30	30		if (work) return loadConnectorById(context, user, work.connector_id);
31	31		return null;
	32	+	};
	33	+
	34	+	export const batchConnectorForWork = async (context, user, workIds) => {
	35	+	const works = await elFindByIds(context, user, workIds, { toMap: true });
	36	+	return workIds.map((id) => works[id]);
	37	+	};
	38	+
	39	+	export const computeWorkStatus = async (work) => {
	40	+	if (work.status === 'complete') {
	41	+	return { import_processed_number: work.completed_number, import_expected_number: work.import_expected_number };
	42	+	}
	43	+	// If running, information in redis.
	44	+	const redisData = await redisGetWork(work.id);
	45	+	// If data in redis not exist, just send default values
	46	+	return redisData ?? { import_processed_number: null, import_expected_number: null };
32	47		};
33	48		export const connectorsForExport = async (context, user, scope, onlyAlive = false) => {
34	49		return connectorsFor(context, user, CONNECTOR_INTERNAL_EXPORT_FILE, scope, onlyAlive);
		skipped 163 lines

■ ■ ■ ■ ■ ■

opencti-platform/opencti-graphql/src/domain/work.js

1	1		import moment from 'moment';
2	2		import * as R from 'ramda';
3		-	import {
4		-	elDeleteInstances,
5		-	elIndex,
6		-	elLoadById,
7		-	elPaginate,
8		-	elUpdate,
9		-	} from '../database/engine';
	3	+	import { elDeleteInstances, elIndex, elLoadById, elPaginate, elUpdate, } from '../database/engine';
10	4		import { generateWorkId } from '../schema/identifier';
11	5		import { INDEX_HISTORY, isNotEmptyField, READ_INDEX_HISTORY } from '../database/utils';
12	6		import {
		skipped 230 lines

■ ■ ■ ■ ■ ■

opencti-platform/opencti-graphql/src/resolvers/connector.js

1	1		import {
	2	+	batchConnectorForWork,
	3	+	computeWorkStatus,
2	4		connectorDelete,
3	5		connectorForWork,
4	6		connectorsForExport,
5	7		findAllSync,
	8	+	findSyncById,
6	9		loadConnectorById,
7	10		patchSync,
8	11		pingConnector,
9	12		registerConnector,
10	13		registerSync,
11	14		resetStateConnector,
12		-	testSync,
	15	+	syncCleanContext,
13	16		syncDelete,
14		-	syncCleanContext,
15	17		syncEditContext,
16	18		syncEditField,
17		-	findSyncById,
	19	+	testSync,
18	20		} from '../domain/connector';
19	21		import {
20	22		createWork,
21	23		deleteWork,
	24	+	deleteWorkForConnector,
	25	+	findAll,
	26	+	findById,
	27	+	pingWork,
22	28		reportExpectation,
	29	+	updateExpectationsNumber,
23	30		updateProcessedTime,
24	31		updateReceivedTime,
25		-	worksForConnector,
26		-	findAll,
27		-	findById,
28		-	deleteWorkForConnector,
29		-	pingWork, updateExpectationsNumber,
30	32		} from '../domain/work';
31		-	import { findById as findUserById } from '../domain/user';
32		-	import { redisGetWork } from '../database/redis';
	33	+	import { batchUsers } from '../domain/user';
33	34		import { now } from '../utils/format';
34	35		import { connectors, connectorsForImport, connectorsForWorker } from '../database/repository';
	36	+	import { batchLoader } from '../database/middleware';
	37	+
	38	+	const usersLoader = batchLoader(batchUsers);
	39	+	const connectorWorkLoader = batchLoader(batchConnectorForWork);
35	40
36	41		const connectorResolvers = {
37	42		Query: {
		skipped 8 lines
46	51		synchronizers: (_, args, context) => findAllSync(context, context.user, args),
47	52		},
48	53		Connector: {
49		-	works: (connector, args, context) => worksForConnector(context, context.user, connector.id, args),
	54	+	works: (connector, args, context) => connectorWorkLoader.load(connector.id, context, context.user),
50	55		},
51	56		Work: {
52	57		connector: (work, _, context) => connectorForWork(context, context.user, work.id),
53		-	user: (work, _, context) => findUserById(context, context.user, work.user_id),
54		-	tracking: async (work) => {
55		-	// If complete, redis key is deleted, database contains all information
56		-	if (work.status === 'complete') {
57		-	return { import_processed_number: work.completed_number, import_expected_number: work.import_expected_number };
58		-	}
59		-	// If running, information in redis.
60		-	const redisData = await redisGetWork(work.id);
61		-	// If data in redis not exist, just send default values
62		-	if (redisData === undefined) {
63		-	return { import_processed_number: null, import_expected_number: null };
64		-	}
65		-	return redisData;
66		-	},
	58	+	user: (work, _, context) => usersLoader.load(work.user_id, context, context.user),
	59	+	tracking: (work) => computeWorkStatus(work),
67	60		},
68	61		Synchronizer: {
69		-	user: (sync, _, context) => findUserById(context, context.user, sync.user_id),
	62	+	user: (sync, _, context) => usersLoader.load(sync.user_id, context, context.user),
70	63		},
71	64		Mutation: {
72	65		deleteConnector: (_, { id }, context) => connectorDelete(context, context.user, id),
		skipped 33 lines

[back] Improve connector screen by batching works and user fetching