STRLCPY/LogonTracer

Added the option to import data to Elasticsearch
shusei tomonaga committed 4 years ago

ab1d5030

1 parent 97e0dc4e

■ ■ ■ ■ ■ ■

es-index/logontracer-host-index.json

1	+	{
2	+	"settings": {
3	+	"number_of_shards": 1
4	+	},
5	+	"mappings": {
6	+	"properties": {
7	+	"@timestamp": {
8	+	"type": "date",
9	+	"format": "strict_date_optional_time_nanos"
10	+	},
11	+	"IP": {
12	+	"type": "text"
13	+	},
14	+	"hostname": {
15	+	"type": "text"
16	+	},
17	+	"rank": {
18	+	"type": "double"
19	+	}
20	+	}
21	+	}
22	+	}
23	+

■ ■ ■ ■ ■ ■

es-index/logontracer-user-index.json

1	+	{
2	+	"settings": {
3	+	"number_of_shards": 1
4	+	},
5	+	"mappings": {
6	+	"properties": {
7	+	"@timestamp": {
8	+	"type": "date",
9	+	"format": "strict_date_optional_time_nanos"
10	+	},
11	+	"user": {
12	+	"type": "keyword"
13	+	},
14	+	"rights": {
15	+	"type": "keyword"
16	+	},
17	+	"sid": {
18	+	"type": "keyword"
19	+	},
20	+	"status": {
21	+	"type": "text"
22	+	},
23	+	"rank": {
24	+	"type": "double"
25	+	}
26	+	}
27	+	}
28	+	}
29	+

■ ■ ■ ■ ■ ■

logontracer.py

		skipped 204 lines
205	205		help="Elastic Search event object prefix. (default: winlog)")
206	206		parser.add_argument("--es", action="store_true", default=False,
207	207		help="Import data from Elastic Search. (default: False)")
	208	+	parser.add_argument("--postes", action="store_true", default=False,
	209	+	help="Post data to Elastic Search. (default: False)")
208	210		parser.add_argument("-s", "--server", dest="server", action="store", type=str, metavar="SERVER",
209	211		help="Neo4j server. (default: localhost)")
210	212		parser.add_argument("-u", "--user", dest="user", action="store", type=str, metavar="USERNAME",
		skipped 70 lines
281	283		CREATE (user)-[group:Policy]->(id) set group.date='{date}'
282	284
283	285		RETURN user, id
	286	+	"""
	287	+
	288	+	es_doc_user = """
	289	+	{{"@timestamp":"{datetime}", "user":"{user}", "rights":"{rights}", "sid":"{sid}", "status":"{status}", "rank":{rank}}}
	290	+	"""
	291	+
	292	+	es_doc_ip = """
	293	+	{{"@timestamp":"{datetime}", "IP":"{IP}", "hostname":"{hostname}", "rank":{rank}}}
284	294		"""
285	295
286	296		if args.user:
		skipped 275 lines
562	572		model.emissionprob_ = emission_probability
563	573		model.fit(np.array([data_array], dtype="int").T, lengths)
564	574		joblib.dump(model, FPATH + "/model/hmm.pkl")
	575	+
	576	+
	577	+	# Post to Elastic Search cluster
	578	+	def post_es(index, es, doc):
	579	+	es.index(index=index, body=doc)
	580	+
	581	+
	582	+	# Create mattings to Elastic Search
	583	+	def create_map(es, index):
	584	+	with open(FPATH + "/es-index/" + index + ".json", "r") as f:
	585	+	body = f.read()
	586	+	es.indices.create(index=index, body=body)
565	587
566	588
567	589		def to_lxml(record_xml):
		skipped 501 lines
1069	1091		except:
1070	1092		sys.exit("[!] Can't connect Neo4j Database.")
1071	1093
	1094	+	if args.postes:
	1095	+	# Parse Event log
	1096	+	print("[+] Start sending the ES.")
	1097	+
	1098	+	# Create a new ES client
	1099	+	client = Elasticsearch(ES_SERVER)
	1100	+
	1101	+	if client.indices.exists(index="logontracer-user-index") and client.indices.exists(index="logontracer-host-index") :
	1102	+	print("[+] Already created index mappings to ES.")
	1103	+	else:
	1104	+	create_map(client, "logontracer-host-index")
	1105	+	create_map(client, "logontracer-user-index")
	1106	+	print("[+] Creating index mappings to ES.")
	1107	+
	1108	+	es_timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
	1109	+
1072	1110		tx = GRAPH.begin()
1073	1111		hosts_inv = {v: k for k, v in hosts.items()}
1074	1112		for ipaddress in event_set["ipaddress"].drop_duplicates():
		skipped 3 lines
1078	1116		hostname = ipaddress
1079	1117		# add the IPAddress node to neo4j
1080	1118		tx.run(statement_ip.format(**{"IP": ipaddress, "rank": ranks[ipaddress], "hostname": hostname}))
	1119	+
	1120	+	# add host data to Elasticsearch
	1121	+	if args.postes:
	1122	+	es_doc = es_doc_ip.format(**{"datetime": es_timestamp, "IP": ipaddress, "rank": ranks[ipaddress], "hostname": hostname})
	1123	+	post_es("logontracer-host-index", client, es_doc)
1081	1124
1082	1125		i = 0
1083	1126		for username in username_set:
		skipped 28 lines
1112	1155		"counts4769": ",".join(map(str, timelines[i6+4])), "counts4776": ",".join(map(str, timelines[i6+5])),
1113	1156		"detect": ",".join(map(str, detects[i]))}))
1114	1157		i += 1
	1158	+
	1159	+	# add user data to Elasticsearch
	1160	+	if args.postes:
	1161	+	es_doc = es_doc_user.format(**{"datetime": es_timestamp, "user": username[:-1], "rights": rights, "sid": sid, "status": ustatus, "rank": ranks[username]})
	1162	+	post_es("logontracer-user-index", client, es_doc)
1115	1163
1116	1164		for domain in domains:
1117	1165		# add the domain node to neo4j
		skipped 421 lines
1539	1587		except:
1540	1588		sys.exit("[!] Can't connect Neo4j Database.")
1541	1589
	1590	+	if args.postes:
	1591	+	# Parse Event log
	1592	+	print("[+] Start sending the ES.")
	1593	+
	1594	+	if client.indices.exists(index="logontracer-user-index") and client.indices.exists(index="logontracer-host-index") :
	1595	+	print("[+] Already created index mappings to ES.")
	1596	+	else:
	1597	+	create_map(client, "logontracer-host-index")
	1598	+	create_map(client, "logontracer-user-index")
	1599	+	print("[+] Creating index mappings to ES.")
	1600	+
	1601	+	es_timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
	1602	+
1542	1603		tx = GRAPH.begin()
1543	1604		hosts_inv = {v: k for k, v in hosts.items()}
1544	1605		for ipaddress in event_set["ipaddress"].drop_duplicates():
		skipped 3 lines
1548	1609		hostname = ipaddress
1549	1610		# add the IPAddress node to neo4j
1550	1611		tx.run(statement_ip.format(**{"IP": ipaddress, "rank": ranks[ipaddress], "hostname": hostname}))
	1612	+
	1613	+	# add host data to Elasticsearch
	1614	+	if args.postes:
	1615	+	es_doc = es_doc_ip.format(**{"datetime": es_timestamp, "IP": ipaddress, "rank": ranks[ipaddress], "hostname": hostname})
	1616	+	post_es("logontracer-host-index", client, es_doc)
1551	1617
1552	1618		i = 0
1553	1619		for username in username_set:
		skipped 28 lines
1582	1648		"counts4769": ",".join(map(str, timelines[i6+4])), "counts4776": ",".join(map(str, timelines[i6+5])),
1583	1649		"detect": ",".join(map(str, detects[i]))}))
1584	1650		i += 1
	1651	+
	1652	+	# add user data to Elasticsearch
	1653	+	if args.postes:
	1654	+	es_doc = es_doc_user.format(**{"datetime": es_timestamp, "user": username[:-1], "rights": rights, "sid": sid, "status": ustatus, "rank": ranks[username]})
	1655	+	post_es("logontracer-user-index", client, es_doc)
1585	1656
1586	1657		for domain in domains:
1587	1658		# add the domain node to neo4j
		skipped 120 lines

Added the option to import data to Elasticsearch