STRLCPY/maigret

■ ■ ■ ■ ■ ■

CHANGELOG.md

		skipped 1 lines
2	2
3	3		## [Unreleased]
4	4
	5	+	## [0.3.1] - 2021-10-31
	6	+	* fixed false positives
	7	+	* accelerated maigret start time by 3 times
	8	+
5	9		## [0.3.0] - 2021-06-02
6	10		* added support of Tor and I2P sites
7	11		* added experimental DNS checking feature
		skipped 114 lines

■ ■ ■ ■ ■ ■

maigret/__version__.py

1	1		"""Maigret version file"""
2	2
3		-	__version__ = '0.3.0'
	3	+	__version__ = '0.3.1'
4	4

■ ■ ■ ■ ■ ■

maigret/checking.py

		skipped 12 lines
13	13		from typing import Tuple, Optional, Dict, List
14	14		from urllib.parse import quote
15	15
16		-	import aiohttp
17	16		import aiodns
18	17		import tqdm.asyncio
19		-	from aiohttp_socks import ProxyConnector
20	18		from python_socks import _errors as proxy_errors
21	19		from socid_extractor import extract
	20	+	from aiohttp import TCPConnector, ClientSession, http_exceptions
22	21		from aiohttp.client_exceptions import ServerDisconnectedError, ClientConnectorError
23	22
24	23		from .activation import ParsingActivator, import_aiohttp_cookies
		skipped 35 lines
60	59		cookie_jar = kwargs.get('cookie_jar')
61	60		self.logger = kwargs.get('logger', Mock())
62	61
	62	+	# moved here to speed up the launch of Maigret
	63	+	from aiohttp_socks import ProxyConnector
	64	+
63	65		# make http client session
64	66		connector = (
65		-	ProxyConnector.from_url(proxy) if proxy else aiohttp.TCPConnector(ssl=False)
	67	+	ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False)
66	68		)
67	69		connector.verify_ssl = False
68		-	self.session = aiohttp.ClientSession(
	70	+	self.session = ClientSession(
69	71		connector=connector, trust_env=True, cookie_jar=cookie_jar
70	72		)
71	73
		skipped 41 lines
113	115		error = CheckError("Connecting failure", str(e))
114	116		except ServerDisconnectedError as e:
115	117		error = CheckError("Server disconnected", str(e))
116		-	except aiohttp.http_exceptions.BadHttpMessage as e:
	118	+	except http_exceptions.BadHttpMessage as e:
117	119		error = CheckError("HTTP", str(e))
118	120		except proxy_errors.ProxyError as e:
119	121		error = CheckError("Proxy", str(e))
		skipped 19 lines
139	141		cookie_jar = kwargs.get('cookie_jar')
140	142		self.logger = kwargs.get('logger', Mock())
141	143
	144	+	# moved here to speed up the launch of Maigret
	145	+	from aiohttp_socks import ProxyConnector
	146	+
142	147		connector = ProxyConnector.from_url(proxy)
143	148		connector.verify_ssl = False
144		-	self.session = aiohttp.ClientSession(
	149	+	self.session = ClientSession(
145	150		connector=connector, trust_env=True, cookie_jar=cookie_jar
146	151		)
147	152
		skipped 735 lines

■ ■ ■ ■ ■ ■

maigret/maigret.py

1	1		"""
2	2		Maigret main module
3	3		"""
4		-	import aiohttp
5	4		import asyncio
6	5		import logging
7	6		import os
		skipped 2 lines
10	9		from argparse import ArgumentParser, RawDescriptionHelpFormatter
11	10		from typing import List, Tuple
12	11
13		-	import requests
14		-	from socid_extractor import extract, parse, __version__ as socid_version
	12	+	from socid_extractor import extract, parse
15	13
16	14		from .__version__ import __version__
17	15		from .checking import (
		skipped 96 lines
114	112
115	113
116	114		def setup_arguments_parser():
	115	+	from aiohttp import __version__ as aiohttp_version
	116	+	from requests import __version__ as requests_version
	117	+	from socid_extractor import __version__ as socid_version
	118	+
117	119		version_string = '\n'.join(
118	120		[
119	121		f'%(prog)s {__version__}',
120	122		f'Socid-extractor: {socid_version}',
121		-	f'Aiohttp: {aiohttp.__version__}',
122		-	f'Requests: {requests.__version__}',
	123	+	f'Aiohttp: {aiohttp_version}',
	124	+	f'Requests: {requests_version}',
123	125		f'Python: {platform.python_version()}',
124	126		]
125	127		)
		skipped 594 lines

■ ■ ■ ■ ■ ■

maigret/report.py

		skipped 6 lines
7	7		from datetime import datetime
8	8		from typing import Dict, Any
9	9
10		-	import pycountry
11	10		import xmind
12	11		from dateutil.parser import parse as parse_datetime_str
13	12		from jinja2 import Template
14		-	from xhtml2pdf import pisa
15		-	from pyvis.network import Network
16		-	import networkx as nx
17	13
18	14		from .checking import SUPPORTED_IDS
19	15		from .result import QueryStatus
		skipped 58 lines
78	74		def save_pdf_report(filename: str, context: dict):
79	75		template, css = generate_report_template(is_pdf=True)
80	76		filled_template = template.render(**context)
	77	+
	78	+	# moved here to speed up the launch of Maigret
	79	+	from xhtml2pdf import pisa
	80	+
81	81		with open(filename, "w+b") as f:
82	82		pisa.pisaDocument(io.StringIO(filled_template), dest=f, default_css=css)
83	83
		skipped 33 lines
117	117
118	118
119	119		def save_graph_report(filename: str, username_results: list, db: MaigretDatabase):
	120	+	# moved here to speed up the launch of Maigret
	121	+	import networkx as nx
	122	+
120	123		G = nx.Graph()
121	124		graph = MaigretGraph(G)
122	125
		skipped 78 lines
201	204
202	205		[G.remove_node(node) for node in nodes_to_remove]
203	206
	207	+	# moved here to speed up the launch of Maigret
	208	+	from pyvis.network import Network
	209	+
204	210		nt = Network(notebook=True, height="750px", width="100%")
205	211		nt.from_nx(G)
206	212		nt.show(filename)
		skipped 46 lines
253	259		supposed_data: Dict[str, Any] = {}
254	260
255	261		first_seen = None
	262	+
	263	+	# moved here to speed up the launch of Maigret
	264	+	import pycountry
256	265
257	266		for username, id_type, results in username_results:
258	267		found_accounts = 0
		skipped 265 lines

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 1158 lines
1159	1159		],
1160	1160		"checkType": "message",
1161	1161		"absenceStrs": [
1162		-	"does not exist"
	1162	+	"does not exist",
	1163	+	"This user has not filled out their profile page yet."
1163	1164		],
1164	1165		"alexaRank": 80,
1165	1166		"urlMain": "https://armchairgm.fandom.com/",
		skipped 873 lines
2039	2040		"ru",
2040	2041		"wiki"
2041	2042		],
2042		-	"checkType": "status_code",
	2043	+	"checkType": "message",
	2044	+	"absenceStrs": [
	2045	+	"does not exist",
	2046	+	"\u042d\u0442\u043e\u0442 \u0443\u0447\u0430\u0441\u0442\u043d\u0438\u043a \u043f\u043e\u043a\u0430 \u043d\u0435 \u0437\u0430\u043f\u043e\u043b\u043d\u0438\u043b \u0441\u0432\u043e\u0439 \u043f\u0440\u043e\u0444\u0438\u043b\u044c."
	2047	+	],
2043	2048		"alexaRank": 80,
2044	2049		"urlMain": "https://bleach.fandom.com/ru",
2045	2050		"url": "https://bleach.fandom.com/ru/wiki/%D0%A3%D1%87%D0%B0%D1%81%D1%82%D0%BD%D0%B8%D0%BA:{username}",
		skipped 2413 lines
4459	4464		],
4460	4465		"checkType": "message",
4461	4466		"absenceStrs": [
4462		-	"does not exist"
	4467	+	"does not exist",
	4468	+	"This user has not filled out their profile page yet."
4463	4469		],
4464	4470		"alexaRank": 80,
4465	4471		"urlMain": "https://community.fandom.com",
		skipped 4254 lines
8720	8726		"usernameUnclaimed": "noonewouldeverusethis7"
8721	8727		},
8722	8728		"Metacafe": {
	8729	+	"disabled": true,
8723	8730		"tags": [
8724	8731		"in",
8725	8732		"us"
		skipped 4345 lines
13071	13078		"us"
13072	13079		],
13073	13080		"headers": {
13074		-	"authorization": "Bearer BQBtoXAQab7ErdN63dPUer1RqjDLcX2v54xE9AfGPwYnncjU8HS5PlFW5mJE2cgDEDImvT07Xcpjb_ggsww"
	13081	+	"authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4"
13075	13082		},
13076	13083		"errors": {
13077	13084		"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
		skipped 1828 lines
14906	14913		"video"
14907	14914		],
14908	14915		"headers": {
14909		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjU0MjEzMDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.-F7S6fx7mold8Qhve4N3GjIv2Ue8RIaej4kXQUMBxpE"
	14916	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc"
14910	14917		},
14911	14918		"activation": {
14912	14919		"url": "https://vimeo.com/_rv/viewer",
		skipped 4299 lines
19212	19219		"tags": [
19213	19220		"cn"
19214	19221		],
19215		-	"checkType": "status_code",
	19222	+	"checkType": "message",
	19223	+	"absenceStrs": [
	19224	+	"message\":\"Not Found\""
	19225	+	],
	19226	+	"presenseStrs": [
	19227	+	"- SegmentFault \u601d\u5426</title>"
	19228	+	],
19216	19229		"alexaRank": 2697,
19217	19230		"urlMain": "https://segmentfault.com/",
19218	19231		"url": "https://segmentfault.com/u/{username}",
19219		-	"usernameClaimed": "bule",
	19232	+	"usernameClaimed": "john",
19220	19233		"usernameUnclaimed": "noonewouldeverusethis7"
19221	19234		},
19222	19235		"shadow-belgorod.ucoz.ru": {
		skipped 9439 lines

■ ■ ■ ■ ■ ■

setup.py

		skipped 10 lines
11	11		requires = rf.read().splitlines()
12	12
13	13		setup(name='maigret',
14		-	version='0.3.0',
	14	+	version='0.3.1',
15	15		description='Collect a dossier on a person by username from a huge number of sites',
16	16		long_description=long_description,
17	17		long_description_content_type="text/markdown",
		skipped 10 lines

Merge pull request #188 from soxoj/fp-fixes-speed-up