STRLCPY/maigret

Merge pull request #158 from soxoj/dns-resolving
```
Added DNS checker
```
soxoj committed with GitHub 3 years ago

8de1830c

2 parents
4a5c5c3f
ba616965

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■ ■

maigret/checking.py

		skipped 8 lines
9	9		from urllib.parse import quote
10	10
11	11		import aiohttp
	12	+	import aiodns
12	13		import tqdm.asyncio
13	14		from aiohttp_socks import ProxyConnector
14	15		from python_socks import _errors as proxy_errors
		skipped 28 lines
43	44		BAD_CHARS = "#"
44	45
45	46
46		-	class SimpleAiohttpChecker:
	47	+	class CheckerBase:
	48	+	pass
	49	+
	50	+
	51	+	class SimpleAiohttpChecker(CheckerBase):
47	52		def __init__(self, args, *kwargs):
48	53		proxy = kwargs.get('proxy')
49	54		cookie_jar = kwargs.get('cookie_jar')
		skipped 85 lines
135	140		)
136	141
137	142
	143	+	class AiodnsDomainResolver(CheckerBase):
	144	+	def __init__(self, args, *kwargs):
	145	+	loop = asyncio.get_event_loop()
	146	+	self.logger = kwargs.get('logger', Mock())
	147	+	self.resolver = aiodns.DNSResolver(loop=loop)
	148	+
	149	+	def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
	150	+	return self.resolver.query(url, 'A')
	151	+
	152	+	async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
	153	+	status = 404
	154	+	error = None
	155	+	text = ''
	156	+
	157	+	try:
	158	+	res = await future
	159	+	text = str(res[0].host)
	160	+	status = 200
	161	+	except aiodns.error.DNSError:
	162	+	pass
	163	+	except Exception as e:
	164	+	self.logger.error(e, exc_info=True)
	165	+	error = CheckError('DNS resolve error', str(e))
	166	+
	167	+	return text, status, error
	168	+
	169	+
	170	+	class CheckerMock:
	171	+	def __init__(self, args, *kwargs):
	172	+	pass
	173	+
	174	+	def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'):
	175	+	return None
	176	+
	177	+	async def check(self, future) -> Tuple[str, int, Optional[CheckError]]:
	178	+	await asyncio.sleep(0)
	179	+	return '', 0, None
	180	+
	181	+	async def close(self):
	182	+	return
	183	+
	184	+
138	185		# TODO: move to separate class
139	186		def detect_error_page(
140	187		html_text, status_code, fail_flags, ignore_403
		skipped 229 lines
370	417		url = re.sub("(?<!:)/+", "/", url)
371	418
372	419		# always clearweb_checker for now
373		-	checker = options["checkers"][site.network]
	420	+	checker = options["checkers"][site.protocol]
374	421
375	422		# site check is disabled
376	423		if site.disabled and not options['forced']:
		skipped 141 lines
518	565		no_progressbar=False,
519	566		cookies=None,
520	567		retries=0,
	568	+	check_domains=False,
521	569		) -> QueryResultWrapper:
522	570		"""Main search func
523	571
		skipped 47 lines
571	619		)
572	620
573	621		# TODO
574		-	tor_checker = Mock()
	622	+	tor_checker = CheckerMock()
575	623		if tor_proxy:
576	624		tor_checker = TorAiohttpChecker( # type: ignore
577	625		proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger
578	626		)
	627	+
	628	+	# TODO
	629	+	dns_checker = CheckerMock()
	630	+	if check_domains:
	631	+	dns_checker = AiodnsDomainResolver(logger=logger) # type: ignore
579	632
580	633		if logger.level == logging.DEBUG:
581	634		await debug_ip_request(clearweb_checker, logger)
		skipped 13 lines
595	648		options["checkers"] = {
596	649		'': clearweb_checker,
597	650		'tor': tor_checker,
	651	+	'dns': dns_checker,
598	652		}
599	653		options["parsing"] = is_parsing_enabled
600	654		options["timeout"] = timeout
		skipped 193 lines

■ ■ ■ ■ ■ ■

maigret/maigret.py

		skipped 244 lines
245	245		default='socks5://127.0.0.1:9050',
246	246		help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050",
247	247		)
	248	+	parser.add_argument(
	249	+	"--with-domains",
	250	+	action="store_true",
	251	+	default=False,
	252	+	help="Enable (experimental) feature of checking domains on usernames.",
	253	+	)
248	254
249	255		filter_group = parser.add_argument_group(
250	256		'Site filtering', 'Options to set site search scope'
		skipped 351 lines
602	608		max_connections=args.connections,
603	609		no_progressbar=args.no_progressbar,
604	610		retries=args.retries,
	611	+	check_domains=args.with_domains,
605	612		)
606	613
607	614		notify_about_errors(results, query_notify)
		skipped 72 lines

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 27748 lines
27749	27749		"tags": [
27750	27750		"tor"
27751	27751		],
27752		-	"network": "tor",
	27752	+	"protocol": "tor",
27753	27753		"url": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion/user/{username}",
27754	27754		"urlMain": "http://answerszuvs3gg2l64e6hmnryudl5zgrmwm3vh65hzszdghblddvfiqd.onion",
27755	27755		"usernameClaimed": "theredqueen",
		skipped 5 lines
27761	27761		"presenseStrs": [
27762	27762		"qa-part-form-profile"
27763	27763		]
	27764	+	},
	27765	+	".com": {
	27766	+	"protocol": "dns",
	27767	+	"url": "{username}.com",
	27768	+	"urlMain": "{username}.com",
	27769	+	"usernameClaimed": "soxoj",
	27770	+	"usernameUnclaimed": "noonewouldeverusethis7",
	27771	+	"checkType": "status_code"
27764	27772		}
27765	27773		},
27766	27774		"engines": {
		skipped 205 lines

■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 122 lines
123	123		alexa_rank = None
124	124		source = None
125	125
126		-	network = ''
	126	+	protocol = ''
127	127
128	128		def __init__(self, name, information):
129	129		self.name = name
		skipped 360 lines

■ ■ ■ ■ ■ ■

requirements.txt

1 + aiodns==3.0.0
1 2 aiohttp==3.7.4
2 3 aiohttp-socks==0.5.5
3 4 arabic-reshaper==2.1.1
skipped 36 lines

All occurrences

■ ■ ■ ■ ■ ■

tests/test_cli.py

		skipped 36 lines
37	37		'use_disabled_sites': False,
38	38		'username': [],
39	39		'verbose': False,
	40	+	'with_domains': False,
40	41		'xmind': False,
41	42		}
42	43
		skipped 53 lines

	1	+	aiodns==3.0.0
1	2		aiohttp==3.7.4
2	3		aiohttp-socks==0.5.5
3	4		arabic-reshaper==2.1.1
		skipped 36 lines

Merge pull request #158 from soxoj/dns-resolving