STRLCPY/maigret

Merge pull request #48 from soxoj/filter-fixes

Improved sites list filtering, pretty messages

soxoj committed with GitHub 4 years ago

de9a6135

2 parents
37d6247d
b4fb1003

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 5 files Show one by one

■ ■ ■ ■ ■ ■

maigret/maigret.py

		skipped 804 lines
805	805		if args.top_sites == 0 or args.all_sites:
806	806		args.top_sites = sys.maxsize
807	807
	808	+	# Create notify object for query results.
	809	+	query_notify = QueryNotifyPrint(result=None,
	810	+	verbose=args.verbose,
	811	+	print_found_only=not args.print_not_found,
	812	+	skip_check_errors=not args.print_check_errors,
	813	+	color=not args.no_color)
	814	+
808	815		# Create object with all information about sites we are aware of.
809		-	try:
810		-	db = MaigretDatabase().load_from_file(args.json_file)
811		-	site_data = db.ranked_sites_dict(top=args.top_sites, tags=args.tags, names=args.site_list)
812		-	except Exception as error:
813		-	print(f"ERROR: {error}")
814		-	sys.exit(1)
	816	+	db = MaigretDatabase().load_from_file(args.json_file)
	817	+	get_top_sites_for_id = lambda x: db.ranked_sites_dict(top=args.top_sites, tags=args.tags,
	818	+	names=args.site_list,
	819	+	disabled=False, id_type=x)
	820	+
	821	+	site_data = get_top_sites_for_id(args.id_type)
815	822
816	823		# Database self-checking
817	824		if args.self_check:
		skipped 14 lines
832	839		# Define one report filename template
833	840		report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}')
834	841
835		-	# Database consistency
836		-	enabled_count = len(list(filter(lambda x: not x.disabled, site_data.values())))
837		-	print(f'Sites in database, enabled/total: {enabled_count}/{len(site_data)}')
838		-
839		-	if not enabled_count:
840		-	print('No sites to check, exiting!')
841		-	sys.exit(2)
	842	+	# Database stats
	843	+	# TODO: verbose info about filtered sites
	844	+	# enabled_count = len(list(filter(lambda x: not x.disabled, site_data.values())))
	845	+	# print(f'Sites in database, enabled/total: {enabled_count}/{len(site_data)}')
842	846
843		-	if usernames == ['-']:
	847	+	if usernames == {}:
844	848		# magic params to exit after init
845		-	print('No usernames to check, exiting.')
	849	+	query_notify.warning('No usernames to check, exiting.')
846	850		sys.exit(0)
847	851
848		-	# Create notify object for query results.
849		-	query_notify = QueryNotifyPrint(result=None,
850		-	verbose=args.verbose,
851		-	print_found_only=not args.print_not_found,
852		-	skip_check_errors=not args.print_check_errors,
853		-	color=not args.no_color)
	852	+	if not site_data:
	853	+	query_notify.warning('No sites to check, exiting!')
	854	+	sys.exit(2)
	855	+	else:
	856	+	query_notify.warning(f'Starting a search on top {len(site_data)} sites from the Maigret database...')
	857	+	if not args.all_sites:
	858	+	query_notify.warning(f'You can run search by full list of sites with flag `-a`', '!')
854	859
855	860		already_checked = set()
856		-
857	861		general_results = []
858	862
859	863		while usernames:
		skipped 10 lines
870	874
871	875		if found_unsupported_chars:
872	876		pretty_chars_str = ','.join(map(lambda s: f'"{s}"', found_unsupported_chars))
873		-	print(f'Found unsupported URL characters: {pretty_chars_str}, skip search by username "{username}"')
	877	+	query_notify.warning(f'Found unsupported URL characters: {pretty_chars_str}, skip search by username "{username}"')
874	878		continue
875	879
	880	+	sites_to_check = get_top_sites_for_id(id_type)
	881	+
876	882		results = await maigret(username,
877		-	dict(site_data),
	883	+	dict(sites_to_check),
878	884		query_notify,
879	885		proxy=args.proxy,
880	886		timeout=args.timeout,
		skipped 24 lines
905	911		if args.xmind:
906	912		filename = report_filepath_tpl.format(username=username, postfix='.xmind')
907	913		save_xmind_report(filename, username, results)
908		-	print(f'XMind report for {username} saved in {filename}')
	914	+	query_notify.warning(f'XMind report for {username} saved in {filename}')
909	915
910	916		if args.csv:
911	917		filename = report_filepath_tpl.format(username=username, postfix='.csv')
912	918		save_csv_report(filename, username, results)
913		-	print(f'CSV report for {username} saved in {filename}')
	919	+	query_notify.warning(f'CSV report for {username} saved in {filename}')
914	920
915	921		if args.txt:
916	922		filename = report_filepath_tpl.format(username=username, postfix='.txt')
917	923		save_txt_report(filename, username, results)
918		-	print(f'TXT report for {username} saved in {filename}')
	924	+	query_notify.warning(f'TXT report for {username} saved in {filename}')
919	925
920	926		# reporting for all the result
921	927		if general_results:
922	928		if args.html or args.pdf:
923		-	print('Generating report info...')
	929	+	query_notify.warning('Generating report info...')
924	930		report_context = generate_report_context(general_results)
925	931		# determine main username
926	932		username = report_context['username']
		skipped 1 lines
928	934		if args.html:
929	935		filename = report_filepath_tpl.format(username=username, postfix='.html')
930	936		save_html_report(filename, report_context)
931		-	print(f'HTML report on all usernames saved in {filename}')
	937	+	query_notify.warning(f'HTML report on all usernames saved in {filename}')
932	938
933	939		if args.pdf:
934	940		filename = report_filepath_tpl.format(username=username, postfix='.pdf')
935	941		save_pdf_report(filename, report_context)
936		-	print(f'PDF report on all usernames saved in {filename}')
	942	+	query_notify.warning(f'PDF report on all usernames saved in {filename}')
937	943		# update database
938	944		db.save_to_file(args.json_file)
939	945
		skipped 11 lines

■ ■ ■ ■ ■ ■

maigret/notify.py

		skipped 167 lines
168	168		else:
169	169		print(f"[*] {title} {message} on:")
170	170
171		-	return
	171	+	def warning(self, message, symbol='-'):
	172	+	msg = f'[{symbol}] {message}'
	173	+	if self.color:
	174	+	print(Style.BRIGHT + Fore.YELLOW + msg)
	175	+	else:
	176	+	print(msg)
172	177
173	178		def get_additional_data_text(self, items, prepend=''):
174	179		text = ''
		skipped 113 lines

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 10371 lines
10372	10372		"us"
10373	10373		],
10374	10374		"checkType": "message",
10375		-	"absenceStrs": "The page you are looking for doesn\u2019t exist",
	10375	+	"presenseStrs": [
	10376	+	"{\"username\""
	10377	+	],
	10378	+	"absenceStrs": [
	10379	+	"We seem to have lost this page"
	10380	+	],
10376	10381		"alexaRank": 12727,
10377	10382		"url": "https://www.producthunt.com/@{username}",
10378	10383		"urlMain": "https://www.producthunt.com/",
		skipped 3183 lines
13562	13567		"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
13563	13568		"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
13564	13569		"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
13565		-	"x-guest-token": "1357438625504518145"
	13570	+	"x-guest-token": "1358064134064140290"
13566	13571		},
13567	13572		"errors": {
13568	13573		"Bad guest token": "x-guest-token update required"
		skipped 360 lines
13929	13934		"video"
13930	13935		],
13931	13936		"headers": {
13932		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTI0NzQ1MDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KmVN4YyuyqhUo8xr006lpL5k3_Uj2Y_ygk2r8cEO9Qo"
	13937	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTI2MjQ4NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.kgp8r380d1aDWcd-ROncr0Tqf8EdA-l35EeEY9is6TI"
13933	13938		},
13934	13939		"activation": {
13935	13940		"url": "https://vimeo.com/_rv/viewer",
		skipped 9142 lines

■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 139 lines
140	140		def sites_dict(self):
141	141		return {site.name: site for site in self._sites}
142	142
143		-	def ranked_sites_dict(self, reverse=False, top=sys.maxsize, tags=[], names=[]):
	143	+	def ranked_sites_dict(self, reverse=False, top=sys.maxsize, tags=[], names=[],
	144	+	disabled=True, id_type='username'):
	145	+	"""
	146	+	Ranking and filtering of the sites list
	147	+	"""
144	148		normalized_names = list(map(str.lower, names))
145	149		normalized_tags = list(map(str.lower, tags))
146	150
147		-	def is_tags_ok(site):
148		-	intersected_tags = set(site.tags).intersection(set(normalized_tags))
149		-	is_disabled = 'disabled' in tags and site.disabled
150		-	return intersected_tags or is_disabled
151		-
152	151		is_name_ok = lambda x: x.name.lower() in normalized_names
153	152		is_engine_ok = lambda x: isinstance(x.engine, str) and x.engine.lower() in normalized_tags
	153	+	is_tags_ok = lambda x: set(x.tags).intersection(set(normalized_tags))
	154	+	is_disabled_needed = lambda x: not x.disabled or ('disabled' in tags or disabled)
	155	+	is_id_type_ok = lambda x: x.type == id_type
154	156
155		-	if not tags and not names:
156		-	filtered_list = self.sites
157		-	else:
158		-	filtered_list = [s for s in self.sites if is_tags_ok(s) or is_name_ok(s) or is_engine_ok(s)]
	157	+	filter_tags_engines_fun = lambda x: not tags or is_engine_ok(x) or is_tags_ok(x)
	158	+	filter_names_fun = lambda x: not names or is_name_ok(x)
	159	+
	160	+	filter_fun = lambda x: filter_tags_engines_fun(x) and filter_names_fun(x) \
	161	+	and is_disabled_needed(x) and is_id_type_ok(x)
	162	+
	163	+	filtered_list = [s for s in self.sites if filter_fun(s)]
159	164
160	165		sorted_list = sorted(filtered_list, key=lambda x: x.alexa_rank, reverse=reverse)[:top]
161	166		return {site.name: site for site in sorted_list}
		skipped 123 lines

■ ■ ■ ■ ■ ■

tests/test_sites.py

		skipped 130 lines
131	131		# filtering by engine
132	132		assert list(db.ranked_sites_dict(tags=['ucoz']).keys()) == ['3']
133	133
	134	+	# disjunction
	135	+	assert list(db.ranked_sites_dict(names=['2'], tags=['forum']).keys()) == ['2']
	136	+	assert list(db.ranked_sites_dict(names=['2'], tags=['ucoz']).keys()) == []
	137	+	assert list(db.ranked_sites_dict(names=['4'], tags=['ru']).keys()) == []
	138	+
	139	+	# reverse
	140	+	assert list(db.ranked_sites_dict(reverse=True).keys()) == ['3', '2', '1']
	141	+
	142	+
	143	+	def test_ranked_sites_dict_names():
	144	+	db = MaigretDatabase()
	145	+	db.update_site(MaigretSite('3', {'alexaRank': 30}))
	146	+	db.update_site(MaigretSite('1', {'alexaRank': 2}))
	147	+	db.update_site(MaigretSite('2', {'alexaRank': 10}))
	148	+
134	149		# filtering by names
135	150		assert list(db.ranked_sites_dict(names=['1', '2']).keys()) == ['1', '2']
136	151		assert list(db.ranked_sites_dict(names=['2', '3']).keys()) == ['2', '3']
137	152
138		-	# disjunction
139		-	assert list(db.ranked_sites_dict(names=['2'], tags=['forum']).keys()) == ['1', '2']
140		-	assert list(db.ranked_sites_dict(names=['2'], tags=['forum'], reverse=True).keys()) == ['2', '1']
141		-	assert list(db.ranked_sites_dict(names=['2'], tags=['ucoz']).keys()) == ['2', '3']
142		-	assert list(db.ranked_sites_dict(names=['4'], tags=['ru']).keys()) == ['2']
143		-	assert list(db.ranked_sites_dict(names=['4'], tags=['nosuchtag']).keys()) == []
	153	+
	154	+	def test_ranked_sites_dict_disabled():
	155	+	db = MaigretDatabase()
	156	+	db.update_site(MaigretSite('1', {'disabled': True}))
	157	+	db.update_site(MaigretSite('2', {}))
	158	+
	159	+	assert len(db.ranked_sites_dict()) == 2
	160	+	assert len(db.ranked_sites_dict(disabled=False)) == 1
	161	+
	162	+	def test_ranked_sites_dict_id_type():
	163	+	db = MaigretDatabase()
	164	+	db.update_site(MaigretSite('1', {}))
	165	+	db.update_site(MaigretSite('2', {'type': 'username'}))
	166	+	db.update_site(MaigretSite('3', {'type': 'gaia_id'}))
	167	+
	168	+	assert len(db.ranked_sites_dict()) == 2
	169	+	assert len(db.ranked_sites_dict(id_type='username')) == 2
	170	+	assert len(db.ranked_sites_dict(id_type='gaia_id')) == 1
144	171