STRLCPY/maigret

Merge pull request #93 from soxoj/docs-requirements
```
Documentation and API improving
```
soxoj committed with GitHub 4 years ago

69c315b0

2 parents
7490a412
b755628a

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

CHANGELOG.md

1	1		# Changelog
2	2
3	3		## [Unreleased]
	4	+	* simplified maigret search API
	5	+	* improved documentation
	6	+	* fixed 403 response code ignoring bug
4	7
5	8		## [0.1.16] - 2021-03-21
6	9		* improved URL parsing mode
		skipped 70 lines

■ ■ ■ ■ ■ ■

maigret/__init__.py

1 1 """Maigret"""
2 2
3 + from .checking import maigret as search
4 +

All occurrences

■ ■ ■ ■ ■ ■

maigret/checking.py

1	1		import asyncio
2	2		import logging
	3	+	from mock import Mock
3	4		import re
4	5		import ssl
5	6		import sys
		skipped 388 lines
394	395		return results_info
395	396
396	397
397		-	async def maigret(username, site_dict, query_notify, logger,
	398	+	async def maigret(username, site_dict, logger, query_notify=None,
398	399		proxy=None, timeout=None, is_parsing_enabled=False,
399	400		id_type='username', debug=False, forced=False,
400	401		max_connections=100, no_progressbar=False,
401	402		cookies=None):
402	403		"""Main search func
403	404
404		-	Checks for existence of username on various social media sites.
	405	+	Checks for existence of username on certain sites.
405	406
406	407		Keyword Arguments:
407		-	username -- String indicating username that report
408		-	should be created against.
409		-	site_dict -- Dictionary containing all of the site data.
	408	+	username -- Username string will be used for search.
	409	+	site_dict -- Dictionary containing sites data.
410	410		query_notify -- Object with base type of QueryNotify().
411	411		This will be used to notify the caller about
412	412		query results.
413		-	proxy -- String indicating the proxy URL
	413	+	logger -- Standard Python logger object.
414	414		timeout -- Time in seconds to wait before timing out request.
415	415		Default is no timeout.
416		-	is_parsing_enabled -- Search for other usernames in website pages.
	416	+	is_parsing_enabled -- Extract additional info from account pages.
	417	+	id_type -- Type of username to search.
	418	+	Default is 'username', see all supported here:
	419	+	https://github.com/soxoj/maigret/wiki/Supported-identifier-types
	420	+	max_connections -- Maximum number of concurrent connections allowed.
	421	+	Default is 100.
	422	+	no_progressbar -- Displaying of ASCII progressbar during scanner.
	423	+	cookies -- Filename of a cookie jar file to use for each request.
417	424
418	425		Return Value:
419	426		Dictionary containing results from report. Key of dictionary is the name
		skipped 10 lines
430	437		"""
431	438
432	439		# Notify caller that we are starting the query.
	440	+	if not query_notify:
	441	+	query_notify = Mock()
	442	+
433	443		query_notify.start(username, id_type)
434	444
435	445		# TODO: connector
		skipped 173 lines
609	619
610	620
611	621		async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False):
612		-	query_notify = Mock()
613	622		changes = {
614	623		'disabled': False,
615	624		}
		skipped 13 lines
629	638		for username, status in check_data:
630	639		async with semaphore:
631	640		results_dict = await maigret(
632		-	username,
633		-	{site.name: site},
634		-	query_notify,
635		-	logger,
	641	+	username=username,
	642	+	site_dict={site.name: site},
	643	+	logger=logger,
636	644		timeout=30,
637	645		id_type=site.type,
638	646		forced=True,
		skipped 78 lines

■ ■ ■ ■ ■ ■

maigret/maigret.py

		skipped 340 lines
341	341
342	342		sites_to_check = get_top_sites_for_id(id_type)
343	343
344		-	results = await maigret(username,
345		-	dict(sites_to_check),
346		-	query_notify,
	344	+	results = await maigret(username=username,
	345	+	site_dict=dict(sites_to_check),
	346	+	query_notify=query_notify,
347	347		proxy=args.proxy,
348	348		timeout=args.timeout,
349	349		is_parsing_enabled=parsing_enabled,
		skipped 82 lines

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 1412 lines
1413	1413		"usernameUnclaimed": "noonewouldeverusethis"
1414	1414		},
1415	1415		"Avto-forum.name": {
	1416	+	"ignore403": true,
1416	1417		"tags": [
1417	1418		"ru"
1418	1419		],
1419	1420		"engine": "XenForo",
1420	1421		"alexaRank": 716960,
1421		-	"ignore403": true,
1422	1422		"urlMain": "https://avto-forum.name",
1423	1423		"usernameClaimed": "mariya",
1424	1424		"usernameUnclaimed": "noonewouldeverusethis7"
1425	1425		},
1426	1426		"Avtoforum": {
	1427	+	"ignore403": true,
1427	1428		"tags": [
1428	1429		"ru"
1429	1430		],
1430	1431		"engine": "XenForo",
1431		-	"ignore403": true,
1432	1432		"urlMain": "https://avtoforum.org",
1433	1433		"usernameClaimed": "tim",
1434	1434		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 131 lines
1566	1566		"usernameUnclaimed": "noonewouldeverusethis7"
1567	1567		},
1568	1568		"XSS.is": {
	1569	+	"ignore403": true,
1569	1570		"tags": [
1570	1571		"hacking",
1571	1572		"ru"
1572	1573		],
1573	1574		"engine": "XenForo",
1574	1575		"alexaRank": 165220,
1575		-	"ignore403": true,
1576	1576		"urlMain": "https://xss.is",
1577	1577		"usernameClaimed": "adam",
1578	1578		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 63 lines
1642	1642		"usernameUnclaimed": "noonewouldeverusethis7"
1643	1643		},
1644	1644		"BeerMoneyForum": {
	1645	+	"ignore403": true,
1645	1646		"tags": [
1646	1647		"jp",
1647	1648		"ve"
		skipped 1 lines
1649	1650		"checkType": "message",
1650	1651		"absenceStrs": "The specified member cannot be found.",
1651	1652		"alexaRank": 11581,
1652		-	"ignore403": true,
1653	1653		"url": "https://www.beermoneyforum.com/members/?username={username}",
1654	1654		"urlMain": "https://www.beermoneyforum.com",
1655	1655		"usernameClaimed": "Yugocean",
		skipped 363 lines
2019	2019		"usernameUnclaimed": "noonewouldeverusethis7"
2020	2020		},
2021	2021		"BoomInfo": {
	2022	+	"ignore403": true,
2022	2023		"tags": [
2023	2024		"ru",
2024	2025		"ua"
		skipped 1 lines
2026	2027		"checkType": "message",
2027	2028		"absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f.",
2028	2029		"alexaRank": 1680672,
2029		-	"ignore403": true,
2030	2030		"url": "https://boominfo.ru/members/?username={username}",
2031	2031		"urlMain": "https://boominfo.ru",
2032	2032		"usernameClaimed": "boominfo",
		skipped 1546 lines
3579	3579		"usernameUnclaimed": "noonewouldeverusethis777"
3580	3580		},
3581	3581		"Dumpz": {
	3582	+	"ignore403": true,
3582	3583		"tags": [
3583	3584		"ru"
3584	3585		],
3585	3586		"engine": "XenForo",
3586	3587		"alexaRank": 1291982,
3587		-	"ignore403": true,
3588	3588		"urlMain": "https://dumpz.ws",
3589	3589		"usernameClaimed": "emailx45",
3590	3590		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 245 lines
3836	3836		"usernameUnclaimed": "noonewouldeverusethis7"
3837	3837		},
3838	3838		"Erogen.club": {
	3839	+	"ignore403": true,
3839	3840		"tags": [
3840	3841		"ru",
3841	3842		"ua"
3842	3843		],
3843	3844		"engine": "XenForo",
3844	3845		"alexaRank": 479929,
3845		-	"ignore403": true,
3846	3846		"urlMain": "https://erogen.club",
3847	3847		"usernameClaimed": "yanok",
3848	3848		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 1953 lines
5802	5802		},
5803	5803		"Gunandgame": {
5804	5804		"disabled": true,
	5805	+	"ignore403": true,
5805	5806		"tags": [
5806	5807		"us"
5807	5808		],
5808	5809		"checkType": "message",
5809	5810		"absenceStrs": "The specified member cannot be found. Please enter a member's entire name.",
5810		-	"ignore403": true,
5811	5811		"url": "https://www.gunandgame.com/members/?username={username}",
5812	5812		"urlMain": "https://www.gunandgame.co",
5813	5813		"usernameClaimed": "adam",
5814	5814		"usernameUnclaimed": "noonewouldeverusethis7"
5815	5815		},
5816	5816		"Gunboards": {
	5817	+	"ignore403": true,
5817	5818		"tags": [
5818	5819		"in",
5819	5820		"us"
		skipped 3 lines
5823	5824		],
5824	5825		"engine": "XenForo",
5825	5826		"alexaRank": 464194,
5826		-	"ignore403": true,
5827	5827		"urlMain": "https://forums.gunboards.com",
5828	5828		"usernameClaimed": "adam",
5829	5829		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 2661 lines
8491	8491		},
8492	8492		"Musiker-board": {
8493	8493		"disabled": true,
	8494	+	"ignore403": true,
8494	8495		"tags": [
8495	8496		"de"
8496	8497		],
8497	8498		"engine": "XenForo",
8498	8499		"alexaRank": 151707,
8499		-	"ignore403": true,
8500	8500		"urlMain": "https://www.musiker-board.de",
8501	8501		"usernameClaimed": "adam",
8502	8502		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 538 lines
9041	9041		"usernameUnclaimed": "noonewouldeverusethis7"
9042	9042		},
9043	9043		"Niketalk": {
	9044	+	"ignore403": true,
9044	9045		"tags": [
9045	9046		"us"
9046	9047		],
9047	9048		"checkType": "message",
9048	9049		"absenceStrs": "The specified member cannot be found",
9049	9050		"alexaRank": 165332,
9050		-	"ignore403": true,
9051	9051		"url": "https://niketalk.com/members/?username={username}",
9052	9052		"urlMain": "https://niketalk.com",
9053	9053		"usernameClaimed": "adam",
		skipped 570 lines
9624	9624		"usernameUnclaimed": "noonewouldeverusethis7"
9625	9625		},
9626	9626		"Pbnation": {
	9627	+	"ignore403": true,
9627	9628		"tags": [
9628	9629		"ca",
9629	9630		"us"
		skipped 1 lines
9631	9632		"checkType": "message",
9632	9633		"absenceStrs": "This user has not registered",
9633	9634		"alexaRank": 107502,
9634		-	"ignore403": true,
9635	9635		"url": "https://www.pbnation.com/member.php?username={username}",
9636	9636		"urlMain": "https://www.pbnation.com/",
9637	9637		"usernameClaimed": "adam",
		skipped 664 lines
10302	10302		"usernameUnclaimed": "noonewouldeverusethis7"
10303	10303		},
10304	10304		"Prosvetlenie": {
	10305	+	"ignore403": true,
10305	10306		"tags": [
10306	10307		"kg",
10307	10308		"ru"
		skipped 1 lines
10309	10310		"checkType": "message",
10310	10311		"absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
10311	10312		"alexaRank": 2256482,
10312		-	"ignore403": true,
10313	10313		"url": "http://www.prosvetlenie.org/forum/members/?username={username}",
10314	10314		"urlMain": "http://www.prosvetlenie.org",
10315	10315		"usernameClaimed": "odin",
		skipped 205 lines
10521	10521		"usernameUnclaimed": "noonewouldeverusethis7"
10522	10522		},
10523	10523		"RPGGeek": {
	10524	+	"ignore403": true,
10524	10525		"tags": [
10525	10526		"gaming",
10526	10527		"us"
		skipped 1 lines
10528	10529		"checkType": "message",
10529	10530		"absenceStrs": "User does not exist",
10530	10531		"alexaRank": 177522,
10531		-	"ignore403": true,
10532	10532		"url": "https://rpggeek.com/user/{username}",
10533	10533		"urlMain": "https://rpggeek.com",
10534	10534		"usernameClaimed": "adam",
		skipped 1 lines
10536	10536		},
10537	10537		"RPGRussia": {
10538	10538		"disabled": true,
	10539	+	"ignore403": true,
10539	10540		"tags": [
10540	10541		"ru",
10541	10542		"us"
10542	10543		],
10543	10544		"engine": "XenForo",
10544	10545		"alexaRank": 256354,
10545		-	"ignore403": true,
10546	10546		"urlMain": "https://rpgrussia.com",
10547	10547		"usernameClaimed": "adam",
10548	10548		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 679 lines
11228	11228		"usernameUnclaimed": "noonewouldeverusethis7"
11229	11229		},
11230	11230		"Rusfishing": {
	11231	+	"ignore403": true,
11231	11232		"tags": [
11232	11233		"ru"
11233	11234		],
11234	11235		"checkType": "message",
11235	11236		"absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
11236	11237		"alexaRank": 77147,
11237		-	"ignore403": true,
11238	11238		"url": "https://www.rusfishing.ru/forum/members/?username={username}",
11239	11239		"urlMain": "https://www.rusfishing.ru",
11240	11240		"usernameClaimed": "ale8443",
		skipped 292 lines
11533	11533		"usernameUnclaimed": "noonewouldeverusethis7"
11534	11534		},
11535	11535		"Sexforum.ws": {
	11536	+	"ignore403": true,
11536	11537		"tags": [
11537	11538		"ru"
11538	11539		],
11539	11540		"engine": "XenForo",
11540	11541		"alexaRank": 1815966,
11541		-	"ignore403": true,
11542	11542		"urlMain": "http://sexforum.ws",
11543	11543		"usernameClaimed": "katrin1988",
11544	11544		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 572 lines
12117	12117		"us"
12118	12118		],
12119	12119		"headers": {
12120		-	"authorization": "Bearer BQAhjPM0jv30LtMfeJXLPMYFBmAKG4RGzVUncYmOS9ufDGyNie5scjyVt8ujXA2bayqRX4f690lZUz0Kx9w"
	12120	+	"authorization": "Bearer BQBgSgdI2iyspCVcbJZjVlMt3-84iAe7qyMh_ZO095wRaD68tB-Zz-US5cXPD0DSVfluwjZWn1Bf2EVJIN8"
12121	12121		},
12122	12122		"errors": {
12123	12123		"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
		skipped 959 lines
13083	13083		"usernameUnclaimed": "noonewouldeverusethis7"
13084	13084		},
13085	13085		"TotalStavki": {
	13086	+	"ignore403": true,
13086	13087		"tags": [
13087	13088		"ru"
13088	13089		],
13089	13090		"checkType": "message",
13090	13091		"absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
13091	13092		"alexaRank": 5459551,
13092		-	"ignore403": true,
13093	13093		"url": "https://totalstavki.ru/forum/members/?username={username}",
13094	13094		"urlMain": "https://totalstavki.ru",
13095	13095		"usernameClaimed": "turbo",
		skipped 323 lines
13419	13419		"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
13420	13420		"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
13421	13421		"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
13422		-	"x-guest-token": "1376173136807403521"
	13422	+	"x-guest-token": "1376637415348113408"
13423	13423		},
13424	13424		"errors": {
13425	13425		"Bad guest token": "x-guest-token update required"
		skipped 340 lines
13766	13766		"usernameUnclaimed": "noonewouldeverusethis7"
13767	13767		},
13768	13768		"VideogameGeek": {
	13769	+	"ignore403": true,
13769	13770		"tags": [
13770	13771		"gaming",
13771	13772		"us"
		skipped 1 lines
13773	13774		"checkType": "message",
13774	13775		"absenceStrs": "User does not exist",
13775	13776		"alexaRank": 719092,
13776		-	"ignore403": true,
13777	13777		"url": "https://videogamegeek.com/user/{username}",
13778	13778		"urlMain": "https://videogamegeek.com",
13779	13779		"usernameClaimed": "adam",
		skipped 16 lines
13796	13796		"video"
13797	13797		],
13798	13798		"headers": {
13799		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTY5NDExNDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.ArAL1V2XErUD3C2FnZSmcj96Nc2sfG6WHbUw_AtrSqA"
	13799	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTcwNTE4NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.iZzO-_VDARa_honzp7KvRSaSK0qMQ7n8dp9k6gah_bE"
13800	13800		},
13801	13801		"activation": {
13802	13802		"url": "https://vimeo.com/_rv/viewer",
		skipped 78 lines
13881	13881		"usernameUnclaimed": "noonewouldeverusethis7"
13882	13882		},
13883	13883		"Vlmi": {
	13884	+	"ignore403": true,
13884	13885		"tags": [
13885	13886		"ru",
13886	13887		"ua"
13887	13888		],
13888	13889		"engine": "XenForo",
13889	13890		"alexaRank": 725829,
13890		-	"ignore403": true,
13891	13891		"urlMain": "https://vlmi.biz",
13892	13892		"usernameClaimed": "mixa",
13893	13893		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 610 lines
14504	14504		"usernameUnclaimed": "noonewouldeverusethis7"
14505	14505		},
14506	14506		"Wuz": {
	14507	+	"ignore403": true,
14507	14508		"tags": [
14508	14509		"by",
14509	14510		"ru"
		skipped 1 lines
14511	14512		"checkType": "message",
14512	14513		"absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
14513	14514		"alexaRank": 1927898,
14514		-	"ignore403": true,
14515	14515		"url": "http://wuz.by/forum/members/?username={username}",
14516	14516		"urlMain": "http://wuz.by",
14517	14517		"usernameClaimed": "adam",
		skipped 1539 lines
16057	16057		"usernameUnclaimed": "noonewouldeverusethis7"
16058	16058		},
16059	16059		"forums.overclockers.co.uk": {
	16060	+	"ignore403": true,
16060	16061		"tags": [
16061	16062		"gb",
16062	16063		"uk"
		skipped 1 lines
16064	16065		"checkType": "message",
16065	16066		"absenceStrs": "The specified member cannot be found. Please enter a member's entire name.",
16066	16067		"alexaRank": 10013,
16067		-	"ignore403": true,
16068	16068		"url": "https://forums.overclockers.co.uk/members/?username={username}",
16069	16069		"urlMain": "https://forums.overclockers.co.uk",
16070	16070		"usernameClaimed": "adam",
		skipped 7397 lines
23468	23468		"alexaRank": 332838,
23469	23469		"urlMain": "https://skyblock.net",
23470	23470		"usernameClaimed": "alex",
	23471	+	"usernameUnclaimed": "noonewouldeverusethis7"
	23472	+	},
	23473	+	"codeberg.org": {
	23474	+	"checkType": "message",
	23475	+	"presenseStrs": [
	23476	+	"user profile",
	23477	+	" username text center"
	23478	+	],
	23479	+	"absenceStrs": [
	23480	+	"og:description",
	23481	+	" ui centered image"
	23482	+	],
	23483	+	"url": "https://codeberg.org/{username}",
	23484	+	"urlMain": "https://codeberg.org",
	23485	+	"usernameClaimed": "pcastela",
23471	23486		"usernameUnclaimed": "noonewouldeverusethis7"
23472	23487		}
23473	23488		},
		skipped 157 lines

■ ■ ■ ■ ■ ■

maigret/submit.py

		skipped 26 lines
27	27
28	28
29	29		async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False):
30		-	query_notify = Mock()
31	30		changes = {
32	31		'disabled': False,
33	32		}
		skipped 7 lines
41	40
42	41		for username, status in check_data:
43	42		results_dict = await maigret(
44		-	username,
45		-	{site.name: site},
46		-	query_notify,
47		-	logger,
	43	+	username=username,
	44	+	site_dict={site.name: site},
	45	+	logger=logger,
48	46		timeout=30,
49	47		id_type=site.type,
50	48		forced=True,
		skipped 185 lines

■ ■ ■ ■ ■ ■

requirements.txt

		skipped 17 lines
18	18		MarkupSafe==1.1.1
19	19		mock==4.0.2
20	20		multidict==5.1.0
21		-	Pillow==8.1.1
22	21		pycountry==20.7.3
23	22		PyPDF2==1.26.0
24	23		PySocks==1.7.1
25	24		python-bidi==0.4.2
26	25		python-socks==1.1.2
27		-	reportlab==3.5.59
28	26		requests>=2.24.0
29	27		requests-futures==1.0.0
30	28		six==1.15.0
		skipped 11 lines

1	1		"""Maigret"""
2	2
	3	+	from .checking import maigret as search
	4	+

Merge pull request #93 from soxoj/docs-requirements