STRLCPY/maigret

Fix false positives (#370)

* Fixed several false positives, improved statistics info

* Disabled some sites, fixed fp percent count method

* Updated site list and statistics

Soxoj committed with GitHub 3 years ago

bc787cdf

1 parent dcf5181e

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 3 files Show one by one

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 1879 lines
1880	1880		],
1881	1881		"checkType": "message",
1882	1882		"absenceStrs": [
1883		-	"\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u0441 \u0442\u0430\u043a\u0438\u043c \u0438\u043c\u0435\u043d\u0435\u043c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442!"
	1883	+	"\u041e\u0448\u0438\u0431\u043a\u0430 / \u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u0441 \u0442\u0430\u043a\u0438\u043c \u0438\u043c\u0435\u043d\u0435\u043c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442"
1884	1884		],
1885	1885		"alexaRank": 2303903,
1886	1886		"urlMain": "https://bgforum.ru",
		skipped 1711 lines
3598	3598		"tags": [
3599	3599		"ru"
3600	3600		],
	3601	+	"disabled": true,
3601	3602		"checkType": "status_code",
3602	3603		"urlMain": "https://dinsk.su",
3603	3604		"url": "https://dinsk.su/user/{username}",
		skipped 967 lines
4571	4572		"tags": [
4572	4573		"ru"
4573	4574		],
	4575	+	"disabled": true,
4574	4576		"checkType": "status_code",
4575	4577		"alexaRank": 1225740,
4576	4578		"urlMain": "https://favera.ru",
		skipped 3969 lines
8546	8548		"tags": [
8547	8549		"forum"
8548	8550		],
	8551	+	"disabled": true,
8549	8552		"checkType": "message",
8550	8553		"absenceStrs": [
8551	8554		"The specified member cannot be found"
		skipped 530 lines
9082	9085		},
9083	9086		"Movescount": {
9084	9087		"tags": [
9085		-	"es",
9086		-	"in",
9087		-	"pk",
9088		-	"ru",
9089		-	"us"
	9088	+	"maps"
9090	9089		],
	9090	+	"disabled": true,
9091	9091		"checkType": "message",
9092	9092		"absenceStrs": [
9093	9093		"error=4&"
		skipped 3797 lines
12891	12891		},
12892	12892		"Snooth": {
12893	12893		"tags": [
12894		-	"in"
	12894	+	"news"
12895	12895		],
12896	12896		"checkType": "message",
12897	12897		"absenceStrs": [
12898		-	"<title>Profiles on Snooth</title>"
	12898	+	"<title>Page not found"
	12899	+	],
	12900	+	"presenseStrs": [
	12901	+	"content=\"https://www.snooth.com/author/"
12899	12902		],
12900	12903		"alexaRank": 4088489,
12901	12904		"urlMain": "https://www.snooth.com/",
12902		-	"url": "https://www.snooth.com/profiles/{username}/",
12903		-	"usernameClaimed": "GregT",
	12905	+	"url": "https://www.snooth.com/author/{username}/",
	12906	+	"usernameClaimed": "joshua",
12904	12907		"usernameUnclaimed": "noonewouldeverusethis7"
12905	12908		},
12906	12909		"SocialLibremOne": {
		skipped 2082 lines
14989	14992		"video"
14990	14993		],
14991	14994		"headers": {
14992		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2NDU4Nzg1NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.Bs6VBcKPsl-5dqoThdAImBIex1mas1UcyG2pSnIYqYk"
	14995	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2NDU4Nzk3NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.T8E8Vrx0sO-9WP4RdZGNQZw2EB1hYTIXbIguXIZbfNQ"
14993	14996		},
14994	14997		"activation": {
14995	14998		"url": "https://vimeo.com/_rv/viewer",
		skipped 9837 lines
24833	24836		"usernameClaimed": "alex",
24834	24837		"usernameUnclaimed": "noonewouldeverusethis7"
24835	24838		},
24836		-	"hashnode.com": {
	24839	+	"hashnode": {
24837	24840		"tags": [
24838	24841		"in"
24839	24842		],
		skipped 6 lines
24846	24849		" name="
24847	24850		],
24848	24851		"absenceStrs": [
24849		-	"></title><meta name="
	24852	+	"We can\u2019t find the page you\u2019re looking for!"
24850	24853		],
24851	24854		"urlMain": "https://hashnode.com",
24852	24855		"url": "https://hashnode.com/@{username}",
		skipped 2094 lines
26947	26950		},
26948	26951		"forum.rastrnet.ru": {
26949	26952		"urlMain": "http://forum.rastrnet.ru",
	26953	+	"errors": {
	26954	+	"\u0418\u0437\u0432\u0438\u043d\u0438\u0442\u0435, \u043f\u0440\u043e\u0432\u043e\u0434\u044f\u0442\u0441\u044f \u0442\u0435\u0445\u043d\u0438\u0447\u0435\u0441\u043a\u0438\u0435 \u0440\u0430\u0431\u043e\u0442\u044b.": "Site error"
	26955	+	},
26950	26956		"engine": "vBulletin",
26951	26957		"usernameClaimed": "alex",
26952	26958		"usernameUnclaimed": "noonewouldeverusethis7",
		skipped 2154 lines

■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 437 lines
438	438		url_type = site.get_url_template()
439	439		urls[url_type] = urls.get(url_type, 0) + 1
440	440
441		-	if site.check_type == 'message':
	441	+	if site.check_type == 'message' and not site.disabled:
442	442		message_checks += 1
443	443		if site.absence_strs and site.presense_strs:
444	444		continue
		skipped 31 lines

■ ■ ■ ■ ■ ■

sites.md

		skipped 811 lines
812	812		1. ![](https://www.google.com/s2/favicons?domain=https://bbs.huami.com) [bbs.huami.com (https://bbs.huami.com)](https://bbs.huami.com): top 10M, cn, in, ir, ru, us, search is disabled
813	813		1. ![](https://www.google.com/s2/favicons?domain=https://amazfitwatchfaces.com) [AmazfitWatchFaces (https://amazfitwatchfaces.com)](https://amazfitwatchfaces.com): top 10M, ae, es, forum, gr, id, ir, ru
814	814		1. ![](https://www.google.com/s2/favicons?domain=https://moikrug.ru/) [MoiKrug (https://moikrug.ru/)](https://moikrug.ru/): top 10M, career, us
815		-	1. ![](https://www.google.com/s2/favicons?domain=http://www.movescount.com) [Movescount (http://www.movescount.com)](http://www.movescount.com): top 10M, es, in, pk, ru, us
	815	+	1. ![](https://www.google.com/s2/favicons?domain=http://www.movescount.com) [Movescount (http://www.movescount.com)](http://www.movescount.com): top 10M, maps, search is disabled
816	816		1. ![](https://www.google.com/s2/favicons?domain=https://tamtam.chat/) [TamTam (https://tamtam.chat/)](https://tamtam.chat/): top 10M, ru
817	817		1. ![](https://www.google.com/s2/favicons?domain=https://forum.velomania.ru/) [Velomania (https://forum.velomania.ru/)](https://forum.velomania.ru/): top 10M, forum, ru
818	818		1. ![](https://www.google.com/s2/favicons?domain=https://forum.itvdn.com) [ITVDN Forum (https://forum.itvdn.com)](https://forum.itvdn.com): top 10M, forum, ru, ua
		skipped 380 lines
1199	1199		1. ![](https://www.google.com/s2/favicons?domain=https://mastodon.xyz/) [mastodon.xyz (https://mastodon.xyz/)](https://mastodon.xyz/): top 10M, th
1200	1200		1. ![](https://www.google.com/s2/favicons?domain=https://www.gays.com) [Gays (https://www.gays.com)](https://www.gays.com): top 10M, in, search is disabled
1201	1201		1. ![](https://www.google.com/s2/favicons?domain=http://transit-club.com) [transit-club.com (http://transit-club.com)](http://transit-club.com): top 10M, ru
1202		-	1. ![](https://www.google.com/s2/favicons?domain=https://favera.ru) [Favera (https://favera.ru)](https://favera.ru): top 10M, ru
	1202	+	1. ![](https://www.google.com/s2/favicons?domain=https://favera.ru) [Favera (https://favera.ru)](https://favera.ru): top 10M, ru, search is disabled
1203	1203		1. ![](https://www.google.com/s2/favicons?domain=https://soylentnews.org) [soylentnews (https://soylentnews.org)](https://soylentnews.org): top 10M, us
1204	1204		1. ![](https://www.google.com/s2/favicons?domain=http://chan4chan.com/) [Chan4chan (http://chan4chan.com/)](http://chan4chan.com/): top 10M, hu
1205	1205		1. ![](https://www.google.com/s2/favicons?domain=http://the-mainboard.com/index.php) [the-mainboard.com (http://the-mainboard.com/index.php)](http://the-mainboard.com/index.php): top 10M, forum, us
		skipped 184 lines
1390	1390		1. ![](https://www.google.com/s2/favicons?domain=https://uaksu.forum24.ru/) [Uaksu (https://uaksu.forum24.ru/)](https://uaksu.forum24.ru/): top 10M, forum, ru, ua
1391	1391		1. ![](https://www.google.com/s2/favicons?domain=http://0-3.ru) [0-3.RU (http://0-3.ru)](http://0-3.ru): top 10M, forum, ru
1392	1392		1. ![](https://www.google.com/s2/favicons?domain=http://www.forumsi.org) [Forumsi (http://www.forumsi.org)](http://www.forumsi.org): top 10M, forum, ru
1393		-	1. ![](https://www.google.com/s2/favicons?domain=https://www.snooth.com/) [Snooth (https://www.snooth.com/)](https://www.snooth.com/): top 10M, in
	1393	+	1. ![](https://www.google.com/s2/favicons?domain=https://www.snooth.com/) [Snooth (https://www.snooth.com/)](https://www.snooth.com/): top 10M, news
1394	1394		1. ![](https://www.google.com/s2/favicons?domain=http://soft-deniz.ucoz.ru) [soft-deniz.ucoz.ru (http://soft-deniz.ucoz.ru)](http://soft-deniz.ucoz.ru): top 10M
1395	1395		1. ![](https://www.google.com/s2/favicons?domain=http://oih.at.ua) [oih.at.ua (http://oih.at.ua)](http://oih.at.ua): top 10M, ua
1396	1396		1. ![](https://www.google.com/s2/favicons?domain=http://gorodanapa.ru/) [Gorodanapa (http://gorodanapa.ru/)](http://gorodanapa.ru/): top 10M, ru, search is disabled
		skipped 162 lines
1559	1559		1. ![](https://www.google.com/s2/favicons?domain=https://cyberclock.cc) [Cyberclock (https://cyberclock.cc)](https://cyberclock.cc): top 100M, ru
1560	1560		1. ![](https://www.google.com/s2/favicons?domain=http://www.cydak.ru) [Cydak (http://www.cydak.ru)](http://www.cydak.ru): top 100M, ru
1561	1561		1. ![](https://www.google.com/s2/favicons?domain=https://www.designspiration.net/) [Designspiration (https://www.designspiration.net/)](https://www.designspiration.net/): top 100M
1562		-	1. ![](https://www.google.com/s2/favicons?domain=https://dinsk.su) [Dinsk (https://dinsk.su)](https://dinsk.su): top 100M, ru
	1562	+	1. ![](https://www.google.com/s2/favicons?domain=https://dinsk.su) [Dinsk (https://dinsk.su)](https://dinsk.su): top 100M, ru, search is disabled
1563	1563		1. ![](https://www.google.com/s2/favicons?domain=https://forum.djangoproject.co) [Djangoproject.co (https://forum.djangoproject.co)](https://forum.djangoproject.co): top 100M, coding, forum
1564	1564		1. ![](https://www.google.com/s2/favicons?domain=https://www.dublikat.shop) [Dublikat (https://www.dublikat.shop)](https://www.dublikat.shop): top 100M, ru
1565	1565		1. ![](https://www.google.com/s2/favicons?domain=http://eightbit.me/) [Eightbit (http://eightbit.me/)](http://eightbit.me/): top 100M
		skipped 20 lines
1586	1586		1. ![](https://www.google.com/s2/favicons?domain=https://macqa.ru) [Macqa (https://macqa.ru)](https://macqa.ru): top 100M, ru
1587	1587		1. ![](https://www.google.com/s2/favicons?domain=https://magiimir.com) [Magiimir (https://magiimir.com)](https://magiimir.com): top 100M, ru
1588	1588		1. ![](https://www.google.com/s2/favicons?domain=https://mamochki.by/) [Mamochki (https://mamochki.by/)](https://mamochki.by/): top 100M, by, ru
1589		-	1. ![](https://www.google.com/s2/favicons?domain=https://mastersofcrypto.com) [Mastersofcrypto (https://mastersofcrypto.com)](https://mastersofcrypto.com): top 100M, forum
	1589	+	1. ![](https://www.google.com/s2/favicons?domain=https://mastersofcrypto.com) [Mastersofcrypto (https://mastersofcrypto.com)](https://mastersofcrypto.com): top 100M, forum, search is disabled
1590	1590		1. ![](https://www.google.com/s2/favicons?domain=http:/mixlr.com/) [Mixlr (http:/mixlr.com/)](http:/mixlr.com/): top 100M, gb
1591	1591		1. ![](https://www.google.com/s2/favicons?domain=https://www.munzee.com/) [Munzee (https://www.munzee.com/)](https://www.munzee.com/): top 100M, gb
1592	1592		1. ![](https://www.google.com/s2/favicons?domain=http://murmansk-life.ru) [MurmanskLife (http://murmansk-life.ru)](http://murmansk-life.ru): top 100M, ru
		skipped 720 lines
2313	2313		1. ![](https://www.google.com/s2/favicons?domain=http://mednolit.ru) [mednolit.ru (http://mednolit.ru)](http://mednolit.ru): top 100M, ru
2314	2314		1. ![](https://www.google.com/s2/favicons?domain=http://mikele-loconte.ru) [mikele-loconte.ru (http://mikele-loconte.ru)](http://mikele-loconte.ru): top 100M
2315	2315		1. ![](https://www.google.com/s2/favicons?domain=http://mkuniverse.ru) [mkuniverse.ru (http://mkuniverse.ru)](http://mkuniverse.ru): top 100M
2316		-	1. ![](https://www.google.com/s2/favicons?domain=https://hashnode.com) [hashnode.com (https://hashnode.com)](https://hashnode.com): top 100M, in
	2316	+	1. ![](https://www.google.com/s2/favicons?domain=https://hashnode.com) [hashnode (https://hashnode.com)](https://hashnode.com): top 100M, in
2317	2317		1. ![](https://www.google.com/s2/favicons?domain=https://www.change.org) [www.change.org (https://www.change.org)](https://www.change.org): top 100M, us
2318	2318		1. ![](https://www.google.com/s2/favicons?domain=https://www.ifunny.co) [www.ifunny.co (https://www.ifunny.co)](https://www.ifunny.co): top 100M, us
2319	2319		1. ![](https://www.google.com/s2/favicons?domain=https://localcryptosapi.com) [LocalCryptos (https://localcryptosapi.com)](https://localcryptosapi.com): top 100M
		skipped 279 lines
2599	2599		1. ![](https://www.google.com/s2/favicons?domain=https://www.hozpitality.com) [hozpitality (https://www.hozpitality.com)](https://www.hozpitality.com): top 100M
2600	2600		1. ![](https://www.google.com/s2/favicons?domain=https://kazanlashkigalab.com) [kazanlashkigalab.com (https://kazanlashkigalab.com)](https://kazanlashkigalab.com): top 100M, kz
2601	2601
2602		-	Alexa.com rank data fetched at (2022-02-26 12:19:53.127789 UTC)
	2602	+	Alexa.com rank data fetched at (2022-02-26 12:55:54.605333 UTC)
2603	2603		## Statistics
2604	2604
2605		-	Enabled/total sites: 2447/2595 = 94.3%
	2605	+	Enabled/total sites: 2443/2595 = 94.14%
2606	2606
2607		-	Incomplete checks: 582/1978 = 29.42% (false positive risks)
	2607	+	Incomplete checks: 525/1853 = 28.33% (false positive risks)
2608	2608
2609	2609		Top 20 profile URLs:
2610	2610		- (796) `{urlMain}/index/8-0-{username} (uCoz)`
		skipped 14 lines
2625	2625		- (18) `/forum/members/?username={username}`
2626	2626		- (18) `/forum/search.php?keywords=&terms=all&author={username}`
2627	2627		- (17) `/search.php?keywords=&terms=all&author={username}`
2628		-	- (15) `/author/{username}`
	2628	+	- (16) `/author/{username}`
2629	2629		- (14) `/profile.php?mode=viewprofile&u={username}`
2630	2630
2631	2631		Top 20 tags:
2632		-	- (255) `forum`
	2632	+	- (271) `forum`
	2633	+	- (79) `NO_TAGS` (non-standard)
2633	2634		- (50) `gaming`
2634		-	- (40) `NO_TAGS` (non-standard)
	2635	+	- (24) `photo`
2635	2636		- (24) `coding`
2636		-	- (23) `photo`
2637	2637		- (18) `news`
2638	2638		- (18) `blog`
2639		-	- (17) `music`
	2639	+	- (18) `music`
2640	2640		- (15) `tech`
2641	2641		- (13) `freelance`
2642	2642		- (12) `sharing`
		skipped 2 lines
2645	2645		- (10) `dating`
2646	2646		- (10) `art`
2647	2647		- (9) `hobby`
2648		-	- (8) `movies`
	2648	+	- (9) `movies`
2649	2649		- (7) `sport`
2650	2650		- (7) `hacking`
2651	2651		- (5) `stock`
		skipped 1 lines