STRLCPY/maigret

Merge pull request #56 from soxoj/stats-discourse
```
Added stats flag, added Discourse engine
```
soxoj committed with GitHub 4 years ago

2e3eceed

2 parents
a9543e83
9bc3615a

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

maigret/maigret.py

		skipped 111 lines
112	112		action="store_true", default=False,
113	113		help="Do self check for sites and database and disable non-working ones."
114	114		)
	115	+	parser.add_argument("--stats",
	116	+	action="store_true", default=False,
	117	+	help="Show database statistics."
	118	+	)
115	119		parser.add_argument("--use-disabled-sites",
116	120		action="store_true", default=False,
117	121		help="Use disabled sites to search (may cause many false positives)."
		skipped 134 lines
252	256		print('Database was successfully updated.')
253	257		else:
254	258		print('Updates will be applied only for current search session.')
255		-	print(db.get_stats(site_data))
	259	+	print(db.get_scan_stats(site_data))
	260	+
	261	+	if args.stats:
	262	+	print(db.get_db_stats(db.sites_dict))
256	263
257	264		# Make reports folder is not exists
258	265		os.makedirs(args.folderoutput, exist_ok=True)
		skipped 136 lines

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 1228 lines
1229	1229		},
1230	1230		"Ask Fedora": {
1231	1231		"tags": [
1232		-	"in",
1233	1232		"us"
1234	1233		],
1235		-	"checkType": "status_code",
	1234	+	"absenceStrs": ["Sorry, we couldn't find that page."],
1236	1235		"alexaRank": 38649,
1237		-	"url": "https://ask.fedoraproject.org/u/{username}",
	1236	+	"engine": "Discourse",
1238	1237		"urlMain": "https://ask.fedoraproject.org/",
1239		-	"usernameClaimed": "red",
	1238	+	"usernameClaimed": "grsm",
1240	1239		"usernameUnclaimed": "noonewouldeverusethis7"
1241	1240		},
1242	1241		"AskFM": {
		skipped 934 lines
2177	2176		"tags": [
2178	2177		"us"
2179	2178		],
2180		-	"checkType": "status_code",
	2179	+	"engine": "Discourse",
2181	2180		"alexaRank": 208539,
2182		-	"url": "https://community.byte.co/u/{username}/summary",
2183	2181		"urlMain": "https://community.byte.co",
2184		-	"usernameClaimed": "leckakay",
	2182	+	"usernameClaimed": "red",
2185	2183		"usernameUnclaimed": "noonewouldeverusethis7"
2186	2184		},
2187	2185		"CNET": {
		skipped 184 lines
2372	2370		"tags": [
2373	2371		"de"
2374	2372		],
2375		-	"checkType": "status_code",
2376		-	"url": "https://www.casino-affiliate-forum.com/u/{username}/summary",
	2373	+	"engine": "Discourse",
2377	2374		"urlMain": "https://www.casino-affiliate-forum.com",
2378	2375		"usernameClaimed": "torstenw",
2379	2376		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 264 lines
2644	2641		},
2645	2642		"CloudflareCommunity": {
2646	2643		"tags": [
2647		-	"in"
	2644	+	"tech"
2648	2645		],
2649		-	"checkType": "status_code",
2650	2646		"alexaRank": 977,
2651		-	"url": "https://community.cloudflare.com/u/{username}",
	2647	+	"engine": "Discourse",
2652	2648		"urlMain": "https://community.cloudflare.com/",
2653	2649		"usernameClaimed": "blue",
2654	2650		"usernameUnclaimed": "noonewouldeverusethis"
		skipped 704 lines
3359	3355		},
3360	3356		"DiscoursePi-hole": {
3361	3357		"tags": [
3362		-	"global",
3363	3358		"us"
3364	3359		],
3365		-	"checkType": "status_code",
3366	3360		"alexaRank": 67872,
3367		-	"url": "https://discourse.pi-hole.net/u/{username}/summary",
	3361	+	"engine": "Discourse",
3368	3362		"urlMain": "https://discourse.pi-hole.net",
3369	3363		"usernameClaimed": "adam",
3370	3364		"usernameUnclaimed": "noonewouldeverusethis7"
3371	3365		},
3372	3366		"Discuss.Elastic.co": {
3373	3367		"tags": [
3374		-	"in",
3375	3368		"tech",
3376	3369		"us"
3377	3370		],
3378		-	"checkType": "status_code",
3379	3371		"alexaRank": 5906,
3380		-	"url": "https://discuss.elastic.co/u/{username}",
	3372	+	"engine": "Discourse",
3381	3373		"urlMain": "https://discuss.elastic.co/",
3382	3374		"usernameClaimed": "blue",
3383	3375		"usernameUnclaimed": "noonewouldeverusethis7"
3384	3376		},
3385	3377		"DiscussPython": {
3386	3378		"tags": [
3387		-	"coding",
3388		-	"global",
3389		-	"in",
3390		-	"us"
	3379	+	"coding"
3391	3380		],
3392		-	"checkType": "status_code",
3393	3381		"alexaRank": 943,
3394		-	"url": "https://discuss.python.org/u/{username}/summary",
	3382	+	"engine": "Discourse",
3395	3383		"urlMain": "https://discuss.python.org/",
3396	3384		"usernameClaimed": "dustin",
3397	3385		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 47 lines
3445	3433		},
3446	3434		"Djangoproject.co": {
3447	3435		"tags": [
3448		-	"global"
	3436	+	"coding"
3449	3437		],
3450	3438		"checkType": "status_code",
3451		-	"url": "https://forum.djangoproject.com/u/{username}/summary",
	3439	+	"engine": "Discourse",
3452	3440		"urlMain": "https://forum.djangoproject.co",
3453	3441		"usernameClaimed": "mikhail349",
3454	3442		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 324 lines
3779	3767		},
3780	3768		"Elixirforum": {
3781	3769		"tags": [
3782		-	"global",
3783	3770		"in",
3784	3771		"mx",
3785	3772		"us",
3786		-	"ve"
	3773	+	"ve",
	3774	+	"coding"
3787	3775		],
3788		-	"checkType": "status_code",
3789	3776		"alexaRank": 82408,
3790		-	"url": "https://elixirforum.com/u/{username}/summary",
	3777	+	"engine": "Discourse",
3791	3778		"urlMain": "https://elixirforum.com",
3792	3779		"usernameClaimed": "clmay",
3793	3780		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 61 lines
3855	3842		"au",
3856	3843		"in"
3857	3844		],
3858		-	"checkType": "message",
3859		-	"absenceStrs": "That page doesn\u2019t exist or is private.",
3860	3845		"alexaRank": 984,
3861		-	"url": "https://forums.envato.com/u/{username}/summary",
3862		-	"urlMain": "https://forums.envato.com/",
	3846	+	"engine": "Discourse",
	3847	+	"urlMain": "https://forums.envato.com",
3863	3848		"usernameClaimed": "zigro",
3864	3849		"usernameUnclaimed": "noonewouldeverusethis7"
3865	3850		},
		skipped 49 lines
3915	3900		},
3916	3901		"Ethereum-magicians": {
3917	3902		"tags": [
3918		-	"cr",
3919		-	"global"
	3903	+	"cr"
3920	3904		],
3921		-	"checkType": "status_code",
3922	3905		"alexaRank": 778429,
3923		-	"url": "https://ethereum-magicians.org/u/{username}/summary",
	3906	+	"engine": "Discourse",
3924	3907		"urlMain": "https://ethereum-magicians.org",
3925	3908		"usernameClaimed": "amxx",
3926	3909		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 19 lines
3946	3929		"cr",
3947	3930		"us"
3948	3931		],
3949		-	"checkType": "status_code",
3950	3932		"alexaRank": 308342,
3951		-	"url": "https://ethresear.ch/u/{username}/summary",
	3933	+	"engine": "Discourse",
3952	3934		"urlMain": "https://ethresear.ch",
3953	3935		"usernameClaimed": "weijiekoh",
3954	3936		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 83 lines
4038	4020		},
4039	4021		"F-droid": {
4040	4022		"tags": [
4041		-	"global",
4042	4023		"in"
4043	4024		],
4044		-	"checkType": "status_code",
4045	4025		"alexaRank": 67782,
4046		-	"url": "https://forum.f-droid.org/u/{username}/summary",
	4026	+	"engine": "Discourse",
4047	4027		"urlMain": "https://forum.f-droid.org",
4048		-	"usernameClaimed": "red",
	4028	+	"usernameClaimed": "blue",
4049	4029		"usernameUnclaimed": "noonewouldeverusethis7"
4050	4030		},
4051	4031		"F3.cool": {
		skipped 839 lines
4891	4871		},
4892	4872		"Freecodecamp": {
4893	4873		"tags": [
4894		-	"global",
4895	4874		"in",
4896	4875		"us"
4897	4876		],
4898		-	"checkType": "status_code",
4899	4877		"alexaRank": 1605,
4900		-	"url": "https://www.freecodecamp.org/forum/u/{username}/summary",
	4878	+	"engine": "Discourse",
4901	4879		"urlMain": "https://www.freecodecamp.org/forum/",
4902	4880		"usernameClaimed": "red",
4903	4881		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 697 lines
5601	5579		},
5602	5580		"Golangbridge": {
5603	5581		"tags": [
5604		-	"global",
5605	5582		"in",
5606	5583		"sa",
5607	5584		"ua",
5608	5585		"us",
5609	5586		"vn"
5610	5587		],
5611		-	"checkType": "status_code",
5612	5588		"alexaRank": 267803,
5613		-	"url": "https://forum.golangbridge.org/u/{username}/summary",
	5589	+	"engine": "Discourse",
5614	5590		"urlMain": "https://forum.golangbridge.org/",
5615	5591		"usernameClaimed": "red",
5616	5592		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 553 lines
6170	6146		"rs",
6171	6147		"us"
6172	6148		],
6173		-	"checkType": "status_code",
6174	6149		"alexaRank": 549532,
6175		-	"url": "https://www.hitmanforum.com/u/{username}/summary",
	6150	+	"engine": "Discourse",
6176	6151		"urlMain": "https://www.hitmanforum.com",
6177		-	"usernameClaimed": "pushdagger",
	6152	+	"usernameClaimed": "john",
6178	6153		"usernameUnclaimed": "noonewouldeverusethis7"
6179	6154		},
6180	6155		"Hockeyforum": {
		skipped 46 lines
6227	6202		},
6228	6203		"Hoobly": {
6229	6204		"tags": [
6230		-	"global",
6231		-	"in"
	6205	+	"classified"
6232	6206		],
6233	6207		"checkType": "status_code",
6234	6208		"alexaRank": 19173,
		skipped 180 lines
6415	6389		"ru",
6416	6390		"ua"
6417	6391		],
6418		-	"checkType": "status_code",
6419	6392		"alexaRank": 201191,
6420		-	"url": "https://forum.itvdn.com/u/{username}",
	6393	+	"engine": "Discourse",
6421	6394		"urlMain": "https://forum.itvdn.com",
6422	6395		"usernameClaimed": "pizzaro",
6423	6396		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 155 lines
6579	6552		},
6580	6553		"Infura": {
6581	6554		"tags": [
6582		-	"global",
6583	6555		"kr",
6584	6556		"us"
6585	6557		],
6586		-	"checkType": "status_code",
6587	6558		"alexaRank": 58165,
6588		-	"url": "https://community.infura.io/u/{username}/summary",
	6559	+	"engine": "Discourse",
6589	6560		"urlMain": "https://community.infura.io",
6590	6561		"usernameClaimed": "adam",
6591	6562		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 1359 lines
7951	7922		},
7952	7923		"Mapillary Forum": {
7953	7924		"tags": [
7954		-	"global",
7955	7925		"forum"
7956	7926		],
7957		-	"checkType": "status_code",
7958	7927		"alexaRank": 651639,
7959		-	"url": "https://forum.mapillary.com/u/{username}/summary",
	7928	+	"engine": "Discourse",
7960	7929		"urlMain": "https://forum.mapillary.com",
7961	7930		"usernameClaimed": "slashme",
7962	7931		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 224 lines
8187	8156		},
8188	8157		"MetaDiscourse": {
8189	8158		"tags": [
8190		-	"global",
8191	8159		"us"
8192	8160		],
8193		-	"checkType": "status_code",
8194	8161		"alexaRank": 27477,
8195		-	"url": "https://meta.discourse.org/u/{username}",
	8162	+	"engine": "Discourse",
8196	8163		"urlMain": "https://meta.discourse.org/",
8197	8164		"usernameClaimed": "adam",
8198	8165		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 2420 lines
10619	10586		},
10620	10587		"Quartertothree": {
10621	10588		"tags": [
10622		-	"global",
10623	10589		"us"
10624	10590		],
10625		-	"checkType": "status_code",
10626	10591		"alexaRank": 242598,
10627		-	"url": "https://forum.quartertothree.com/u/{username}/summary",
	10592	+	"engine": "Discourse",
10628	10593		"urlMain": "https://forum.quartertothree.com",
10629	10594		"usernameClaimed": "rei",
10630	10595		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 289 lines
10920	10885		},
10921	10886		"Rasa": {
10922	10887		"tags": [
10923		-	"global",
10924	10888		"in",
10925	10889		"us"
10926	10890		],
10927		-	"checkType": "status_code",
10928	10891		"alexaRank": 62252,
10929		-	"url": "https://forum.rasa.com/u/{username}/summary",
	10892	+	"engine": "Discourse",
10930	10893		"urlMain": "https://forum.rasa.com",
10931	10894		"usernameClaimed": "adam",
10932	10895		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 423 lines
11356	11319		},
11357	11320		"Ruby-forum": {
11358	11321		"tags": [
11359		-	"global",
11360	11322		"us"
11361	11323		],
11362		-	"checkType": "status_code",
11363	11324		"alexaRank": 98718,
11364		-	"url": "https://www.ruby-forum.com/u/{username}/summary",
11365		-	"urlMain": "https://www.ruby-forum.com/",
	11325	+	"engine": "Discourse",
	11326	+	"urlMain": "https://www.ruby-forum.com",
11366	11327		"usernameClaimed": "tomconnolly",
11367	11328		"usernameUnclaimed": "noonewouldeverusethis7"
11368	11329		},
		skipped 91 lines
11460	11421		},
11461	11422		"Rust-lang": {
11462	11423		"tags": [
11463		-	"global",
	11424	+	"coding",
11464	11425		"us"
11465	11426		],
11466		-	"checkType": "status_code",
11467	11427		"alexaRank": 33437,
11468		-	"url": "https://users.rust-lang.org/u/{username}/summary",
	11428	+	"engine": "Discourse",
11469	11429		"urlMain": "https://users.rust-lang.org",
11470	11430		"usernameClaimed": "adam",
11471	11431		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 91 lines
11563	11523		},
11564	11524		"Scala-lang": {
11565	11525		"tags": [
11566		-	"ar",
11567		-	"global",
11568		-	"in",
11569		-	"us"
	11526	+	"coding"
11570	11527		],
11571		-	"checkType": "status_code",
11572	11528		"alexaRank": 65003,
11573		-	"url": "https://users.scala-lang.org/u/{username}/summary",
	11529	+	"engine": "Discourse",
11574	11530		"urlMain": "https://users.scala-lang.org",
11575	11531		"usernameClaimed": "sjrd",
11576	11532		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 333 lines
11910	11866		},
11911	11867		"Signal": {
11912	11868		"tags": [
11913		-	"il",
11914		-	"in",
11915		-	"us"
	11869	+	"tech"
11916	11870		],
11917		-	"checkType": "message",
11918		-	"absenceStrs": "Oops! That page doesn\u2019t exist or is private.",
11919	11871		"alexaRank": 190516,
11920		-	"url": "https://community.signalusers.org/u/{username}",
	11872	+	"engine": "Discourse",
11921	11873		"urlMain": "https://community.signalusers.org",
11922	11874		"usernameClaimed": "jlund",
11923	11875		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 2 lines
11926	11878		"tags": [
11927	11879		"us"
11928	11880		],
11929		-	"checkType": "status_code",
11930	11881		"alexaRank": 882176,
11931		-	"url": "https://www.silver-collector.com/u/{username}/summary",
	11882	+	"engine": "Discourse",
11932	11883		"urlMain": "https://www.silver-collector.com",
11933	11884		"usernameClaimed": "red",
11934	11885		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 644 lines
12579	12530		},
12580	12531		"SublimeForum": {
12581	12532		"tags": [
12582		-	"in"
	12533	+	"coding"
12583	12534		],
12584		-	"checkType": "status_code",
12585	12535		"alexaRank": 9068,
12586		-	"url": "https://forum.sublimetext.com/u/{username}",
	12536	+	"engine": "Discourse",
12587	12537		"urlMain": "https://forum.sublimetext.com/",
12588	12538		"usernameClaimed": "blue",
12589	12539		"usernameUnclaimed": "noonewouldeverusethis"
		skipped 731 lines
13321	13271		"tags": [
13322	13272		"us"
13323	13273		],
13324		-	"checkType": "status_code",
13325	13274		"alexaRank": 82,
13326		-	"url": "https://www.tradingview.com/u/{username}/",
	13275	+	"checkType": "message",
	13276	+	"absenceStrs": ["Sorry, that page doesn't exist!"],
	13277	+	"presenseStrs": ["tv-profile__name-text"],
	13278	+	"url": "https://www.tradingview.com/u/{username}",
13327	13279		"urlMain": "https://www.tradingview.com/",
13328	13280		"usernameClaimed": "blue",
13329	13281		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 50 lines
13380	13332		},
13381	13333		"Travis": {
13382	13334		"tags": [
13383		-	"global",
13384	13335		"us"
13385	13336		],
13386		-	"checkType": "status_code",
13387	13337		"alexaRank": 329414,
13388		-	"url": "https://travis-ci.community/u/{username}/summary",
13389		-	"urlMain": "https://travis-ci.community/",
	13338	+	"engine": "Discourse",
	13339	+	"urlMain": "https://travis-ci.community",
13390	13340		"usernameClaimed": "montana",
13391	13341		"usernameUnclaimed": "noonewouldeverusethis7"
13392	13342		},
		skipped 203 lines
13596	13546		"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
13597	13547		"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
13598	13548		"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
13599		-	"x-guest-token": "1360350238427480074"
	13549	+	"x-guest-token": "1361389689765326850"
13600	13550		},
13601	13551		"errors": {
13602	13552		"Bad guest token": "x-guest-token update required"
		skipped 30 lines
13633	13583		},
13634	13584		"UMHOOPS": {
13635	13585		"tags": [
13636		-	"global"
	13586	+	"sport"
13637	13587		],
13638		-	"checkType": "status_code",
13639	13588		"alexaRank": 1403001,
13640		-	"url": "https://forum.umhoops.com/u/{username}/summary",
	13589	+	"engine": "Discourse",
13641	13590		"urlMain": "https://forum.umhoops.com",
13642	13591		"usernameClaimed": "umhoops",
13643	13592		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 318 lines
13962	13911		"video"
13963	13912		],
13964	13913		"headers": {
13965		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTMxNjg3MDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.XZW2kEGAFCdU6C3QcTuu6QGhno3mwpXAMwWWj-4HK4w"
	13914	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTM0MTY1MDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.bMKafyWyx_AMCotYkNhquT6bDoTmbyG9YVcVrr_onO8"
13966	13915		},
13967	13916		"activation": {
13968	13917		"url": "https://vimeo.com/_rv/viewer",
		skipped 157 lines
14126	14075		"alexaRank": 45172,
14127	14076		"url": "https://www.votetags.info/author/{username}/",
14128	14077		"urlMain": "https://www.votetags.info/",
14129		-	"usernameClaimed": "red",
	14078	+	"usernameClaimed": "danphillip",
14130	14079		"usernameUnclaimed": "noonewouldeverusethis7"
14131	14080		},
14132	14081		"Vsemayki": {
		skipped 141 lines
14274	14223		"tags": [
14275	14224		"us"
14276	14225		],
14277		-	"checkType": "status_code",
14278	14226		"alexaRank": 1578,
14279		-	"url": "https://forum.waypoint.vice.com/u/{username}/summary",
	14227	+	"engine": "Discourse",
14280	14228		"urlMain": "https://forum.waypoint.vice.com",
14281	14229		"usernameClaimed": "red",
14282	14230		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 112 lines
14395	14343		},
14396	14344		"Whonix Forum": {
14397	14345		"tags": [
14398		-	"id",
14399		-	"in",
14400		-	"pk",
14401		-	"us"
	14346	+	"tech"
14402	14347		],
14403		-	"checkType": "status_code",
14404	14348		"alexaRank": 256100,
14405		-	"url": "https://forums.whonix.org/u/{username}",
	14349	+	"engine": "Discourse",
14406	14350		"urlMain": "https://forums.whonix.org/",
14407	14351		"usernameClaimed": "red",
14408	14352		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 1052 lines
15461	15405		"usernameUnclaimed": "noonewouldeverusethis7"
15462	15406		},
15463	15407		"catholic": {
	15408	+	"disabled": true,
15464	15409		"tags": [
15465	15410		"us"
15466	15411		],
15467		-	"checkType": "status_code",
15468	15412		"alexaRank": 64593,
15469		-	"url": "https://forums.catholic.com/u/{username}/summary",
	15413	+	"engine": "Discourse",
15470	15414		"urlMain": "https://forums.catholic.com",
15471	15415		"usernameClaimed": "blue",
15472	15416		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 121 lines
15594	15538		},
15595	15539		"community.asterisk.org": {
15596	15540		"tags": [
15597		-	"global",
15598	15541		"in",
15599	15542		"ir",
15600	15543		"jp"
15601	15544		],
15602		-	"checkType": "status_code",
15603	15545		"alexaRank": 50871,
15604		-	"url": "https://community.asterisk.org/u/{username}/summary",
	15546	+	"engine": "Discourse",
15605	15547		"urlMain": "https://community.asterisk.org",
15606	15548		"usernameClaimed": "bford",
15607	15549		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 2 lines
15610	15552		"tags": [
15611	15553		"global"
15612	15554		],
15613		-	"checkType": "status_code",
15614	15555		"alexaRank": 589898,
15615		-	"url": "https://community.p2pu.org/u/{username}/summary",
	15556	+	"engine": "Discourse",
15616	15557		"urlMain": "https://community.p2pu.org",
15617	15558		"usernameClaimed": "grif",
15618	15559		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 125 lines
15744	15685		},
15745	15686		"discourse.haskell.org": {
15746	15687		"tags": [
15747		-	"ca",
15748		-	"global",
15749		-	"in",
15750		-	"us"
	15688	+	"coding"
15751	15689		],
15752		-	"checkType": "status_code",
15753	15690		"alexaRank": 87660,
15754		-	"url": "https://discourse.haskell.org/u/{username}/summary",
	15691	+	"engine": "Discourse",
15755	15692		"urlMain": "https://discourse.haskell.org",
15756	15693		"usernameClaimed": "philipgaudreau",
15757	15694		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 362 lines
16120	16057		"tags": [
16121	16058		"us"
16122	16059		],
16123		-	"checkType": "status_code",
16124	16060		"alexaRank": 18564,
16125		-	"url": "https://forum.nameberry.com/u/{username}/summary",
	16061	+	"engine": "Discourse",
16126	16062		"urlMain": "https://forum.nameberry.com",
16127	16063		"usernameClaimed": "adam",
16128	16064		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 69 lines
16198	16134		"tags": [
16199	16135		"us"
16200	16136		],
16201		-	"checkType": "status_code",
16202	16137		"alexaRank": 4229,
16203		-	"url": "https://forums.docker.com/u/{username}/summary",
	16138	+	"engine": "Discourse",
16204	16139		"urlMain": "https://forums.docker.com",
16205	16140		"usernameClaimed": "dafritz84",
16206	16141		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 94 lines
16301	16236		"tags": [
16302	16237		"global"
16303	16238		],
16304		-	"checkType": "status_code",
16305	16239		"alexaRank": 3502263,
16306		-	"url": "https://forums.universaldashboard.io/u/{username}/summary",
	16240	+	"engine": "Discourse",
16307	16241		"urlMain": "https://forums.universaldashboard.io/",
16308	16242		"usernameClaimed": "adam",
16309	16243		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 33 lines
16343	16277		},
16344	16278		"funcom": {
16345	16279		"tags": [
16346		-	"eg",
16347		-	"global",
16348	16280		"us"
16349	16281		],
16350		-	"checkType": "status_code",
16351	16282		"alexaRank": 129454,
16352		-	"url": "https://forums.funcom.com/u/{username}/summary",
	16283	+	"engine": "Discourse",
16353	16284		"urlMain": "https://forums.funcom.com",
16354	16285		"usernameClaimed": "everqu",
16355	16286		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 169 lines
16525	16456		"ru",
16526	16457		"us"
16527	16458		],
16528		-	"checkType": "status_code",
16529	16459		"alexaRank": 11293,
16530		-	"url": "https://forum.hiveos.farm/u/{username}/summary",
	16460	+	"engine": "Discourse",
16531	16461		"urlMain": "https://forum.hiveos.farm",
16532	16462		"usernameClaimed": "halogenius",
16533	16463		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 186 lines
16720	16650		},
16721	16651		"juce": {
16722	16652		"tags": [
16723		-	"global",
16724	16653		"us"
16725	16654		],
16726		-	"checkType": "status_code",
16727	16655		"alexaRank": 238948,
16728		-	"url": "https://forum.juce.com/u/{username}",
	16656	+	"engine": "Discourse",
16729	16657		"urlMain": "https://forum.juce.com",
16730	16658		"usernameClaimed": "adam",
16731	16659		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 128 lines
16860	16788		"tags": [
16861	16789		"us"
16862	16790		],
16863		-	"checkType": "status_code",
16864	16791		"alexaRank": 38963,
16865		-	"url": "https://forum.leasehackr.com/u/{username}/summary/",
	16792	+	"engine": "Discourse",
16866	16793		"urlMain": "https://forum.leasehackr.com/",
16867	16794		"usernameClaimed": "adam",
16868	16795		"usernameUnclaimed": "noonewouldeverusethis"
		skipped 539 lines
17408	17335		"tags": [
17409	17336		"us"
17410	17337		],
17411		-	"checkType": "status_code",
	17338	+	"engine": "Discourse",
17412	17339		"alexaRank": 272199,
17413		-	"url": "https://forum.openframeworks.cc/u/{username}/summary",
17414	17340		"urlMain": "https://forum.openframeworks.cc",
17415	17341		"usernameClaimed": "red",
17416	17342		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 338 lines
17755	17681		},
17756	17682		"reverse4you": {
17757	17683		"tags": [
17758		-	"global",
17759	17684		"ru",
17760	17685		"ua"
17761	17686		],
17762		-	"checkType": "status_code",
17763	17687		"alexaRank": 1257899,
17764		-	"url": "https://forum.reverse4you.org/u/{username}/summary",
	17688	+	"engine": "Discourse",
17765	17689		"urlMain": "https://forum.reverse4you.org",
17766	17690		"usernameClaimed": "darwin",
17767	17691		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 244 lines
18012	17936		"tags": [
18013	17937		"us"
18014	17938		],
18015		-	"checkType": "status_code",
18016	17939		"alexaRank": 815504,
18017		-	"url": "https://forums.sourceruns.org/u/{username}/summary",
	17940	+	"engine": "Discourse",
18018	17941		"urlMain": "https://forums.sourceruns.org/",
18019	17942		"usernameClaimed": "cubedude",
18020	17943		"usernameUnclaimed": "noonewouldeverusethis7"
		skipped 120 lines
18141	18064		"tags": [
18142	18065		"ru"
18143	18066		],
18144		-	"checkType": "status_code",
18145	18067		"alexaRank": 446345,
18146		-	"url": "https://forum.swiftbook.ru/u/{username}/summary",
	18068	+	"engine": "Discourse",
18147	18069		"urlMain": "https://forum.swiftbook.ru",
18148		-	"usernameClaimed": "green",
	18070	+	"usernameClaimed": "alex",
18149	18071		"usernameUnclaimed": "noonewouldeverusethis7"
18150	18072		},
18151	18073		"takr-kiev.ucoz.com": {
		skipped 4534 lines
22686	22608		},
22687	22609		"community.getpostman.com": {
22688	22610		"tags": [
22689		-	"in"
	22611	+	"tech"
22690	22612		],
22691		-	"engine": "engine404",
22692	22613		"alexaRank": 10297,
22693		-	"url": "https://community.getpostman.com/u/{username}/",
	22614	+	"engine": "Discourse",
22694	22615		"urlMain": "https://community.getpostman.com",
22695	22616		"usernameUnclaimed": "noonewouldeverusethis7",
22696	22617		"usernameClaimed": "alex"
		skipped 35 lines
22732	22653		"tags": [
22733	22654		"us"
22734	22655		],
22735		-	"engine": "engine404",
22736	22656		"alexaRank": 3210,
22737		-	"url": "https://discuss.codecademy.com/u/{username}/summary",
	22657	+	"engine": "Discourse",
22738	22658		"urlMain": "https://discuss.codecademy.com",
22739	22659		"usernameUnclaimed": "noonewouldeverusethis7",
22740	22660		"usernameClaimed": "red"
		skipped 682 lines
23423	23343		},
23424	23344		"url": "{urlMain}{urlSubpath}/member.php?username={username}"
23425	23345		}
	23346	+	},
	23347	+	"Discourse": {
	23348	+	"name": "Discourse",
	23349	+	"site": {
	23350	+	"presenseStrs": [
	23351	+	"<meta name=\"generator\" content=\"Discourse"
	23352	+	],
	23353	+	"absenceStrs": [
	23354	+	"Oops! That page doesn\u2019t exist or is private.",
	23355	+	"wrap not-found-container"
	23356	+	],
	23357	+	"checkType": "message",
	23358	+	"url": "{urlMain}/u/{username}/summary"
	23359	+	},
	23360	+	"presenseStrs": [
	23361	+	"<meta name=\"generator\" content=\"Discourse"
	23362	+	]
23426	23363		},
23427	23364		"engine404": {
23428	23365		"name": "engine404",
		skipped 28 lines

■ ■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 310 lines
311	311
312	312		return self.load_from_json(data)
313	313
314		-	def get_stats(self, sites_dict):
	314	+	def get_scan_stats(self, sites_dict):
315	315		sites = sites_dict or self.sites_dict
316	316		found_flags = {}
317	317		for _, s in sites.items():
		skipped 3 lines
321	321
322	322		return found_flags
323	323
	324	+	def get_db_stats(self, sites_dict):
	325	+	if not sites_dict:
	326	+	sites_dict = self.sites_dict()
	327	+
	328	+	output = ''
	329	+	disabled_count = 0
	330	+	total_count = len(sites_dict)
	331	+	urls = {}
	332	+
	333	+	for _, site in sites_dict.items():
	334	+	if site.disabled:
	335	+	disabled_count += 1
	336	+
	337	+	url = URLMatcher.extract_main_part(site.url)
	338	+	if url.startswith('{username}'):
	339	+	url = 'SUBDOMAIN'
	340	+	elif url == '':
	341	+	url = f'{site.url} ({site.engine})'
	342	+	else:
	343	+	parts = url.split('/')
	344	+	url = '/' + '/'.join(parts[1:])
	345	+
	346	+	urls[url] = urls.get(url, 0) + 1
	347	+
	348	+	output += f'Enabled/total sites: {total_count-disabled_count}/{total_count}\n'
	349	+	output += 'Top sites\' profile URLs:\n'
	350	+	for url, count in sorted(urls.items(), key=lambda x: x[1], reverse=True)[:20]:
	351	+	if count == 1:
	352	+	break
	353	+	output += f'{count}\t{url}\n'
	354	+
	355	+	return output

Merge pull request #56 from soxoj/stats-discourse