STRLCPY/maigret

Tags updated, added tests for tags Added several sites Updated socid_extractor version to avoid bug #150
Soxoj committed 3 years ago

c9219d91

1 parent aa6cd0ec

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 13041 lines
13042	13042		"us"
13043	13043		],
13044	13044		"headers": {
13045		-	"authorization": "Bearer BQCOP-4T9UEvQLhkgFJ7d8Fyt65Nofw1U0l-pJaUHf0AbhOJv3pPtL6tOZtvtANYeoc4aXeMBhxTEoGk7eo"
	13045	+	"authorization": "Bearer BQA1fXOw4_oZPkyh94NHZm4Fwtwb5CJgmaep1bpLJ-Jvrrgi1-FSEsYwih7SyDAItMVDiSZVvn5pq7XTHuc"
13046	13046		},
13047	13047		"errors": {
13048	13048		"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
		skipped 1421 lines
14470	14470		"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
14471	14471		"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
14472	14472		"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
14473		-	"x-guest-token": "1392232718130466822"
	14473	+	"x-guest-token": "1393520076536549382"
14474	14474		},
14475	14475		"errors": {
14476	14476		"Bad guest token": "x-guest-token update required"
		skipped 400 lines
14877	14877		"video"
14878	14878		],
14879	14879		"headers": {
14880		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjA3NzAwNDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.ehmje1TJRvUpW-_wseK5uXNhHykq2jHHh1LBCGFGLyQ"
	14880	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjEwNzcwMDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.pM-K99b6xokwvi8NBFN3ZdG9jjS7vf54DVxv8yKDJEY"
14881	14881		},
14882	14882		"activation": {
14883	14883		"url": "https://vimeo.com/_rv/viewer",
		skipped 10386 lines
25270	25270		"usernameClaimed": "blue",
25271	25271		"usernameUnclaimed": "noonewouldeverusethis7",
25272	25272		"tags": [
25273		-	"ccbghdifbdchjinnlfejcitcuelgfncjnieuvccbcdut"
	25273	+	"networking"
25274	25274		]
25275	25275		},
25276	25276		"wblitz.net": {
		skipped 193 lines
25470	25470		"usernameClaimed": "ekostyle",
25471	25471		"usernameUnclaimed": "noonewouldeverusethis7",
25472	25472		"tags": [
25473		-	"ccbghdifbdchubbbbietggvjbujdlujekugftcgfdufj"
	25473	+	"blog"
25474	25474		]
25475	25475		},
25476	25476		"www.kinokopilka.pro": {
		skipped 1890 lines
27367	27367		"checkType": "message",
27368	27368		"alexaRank": 7237,
27369	27369		"tags": [
27370		-	"bussiness"
	27370	+	"business"
27371	27371		]
27372	27372		},
27373	27373		"nelubit.ru": {
		skipped 180 lines
27554	27554		"checkType": "message",
27555	27555		"alexaRank": 461,
27556	27556		"tags": [
27557		-	" stock",
	27557	+	"stock",
27558	27558		"photo"
27559	27559		]
27560	27560		},
		skipped 12 lines
27573	27573		"checkType": "message",
27574	27574		"alexaRank": 9936,
27575	27575		"tags": [
27576		-	" stock",
	27576	+	"stock",
27577	27577		"photo"
27578	27578		]
27579	27579		},
		skipped 4 lines
27584	27584		"usernameUnclaimed": "noonewouldeverusethis7",
27585	27585		"alexaRank": 83900,
27586	27586		"tags": [
27587		-	" photo",
	27587	+	"photo",
27588	27588		"blog"
27589	27589		]
27590	27590		},
		skipped 231 lines
27822	27822		"checkType": "message",
27823	27823		"alexaRank": 39894,
27824	27824		"tags": [
27825		-	" networking",
	27825	+	"networking",
27826	27826		"fashion"
	27827	+	]
	27828	+	},
	27829	+	"99designs.com": {
	27830	+	"absenceStrs": [
	27831	+	"mobile-only"
	27832	+	],
	27833	+	"presenseStrs": [
	27834	+	"profileUrl"
	27835	+	],
	27836	+	"url": "https://99designs.com/profiles/{username}",
	27837	+	"urlMain": "https://99designs.com",
	27838	+	"usernameClaimed": "t6s",
	27839	+	"usernameUnclaimed": "noonewouldeverusethis7",
	27840	+	"checkType": "message",
	27841	+	"alexaRank": 4149,
	27842	+	"tags": [
	27843	+	"design",
	27844	+	"photo"
	27845	+	]
	27846	+	},
	27847	+	"Expono": {
	27848	+	"absenceStrs": [
	27849	+	"404 - Page not found<"
	27850	+	],
	27851	+	"presenseStrs": [
	27852	+	"page-user-badge"
	27853	+	],
	27854	+	"url": "http://www.expono.com/{username}",
	27855	+	"urlMain": "http://www.expono.com",
	27856	+	"usernameClaimed": "snila",
	27857	+	"usernameUnclaimed": "noonewouldeverusethis7",
	27858	+	"checkType": "message",
	27859	+	"alexaRank": 155759,
	27860	+	"tags": [
	27861	+	"photo"
	27862	+	]
	27863	+	},
	27864	+	"picturepush.com": {
	27865	+	"absenceStrs": [
	27866	+	".stage img"
	27867	+	],
	27868	+	"presenseStrs": [
	27869	+	"loginname"
	27870	+	],
	27871	+	"url": "https://{username}.picturepush.com/",
	27872	+	"urlMain": "https://picturepush.com",
	27873	+	"usernameClaimed": "yoskark",
	27874	+	"usernameUnclaimed": "noonewouldeverusethis7",
	27875	+	"checkType": "message",
	27876	+	"alexaRank": 107053,
	27877	+	"tags": [
	27878	+	"photo"
	27879	+	]
	27880	+	},
	27881	+	"Purephoto": {
	27882	+	"absenceStrs": [
	27883	+	"Not found <span"
	27884	+	],
	27885	+	"presenseStrs": [
	27886	+	"profile_sidebar"
	27887	+	],
	27888	+	"url": "https://www.purephoto.com/{username}",
	27889	+	"urlMain": "https://www.purephoto.com",
	27890	+	"usernameClaimed": "garretsuhrie",
	27891	+	"usernameUnclaimed": "noonewouldeverusethis7",
	27892	+	"checkType": "message",
	27893	+	"alexaRank": 936968,
	27894	+	"tags": [
	27895	+	"photo"
27827	27896		]
27828	27897		}
27829	27898		},
		skipped 205 lines

■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 60 lines
61	61		"military",
62	62		"auto",
63	63		"gambling",
64		-	"business",
65	64		"cybercriminal",
66	65		"review",
	66	+	"bookmarks",
	67	+	"design",
67	68		]
68	69
69	70
		skipped 416 lines

■ ■ ■ ■ ■ ■

maigret/submit.py

		skipped 254 lines
255	255		features = input("If features was not detected correctly, write it manually: ")
256	256
257	257		if features:
258		-	presence_list = features.split(",")
	258	+	presence_list = list(map(str.strip, features.split(",")))
259	259
260	260		absence_list = sorted(b_minus_a, key=get_match_ratio, reverse=True)[
261	261		:top_features_count
		skipped 2 lines
264	264		features = input("If features was not detected correctly, write it manually: ")
265	265
266	266		if features:
267		-	absence_list = features.split(",")
	267	+	absence_list = list(map(str.strip, features.split(",")))
268	268
269	269		site_data = {
270	270		"absenceStrs": absence_list,
		skipped 84 lines
355	355		return False
356	356
357	357		chosen_site.name = input("Change site name if you want: ") or chosen_site.name
358		-	chosen_site.tags = input("Site tags: ").split(',')
	358	+	chosen_site.tags = list(map(str.strip, input("Site tags: ").split(',')))
359	359		rank = get_alexa_rank(chosen_site.url_main)
360	360		if rank:
361	361		print(f'New alexa rank: {rank}')
		skipped 8 lines

■ ■ ■ ■ ■ ■

requirements.txt

		skipped 25 lines
26	26		requests>=2.24.0
27	27		requests-futures==1.0.0
28	28		six==1.15.0
29		-	socid-extractor>=0.0.19
	29	+	socid-extractor>=0.0.20
30	30		soupsieve==2.1
31	31		stem==1.8.0
32	32		torrequest==0.1.0
		skipped 7 lines

■ ■ ■ ■ ■ ■

tests/test_data.py

1	+	"""Maigret data test functions"""
2	+
3	+	from maigret.utils import is_country_tag
4	+	from maigret.sites import SUPPORTED_TAGS
5	+
6	+
7	+	def test_tags_validity(default_db):
8	+	unknown_tags = set()
9	+
10	+	for site in default_db.sites:
11	+	for tag in filter(lambda x: not is_country_tag(x), site.tags):
12	+	if tag not in SUPPORTED_TAGS:
13	+	unknown_tags.add(tag)
14	+
15	+	assert unknown_tags == set()
16	+

Tags updated, added tests for tags Added several sites Updated socid_extractor version to avoid bug #150