STRLCPY/maigret

Merge pull request #139 from soxoj/photo-sites

Added some photo sites, improved errors detecting

soxoj committed with GitHub 3 years ago

6d4e2687

2 parents
d4234036
b696b982

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 6 files Show one by one

■ ■ ■ ■ ■ ■

maigret/checking.py

		skipped 12 lines
13	13		from aiohttp_socks import ProxyConnector
14	14		from python_socks import _errors as proxy_errors
15	15		from socid_extractor import extract
	16	+	from aiohttp.client_exceptions import ServerDisconnectedError, ClientConnectorError
16	17
17	18		from .activation import ParsingActivator, import_aiohttp_cookies
18	19		from . import errors
		skipped 45 lines
64	65
65	66		except asyncio.TimeoutError as e:
66	67		error = CheckError("Request timeout", str(e))
67		-	except aiohttp.client_exceptions.ClientConnectorError as e:
	68	+	except ClientConnectorError as e:
68	69		error = CheckError("Connecting failure", str(e))
	70	+	except ServerDisconnectedError as e:
	71	+	error = CheckError("Server disconnected", str(e))
69	72		except aiohttp.http_exceptions.BadHttpMessage as e:
70	73		error = CheckError("HTTP", str(e))
71	74		except proxy_errors.ProxyError as e:
		skipped 83 lines
155	158		# additional check for errors
156	159		if status_code and not check_error:
157	160		check_error = detect_error_page(
158		-	html_text, status_code, site.errors, site.ignore403
	161	+	html_text, status_code, site.errors_dict, site.ignore403
159	162		)
160	163
161	164		# parsing activation
		skipped 570 lines

■ ■ ■ ■ ■ ■

maigret/maigret.py

		skipped 47 lines
48	48		text = f'Too many errors of type "{e["err"]}" ({e["perc"]}%)'
49	49		solution = errors.solution_of(e['err'])
50	50		if solution:
51		-	text = '. '.join([text, solution])
	51	+	text = '. '.join([text, solution.capitalize()])
52	52
53	53		query_notify.warning(text, '!')
54	54		was_errs_displayed = True
		skipped 610 lines

■ ■ ■ ■ ■ ■ ■

maigret/resources/data.json

		skipped 8675 lines
8676	8676		},
8677	8677		"Mobypicture": {
8678	8678		"tags": [
8679		-	"in",
8680		-	"nl"
	8679	+	"photo"
8681	8680		],
8682	8681		"checkType": "message",
8683	8682		"absenceStrs": [
		skipped 793 lines
9477	9476		"usernameUnclaimed": "noonewouldeverusethis7"
9478	9477		},
9479	9478		"Newgrounds": {
9480		-	"tags": [
9481		-	"us"
	9479	+	"absenceStrs": [
	9480	+	"icon-steam"
9482	9481		],
9483		-	"regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
9484		-	"checkType": "status_code",
9485		-	"alexaRank": 5748,
	9482	+	"presenseStrs": [
	9483	+	"user-header-name"
	9484	+	],
	9485	+	"url": "https://{username}.newgrounds.com",
9486	9486		"urlMain": "https://newgrounds.com",
9487		-	"url": "https://{username}.newgrounds.com",
9488		-	"usernameClaimed": "blue",
9489		-	"usernameUnclaimed": "noonewouldeverusethis7"
	9487	+	"usernameClaimed": "john",
	9488	+	"usernameUnclaimed": "noonewouldeverusethis7",
	9489	+	"checkType": "message"
9490	9490		},
9491	9491		"Newreporter": {
9492	9492		"tags": [
		skipped 815 lines
10308	10308		},
10309	10309		"Picsart": {
10310	10310		"tags": [
10311		-	"in",
10312		-	"us"
	10311	+	"photo"
10313	10312		],
10314	10313		"checkType": "status_code",
10315	10314		"alexaRank": 9103,
		skipped 2439 lines
12755	12754		"us"
12756	12755		],
12757	12756		"headers": {
12758		-	"authorization": "Bearer BQC9jhqCGwU7SIBfusXOot9I_1wQ2F_4pS7ySsOwO20H9jWZE2ICJ-6rQ6gITvtIrXu6WbDLBTBUB4Ms1wI"
	12757	+	"authorization": "Bearer BQB2AjQkezkQ8gs4IPTf-DaiUD5xf0_agBYgfI8wGAI-tdC7GANMDZMwiYBls8kfD_bIhmtQoYwNfUgB3rg"
12759	12758		},
12760	12759		"errors": {
12761	12760		"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
		skipped 1378 lines
14140	14139		"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
14141	14140		"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
14142	14141		"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
14143		-	"x-guest-token": "1390788770346147842"
	14142	+	"x-guest-token": "1391077876263444481"
14144	14143		},
14145	14144		"errors": {
14146	14145		"Bad guest token": "x-guest-token update required"
		skipped 393 lines
14540	14539		"video"
14541	14540		],
14542	14541		"headers": {
14543		-	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjA0MjU3NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.hn10YMXkg1jk0qy4XBCxdfeN5kbWzjzJcYD8J84mjWk"
	14542	+	"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjA0OTU2NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.p2UxuvEPXrKk6wE_qMze9n9KeZIGo_uZ6nAE-UFKGJs"
14544	14543		},
14545	14544		"activation": {
14546	14545		"url": "https://vimeo.com/_rv/viewer",
		skipped 2521 lines
17068	17067		},
17069	17068		"gfycat": {
17070	17069		"tags": [
17071		-	"sharing"
	17070	+	"sharing",
	17071	+	"photo"
17072	17072		],
17073	17073		"checkType": "status_code",
17074	17074		"alexaRank": 2217,
		skipped 7349 lines
24424	24424		},
24425	24425		"1x": {
24426	24426		"tags": [
24427		-	"in",
24428		-	"photo",
24429		-	"us"
	24427	+	"photo"
24430	24428		],
24431	24429		"checkType": "message",
24432	24430		"presenseStrs": [
		skipped 1812 lines
26245	26243		"Cookie": "SUB=_2AkMXyuc_f8NxqwJRmP8SyWPrbo13zAvEieKhlhbkJRMxHRl-123"
26246	26244		},
26247	26245		"checkType": "message"
	26246	+	},
	26247	+	"Hatena": {
	26248	+	"absenceStrs": [
	26249	+	"404 Not Found"
	26250	+	],
	26251	+	"presenseStrs": [
	26252	+	"profile",
	26253	+	"myprofile",
	26254	+	"profile-dt",
	26255	+	"profile-dd",
	26256	+	"hatena-profile"
	26257	+	],
	26258	+	"url": "http://profile.hatena.com/{username}/",
	26259	+	"urlMain": "http://profile.hatena.com",
	26260	+	"usernameClaimed": "john",
	26261	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26262	+	"checkType": "message"
	26263	+	},
	26264	+	"angel.co": {
	26265	+	"absenceStrs": [
	26266	+	"render_not_found"
	26267	+	],
	26268	+	"presenseStrs": [
	26269	+	"Profile",
	26270	+	"profiles",
	26271	+	"User profile",
	26272	+	"name",
	26273	+	"layouts/profile"
	26274	+	],
	26275	+	"url": "https://angel.co/u/{username}",
	26276	+	"urlMain": "https://angel.co",
	26277	+	"usernameClaimed": "john",
	26278	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26279	+	"checkType": "message"
	26280	+	},
	26281	+	"nelubit.ru": {
	26282	+	"urlMain": "https://nelubit.ru",
	26283	+	"engine": "phpBB/Search",
	26284	+	"usernameClaimed": "alex",
	26285	+	"usernameUnclaimed": "noonewouldeverusethis7"
	26286	+	},
	26287	+	"actual-porn.org": {
	26288	+	"urlMain": "http://actual-porn.org",
	26289	+	"engine": "vBulletin",
	26290	+	"usernameClaimed": "alex",
	26291	+	"usernameUnclaimed": "noonewouldeverusethis7"
	26292	+	},
	26293	+	"Aminus3": {
	26294	+	"absenceStrs": [
	26295	+	"Expires",
	26296	+	" no-cache"
	26297	+	],
	26298	+	"presenseStrs": [
	26299	+	"image/ico",
	26300	+	" title="
	26301	+	],
	26302	+	"url": "https://{username}.aminus3.com/",
	26303	+	"urlMain": "https://aminus3.com",
	26304	+	"usernameClaimed": "beautifulworld",
	26305	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26306	+	"checkType": "message",
	26307	+	"tags": [
	26308	+	"photo"
	26309	+	]
	26310	+	},
	26311	+	"lomography": {
	26312	+	"absenceStrs": [
	26313	+	"<title>404 \u00b7 Lomography</title>"
	26314	+	],
	26315	+	"presenseStrs": [
	26316	+	"Lomography",
	26317	+	" @lomography"
	26318	+	],
	26319	+	"url": "https://www.lomography.com/homes/{username}",
	26320	+	"urlMain": "https://www.lomography.com",
	26321	+	"usernameClaimed": "steved7755",
	26322	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26323	+	"checkType": "message",
	26324	+	"tags": [
	26325	+	"photo"
	26326	+	]
	26327	+	},
	26328	+	"jAlbum.net": {
	26329	+	"absenceStrs": [
	26330	+	"section",
	26331	+	" error_head"
	26332	+	],
	26333	+	"regexCheck": "^[^\\.]+$",
	26334	+	"presenseStrs": [
	26335	+	"alternate",
	26336	+	" og:image"
	26337	+	],
	26338	+	"url": "https://{username}.jalbum.net/",
	26339	+	"urlMain": "https://jalbum.net",
	26340	+	"usernameClaimed": "laza",
	26341	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26342	+	"checkType": "message",
	26343	+	"tags": [
	26344	+	"photo"
	26345	+	]
	26346	+	},
	26347	+	"23hq": {
	26348	+	"absenceStrs": [
	26349	+	"my-modal",
	26350	+	" enable",
	26351	+	" modal"
	26352	+	],
	26353	+	"presenseStrs": [
	26354	+	"frame",
	26355	+	"first active",
	26356	+	"user",
	26357	+	"last",
	26358	+	"country-name"
	26359	+	],
	26360	+	"url": "http://www.23hq.com/{username}",
	26361	+	"urlMain": "http://www.23hq.com",
	26362	+	"usernameClaimed": "nellyb",
	26363	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26364	+	"checkType": "message",
	26365	+	"tags": [
	26366	+	"photo"
	26367	+	]
	26368	+	},
	26369	+	"bliphoto": {
	26370	+	"absenceStrs": [
	26371	+	"<title>Your photo journal \| Blipfoto</title>"
	26372	+	],
	26373	+	"presenseStrs": [
	26374	+	"biography",
	26375	+	"biography-full",
	26376	+	"profile-sidebar",
	26377	+	"profile-content",
	26378	+	"state"
	26379	+	],
	26380	+	"url": "https://www.blipfoto.com/{username}",
	26381	+	"urlMain": "https://www.blipfoto.com",
	26382	+	"usernameClaimed": "Wildstar",
	26383	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26384	+	"checkType": "message",
	26385	+	"tags": [
	26386	+	"photo"
	26387	+	]
	26388	+	},
	26389	+	"Fotki": {
	26390	+	"absenceStrs": [
	26391	+	"'404 - Member Not Found'"
	26392	+	],
	26393	+	"presenseStrs": [
	26394	+	"profile-cities",
	26395	+	"profile-friends",
	26396	+	"profile-aboutme",
	26397	+	"profile-country",
	26398	+	"user_profile_info"
	26399	+	],
	26400	+	"url": "https://members.fotki.com/{username}/about/",
	26401	+	"urlMain": "https://fotki.com",
	26402	+	"usernameClaimed": "normargab",
	26403	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26404	+	"checkType": "message",
	26405	+	"tags": [
	26406	+	"photo"
	26407	+	]
	26408	+	},
	26409	+	"viewbug": {
	26410	+	"absenceStrs": [
	26411	+	"missing-photos"
	26412	+	],
	26413	+	"presenseStrs": [
	26414	+	"profile",
	26415	+	" profile_content"
	26416	+	],
	26417	+	"url": "https://www.viewbug.com/member/{username}",
	26418	+	"urlMain": "https://www.viewbug.com",
	26419	+	"usernameClaimed": "melaniejwood",
	26420	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26421	+	"checkType": "message",
	26422	+	"tags": [
	26423	+	"photo"
	26424	+	]
	26425	+	},
	26426	+	"Piccsy": {
	26427	+	"absenceStrs": [
	26428	+	"my-modal"
	26429	+	],
	26430	+	"presenseStrs": [
	26431	+	"Username"
	26432	+	],
	26433	+	"regexCheck": "^[^\\.]+$",
	26434	+	"url": "http://{username}.piccsy.com/",
	26435	+	"urlMain": "http://piccsy.com",
	26436	+	"usernameClaimed": "orientcement",
	26437	+	"usernameUnclaimed": "noonewouldeverusethis7",
	26438	+	"checkType": "message",
	26439	+	"tags": [
	26440	+	"Photo"
	26441	+	]
26248	26442		}
26249	26443		},
26250	26444		"engines": {
		skipped 204 lines

■ ■ ■ ■ ■ ■

maigret/sites.py

		skipped 181 lines
182	182
183	183		return result
184	184
	185	+	@property
	186	+	def errors_dict(self) -> dict:
	187	+	errors: Dict[str, str] = {}
	188	+	if self.engine_obj:
	189	+	errors.update(self.engine_obj.site.get('errors', {}))
	190	+	errors.update(self.errors)
	191	+	return errors
	192	+
185	193		def get_url_type(self) -> str:
186	194		url = URLMatcher.extract_main_part(self.url)
187	195		if url.startswith("{username}"):
		skipped 281 lines

■ ■ ■ ■ ■ ■

maigret/submit.py

		skipped 199 lines
200	200		# cookies
201	201		cookie_dict = None
202	202		if cookie_file:
	203	+	logger.info(f'Use {cookie_file} for cookies')
203	204		cookie_jar = await import_aiohttp_cookies(cookie_file)
204	205		cookie_dict = {c.key: c.value for c in cookie_jar}
205	206
		skipped 122 lines
328	329		print(
329	330		"Try to run this mode again and increase features count or choose others."
330	331		)
	332	+	return False
331	333		else:
332	334		if (
333	335		input(
334	336		f"Site {chosen_site.name} successfully checked. Do you want to save it in the Maigret DB? [Yn] "
335		-	).lower()
336		-	in "y"
	337	+	)
	338	+	.lower()
	339	+	.strip("y")
337	340		):
338		-	logger.debug(chosen_site.json)
339		-	site_data = chosen_site.strip_engine_data()
340		-	logger.debug(site_data.json)
341		-	db.update_site(site_data)
342		-	return True
	341	+	return False
	342	+
	343	+	chosen_site.name = input("Change site name if you want: ") or chosen_site.name
	344	+	chosen_site.tags = input("Site tags: ").split(',')
343	345
344		-	return False
	346	+	logger.debug(chosen_site.json)
	347	+	site_data = chosen_site.strip_engine_data()
	348	+	logger.debug(site_data.json)
	349	+	db.update_site(site_data)
	350	+	return True
345	351

■ ■ ■ ■ ■ ■

tests/test_sites.py

		skipped 102 lines
103	103
104	104		amperka = db.sites[0]
105	105		assert len(amperka.errors) == 2
	106	+	assert len(amperka.errors_dict) == 2
106	107
107	108		assert amperka.strip_engine_data().errors == {'error1': 'text1'}
108	109		assert amperka.strip_engine_data().json['errors'] == {'error1': 'text1'}
		skipped 73 lines