STRLCPY/changedetection.io

Tidy up proxy logic
dgtlmoon committed 2 years ago

d4d06973

1 parent 77fdf59a

■ ■ ■ ■ ■ ■

changedetectionio/__init__.py

		skipped 1443 lines
1444	1444		if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
1445	1445
1446	1446		# Proxies can be set to have a limit on seconds between which they can be called
1447		-	watch_proxy = watch.get('proxy')
1448		-	if not watch_proxy:
1449		-	watch_proxy = datastore.data['settings']['requests']['proxy']
1450		-	if not watch_proxy:
1451		-	watch_proxy = list(datastore.proxy_list.keys())[0]
1452		-
	1447	+	watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
1453	1448		if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):
1454	1449		# Proxy may also have some threshold minimum
1455	1450		proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
		skipped 36 lines

■ ■ ■ ■ ■ ■

changedetectionio/fetch_site_status.py

		skipped 19 lines
20	20		super().__init__(args, *kwargs)
21	21		self.datastore = datastore
22	22
23		-	# If there was a proxy list enabled, figure out what proxy_args/which proxy to use
24		-	# Returns the proxy as a URL
25		-	# if watch.proxy use that
26		-	# fetcher.proxy_override = watch.proxy or main config proxy
27		-	# Allows override the proxy on a per-request basis
28		-	# ALWAYS use the first one is nothing selected
29		-
30		-	def set_proxy_from_list(self, watch):
31		-	proxy_args = None
32		-	if self.datastore.proxy_list is None:
33		-	return None
34		-
35		-	# If its a valid one
36		-	if watch['proxy'] and watch['proxy'] in list(self.datastore.proxy_list.keys()):
37		-	proxy_args = self.datastore.proxy_list.get(watch['proxy']).get('url')
38		-
39		-	# not valid (including None), try the system one
40		-	else:
41		-	system_proxy = self.datastore.data['settings']['requests']['proxy']
42		-	# Is not None and exists
43		-	if self.datastore.proxy_list.get(system_proxy):
44		-	proxy_args = self.datastore.proxy_list.get(system_proxy).get('url')
45		-
46		-	# Fallback - Did not resolve anything, use the first available
47		-	if proxy_args is None:
48		-	first_default = list(self.datastore.proxy_list)[0]
49		-	proxy_args = self.datastore.proxy_list.get(first_default).get('url')
50		-
51		-	return proxy_args
52		-
53	23		# Doesn't look like python supports forward slash auto enclosure in re.findall
54	24		# So convert it to inline flag "foobar(?i)" type configuration
55	25		def forward_slash_enclosed_regex_to_options(self, regex):
		skipped 58 lines
114	84		# If the klass doesnt exist, just use a default
115	85		klass = getattr(content_fetcher, "html_requests")
116	86
117		-	proxy_url = self.set_proxy_from_list(watch)
118		-	if proxy_url:
	87	+	proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
	88	+	proxy_url = None
	89	+	if proxy_id:
	90	+	proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
119	91		print ("UUID {} Using proxy {}".format(uuid, proxy_url))
	92	+
120	93		fetcher = klass(proxy_override=proxy_url)
121	94
122	95		# Configurable per-watch or global extra delay before extracting text (for webDriver types)
		skipped 202 lines

■ ■ ■ ■ ■ ■

changedetectionio/store.py

		skipped 439 lines
440	440		print ("Registered proxy list", list(self.proxy_list.keys()))
441	441
442	442
	443	+	def get_preferred_proxy_for_watch(self, uuid):
	444	+	"""
	445	+	Returns the preferred proxy by ID key
	446	+	:param uuid: UUID
	447	+	:return: proxy "key" id
	448	+	"""
	449	+
	450	+	proxy_id = None
	451	+	if self.proxy_list is None:
	452	+	return None
	453	+
	454	+	# If its a valid one
	455	+	watch = self.data['watching'].get(uuid)
	456	+
	457	+	if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
	458	+	return watch.get('proxy')
	459	+
	460	+	# not valid (including None), try the system one
	461	+	else:
	462	+	system_proxy_id = self.data['settings']['requests'].get('proxy')
	463	+	# Is not None and exists
	464	+	if self.proxy_list.get(system_proxy_id):
	465	+	return system_proxy_id
	466	+
	467	+	# Fallback - Did not resolve anything, use the first available
	468	+	if system_proxy_id is None:
	469	+	first_default = list(self.proxy_list)[0]
	470	+	return first_default
	471	+
	472	+	return None
443	473
444	474		# Run all updates
445	475		# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
		skipped 105 lines

Tidy up proxy logic