STRLCPY/changedetection.io

Use proxies.json instead of proxies.txt - see wiki Proxies section (#945)
dgtlmoon committed with GitHub 2 years ago

2ba55bb4

1 parent 4c759490

■ ■ ■ ■ ■ ■ ■

changedetectionio/__init__.py

		skipped 546 lines
547	547
548	548		# Defaults for proxy choice
549	549		if datastore.proxy_list is not None: # When enabled
	550	+	# @todo
550	551		# Radio needs '' not None, or incase that the chosen one no longer exists
551	552		if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
552	553		default['proxy'] = ''
		skipped 7 lines
560	561		# @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
561	562		del form.proxy
562	563		else:
563		-	form.proxy.choices = [('', 'Default')] + datastore.proxy_list
	564	+	form.proxy.choices = [('', 'Default')]
	565	+	for p in datastore.proxy_list:
	566	+	form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
	567	+
564	568
565	569		if request.method == 'POST' and form.validate():
566	570		extra_update_obj = {}
		skipped 801 lines
1368	1372		import random
1369	1373		from changedetectionio import update_worker
1370	1374
	1375	+	proxy_last_called_time = {}
	1376	+
1371	1377		recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
1372	1378		print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
1373	1379
		skipped 54 lines
1428	1434		if watch.jitter_seconds == 0:
1429	1435		watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
1430	1436
1431		-
1432	1437		seconds_since_last_recheck = now - watch['last_checked']
	1438	+
1433	1439		if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
1434	1440		if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
	1441	+
	1442	+	# Proxies can be set to have a limit on seconds between which they can be called
	1443	+	watch_proxy = watch.get('proxy')
	1444	+	if watch_proxy and any([watch_proxy in p for p in datastore.proxy_list]):
	1445	+	# Proxy may also have some threshold minimum
	1446	+	proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
	1447	+	if proxy_list_reuse_time_minimum:
	1448	+	proxy_last_used_time = proxy_last_called_time.get(watch_proxy, 0)
	1449	+	time_since_proxy_used = time.time() - proxy_last_used_time
	1450	+	if time_since_proxy_used < proxy_list_reuse_time_minimum:
	1451	+	# Not enough time difference reached, skip this watch
	1452	+	print("Skipped UUID {} on proxy {}, not enough time between proxy requests".format(uuid, watch_proxy))
	1453	+	continue
	1454	+	else:
	1455	+	# Record the last used time
	1456	+	proxy_last_called_time[watch_proxy] = int(time.time())
	1457	+
1435	1458		# Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
1436	1459		priority = int(time.time())
1437	1460		print(
		skipped 19 lines

■ ■ ■ ■ ■ ■

changedetectionio/fetch_site_status.py

		skipped 20 lines
21	21		self.datastore = datastore
22	22
23	23		# If there was a proxy list enabled, figure out what proxy_args/which proxy to use
	24	+	# Returns the proxy as a URL
24	25		# if watch.proxy use that
25	26		# fetcher.proxy_override = watch.proxy or main config proxy
26	27		# Allows override the proxy on a per-request basis
		skipped 6 lines
33	34
34	35		# If its a valid one
35	36		if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
36		-	proxy_args = watch['proxy']
	37	+	proxy_args = self.datastore.proxy_list.get(watch['proxy']).get('url')
37	38
38	39		# not valid (including None), try the system one
39	40		else:
40	41		system_proxy = self.datastore.data['settings']['requests']['proxy']
41	42		# Is not None and exists
42		-	if any([system_proxy in p for p in self.datastore.proxy_list]):
43		-	proxy_args = system_proxy
	43	+	if self.datastore.proxy_list.get():
	44	+	proxy_args = self.datastore.proxy_list.get(system_proxy).get('url')
44	45
45	46		# Fallback - Did not resolve anything, use the first available
46	47		if proxy_args is None:
47		-	proxy_args = self.datastore.proxy_list[0][0]
	48	+	first_default = list(self.datastore.proxy_list)[0]
	49	+	proxy_args = self.datastore.proxy_list.get(first_default).get('url')
48	50
49	51		return proxy_args
50	52
		skipped 17 lines
68	70		stripped_text_from_html = ""
69	71
70	72		watch = self.datastore.data['watching'].get(uuid)
	73	+	if not watch:
	74	+	return
71	75
72	76		# Protect against file:// access
73	77		if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
		skipped 16 lines
90	94		if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
91	95		request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
92	96
93		-	timeout = self.datastore.data['settings']['requests']['timeout']
	97	+	timeout = self.datastore.data['settings']['requests'].get('timeout')
94	98		url = watch.get('url')
95	99		request_body = self.datastore.data['watching'][uuid].get('body')
96	100		request_method = self.datastore.data['watching'][uuid].get('method')
		skipped 13 lines
110	114		# If the klass doesnt exist, just use a default
111	115		klass = getattr(content_fetcher, "html_requests")
112	116
113		-
114		-	proxy_args = self.set_proxy_from_list(watch)
115		-	fetcher = klass(proxy_override=proxy_args)
	117	+	proxy_url = self.set_proxy_from_list(watch)
	118	+	if proxy_url:
	119	+	print ("UUID {} Using proxy {}".format(uuid, proxy_url))
	120	+	fetcher = klass(proxy_override=proxy_url)
116	121
117	122		# Configurable per-watch or global extra delay before extracting text (for webDriver types)
118	123		system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
		skipped 201 lines

■ ■ ■ ■ ■ ■

changedetectionio/store.py

		skipped 112 lines
113	113		self.__data['settings']['application']['api_access_token'] = secret
114	114
115	115		# Proxy list support - available as a selection in settings when text file is imported
116		-	# CSV list
117		-	# "name, address", or just "name"
118		-	proxy_list_file = "{}/proxies.txt".format(self.datastore_path)
	116	+	proxy_list_file = "{}/proxies.json".format(self.datastore_path)
119	117		if path.isfile(proxy_list_file):
120	118		self.import_proxy_list(proxy_list_file)
121	119
		skipped 315 lines
437	435		unlink(item)
438	436
439	437		def import_proxy_list(self, filename):
440		-	import csv
441		-	with open(filename, newline='') as f:
442		-	reader = csv.reader(f, skipinitialspace=True)
443		-	# @todo This loop can could be improved
444		-	l = []
445		-	for row in reader:
446		-	if len(row):
447		-	if len(row)>=2:
448		-	l.append(tuple(row[:2]))
449		-	else:
450		-	l.append(tuple([row[0], row[0]]))
451		-	self.proxy_list = l if len(l) else None
	438	+	with open(filename) as f:
	439	+	self.proxy_list = json.load(f)
	440	+	print ("Registered proxy list", list(self.proxy_list.keys()))
	441	+
452	442
453	443
454	444		# Run all updates
		skipped 106 lines

Use proxies.json instead of proxies.txt - see wiki Proxies section (#945)