STRLCPY/changedetection.io

■ ■ ■ ■ ■ ■

changedetectionio/__init__.py

		skipped 1443 lines
1444	1444		if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
1445	1445
1446	1446		# Proxies can be set to have a limit on seconds between which they can be called
1447		-	watch_proxy = watch.get('proxy')
1448		-	if not watch_proxy:
1449		-	watch_proxy = datastore.data['settings']['requests']['proxy']
1450		-	if not watch_proxy:
1451		-	watch_proxy = list(datastore.proxy_list.keys())[0]
1452		-
	1447	+	watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
1453	1448		if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):
1454	1449		# Proxy may also have some threshold minimum
1455	1450		proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
		skipped 36 lines

■ ■ ■ ■ ■ ■

changedetectionio/fetch_site_status.py

		skipped 19 lines
20	20		super().__init__(args, *kwargs)
21	21		self.datastore = datastore
22	22
23		-	# If there was a proxy list enabled, figure out what proxy_args/which proxy to use
24		-	# Returns the proxy as a URL
25		-	# if watch.proxy use that
26		-	# fetcher.proxy_override = watch.proxy or main config proxy
27		-	# Allows override the proxy on a per-request basis
28		-	# ALWAYS use the first one is nothing selected
29		-
30		-	def set_proxy_from_list(self, watch):
31		-	proxy_args = None
32		-	if self.datastore.proxy_list is None:
33		-	return None
34		-
35		-	# If its a valid one
36		-	if watch['proxy'] and watch['proxy'] in list(self.datastore.proxy_list.keys()):
37		-	proxy_args = self.datastore.proxy_list.get(watch['proxy']).get('url')
38		-
39		-	# not valid (including None), try the system one
40		-	else:
41		-	system_proxy = self.datastore.data['settings']['requests']['proxy']
42		-	# Is not None and exists
43		-	if self.datastore.proxy_list.get(system_proxy):
44		-	proxy_args = self.datastore.proxy_list.get(system_proxy).get('url')
45		-
46		-	# Fallback - Did not resolve anything, use the first available
47		-	if proxy_args is None:
48		-	first_default = list(self.datastore.proxy_list)[0]
49		-	proxy_args = self.datastore.proxy_list.get(first_default).get('url')
50		-
51		-	return proxy_args
52		-
53	23		# Doesn't look like python supports forward slash auto enclosure in re.findall
54	24		# So convert it to inline flag "foobar(?i)" type configuration
55	25		def forward_slash_enclosed_regex_to_options(self, regex):
		skipped 58 lines
114	84		# If the klass doesnt exist, just use a default
115	85		klass = getattr(content_fetcher, "html_requests")
116	86
117		-	proxy_url = self.set_proxy_from_list(watch)
118		-	if proxy_url:
	87	+	proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
	88	+	proxy_url = None
	89	+	if proxy_id:
	90	+	proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
119	91		print ("UUID {} Using proxy {}".format(uuid, proxy_url))
	92	+
120	93		fetcher = klass(proxy_override=proxy_url)
121	94
122	95		# Configurable per-watch or global extra delay before extracting text (for webDriver types)
		skipped 202 lines

■ ■ ■ ■ ■ ■

changedetectionio/run_all_tests.sh

		skipped 48 lines
49	49
50	50		unset PLAYWRIGHT_DRIVER_URL
51	51		docker kill $$-test_browserless
	52	+
	53	+	# Test proxy list handling, starting two squids on different ports
	54	+	# Each squid adds a different header to the response, which is the main thing we test for.
	55	+	docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge
	56	+	docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge
	57	+
	58	+
	59	+	# So, basic HTTP as env var test
	60	+	export HTTP_PROXY=http://localhost:3128
	61	+	export HTTPS_PROXY=http://localhost:3128
	62	+	pytest tests/proxy_list/test_proxy.py
	63	+	docker logs $$-squid-one 2>/dev/null\|grep one.changedetection.io
	64	+	if [ $? -ne 0 ]
	65	+	then
	66	+	echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)"
	67	+	fi
	68	+	unset HTTP_PROXY
	69	+	unset HTTPS_PROXY
	70	+
	71	+
	72	+	# 2nd test actually choose the preferred proxy from proxies.json
	73	+	cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json
	74	+	# Makes a watch use a preferred proxy
	75	+	pytest tests/proxy_list/test_multiple_proxy.py
	76	+
	77	+	# Should be a request in the default "first" squid
	78	+	docker logs $$-squid-one 2>/dev/null\|grep chosen.changedetection.io
	79	+	if [ $? -ne 0 ]
	80	+	then
	81	+	echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
	82	+	fi
	83	+
	84	+	# And one in the 'second' squid (user selects this as preferred)
	85	+	docker logs $$-squid-two 2>/dev/null\|grep chosen.changedetection.io
	86	+	if [ $? -ne 0 ]
	87	+	then
	88	+	echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
	89	+	fi
	90	+
	91	+	# @todo - test system override proxy selection and watch defaults, setup a 3rd squid?
	92	+	docker kill $$-squid-one
	93	+	docker kill $$-squid-two
	94	+
	95	+
	96	+

■ ■ ■ ■ ■ ■

changedetectionio/store.py

		skipped 439 lines
440	440		print ("Registered proxy list", list(self.proxy_list.keys()))
441	441
442	442
	443	+	def get_preferred_proxy_for_watch(self, uuid):
	444	+	"""
	445	+	Returns the preferred proxy by ID key
	446	+	:param uuid: UUID
	447	+	:return: proxy "key" id
	448	+	"""
	449	+
	450	+	proxy_id = None
	451	+	if self.proxy_list is None:
	452	+	return None
	453	+
	454	+	# If its a valid one
	455	+	watch = self.data['watching'].get(uuid)
	456	+
	457	+	if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
	458	+	return watch.get('proxy')
	459	+
	460	+	# not valid (including None), try the system one
	461	+	else:
	462	+	system_proxy_id = self.data['settings']['requests'].get('proxy')
	463	+	# Is not None and exists
	464	+	if self.proxy_list.get(system_proxy_id):
	465	+	return system_proxy_id
	466	+
	467	+	# Fallback - Did not resolve anything, use the first available
	468	+	if system_proxy_id is None:
	469	+	first_default = list(self.proxy_list)[0]
	470	+	return first_default
	471	+
	472	+	return None
443	473
444	474		# Run all updates
445	475		# IMPORTANT - Each update could be run even when they have a new install and the schema is correct
		skipped 105 lines

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/__init__.py

1	+	"""Tests for the app."""
2	+
3	+

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/conftest.py

1	+	#!/usr/bin/python3
2	+
3	+	from .. import conftest
4	+
5	+	#def pytest_addoption(parser):
6	+	# parser.addoption("--url_suffix", action="store", default="identifier for request")
7	+
8	+
9	+	#def pytest_generate_tests(metafunc):
10	+	# # This is called for every test. Only get/set command line arguments
11	+	# # if the argument is specified in the list of test "fixturenames".
12	+	# option_value = metafunc.config.option.url_suffix
13	+	# if 'url_suffix' in metafunc.fixturenames and option_value is not None:
14	+	# metafunc.parametrize("url_suffix", [option_value])

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/proxies.json-example

1	+	{
2	+	"proxy-one": {
3	+	"label": "One",
4	+	"url": "http://127.0.0.1:3128"
5	+	},
6	+	"proxy-two": {
7	+	"label": "two",
8	+	"url": "http://127.0.0.1:3129"
9	+	}
10	+	}
11	+

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/squid.conf

1	+	acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN)
2	+	acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN)
3	+	acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN)
4	+	acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines
5	+	acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN)
6	+	acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN)
7	+	acl localnet src fc00::/7 # RFC 4193 local private network range
8	+	acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines
9	+	acl localnet src 159.65.224.174
10	+	acl SSL_ports port 443
11	+	acl Safe_ports port 80 # http
12	+	acl Safe_ports port 21 # ftp
13	+	acl Safe_ports port 443 # https
14	+	acl Safe_ports port 70 # gopher
15	+	acl Safe_ports port 210 # wais
16	+	acl Safe_ports port 1025-65535 # unregistered ports
17	+	acl Safe_ports port 280 # http-mgmt
18	+	acl Safe_ports port 488 # gss-http
19	+	acl Safe_ports port 591 # filemaker
20	+	acl Safe_ports port 777 # multiling http
21	+	acl CONNECT method CONNECT
22	+
23	+	http_access deny !Safe_ports
24	+	http_access deny CONNECT !SSL_ports
25	+	http_access allow localhost manager
26	+	http_access deny manager
27	+	http_access allow localhost
28	+	http_access allow localnet
29	+	http_access deny all
30	+	http_port 3128
31	+	coredump_dir /var/spool/squid
32	+	refresh_pattern ^ftp: 1440 20% 10080
33	+	refresh_pattern ^gopher: 1440 0% 1440
34	+	refresh_pattern -i (/cgi-bin/\|\?) 0 0% 0
35	+	refresh_pattern \/(Packages\|Sources)(\|\.bz2\|\.gz\|\.xz)$ 0 0% 0 refresh-ims
36	+	refresh_pattern \/Release(\|\.gpg)$ 0 0% 0 refresh-ims
37	+	refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
38	+	refresh_pattern \/(Translation-.*)(\|\.bz2\|\.gz\|\.xz)$ 0 0% 0 refresh-ims
39	+	refresh_pattern . 0 20% 4320
40	+	logfile_rotate 0
41	+
42	+

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/test_multiple_proxy.py

1	+	#!/usr/bin/python3
2	+
3	+	import time
4	+	from flask import url_for
5	+	from ..util import live_server_setup
6	+
7	+	def test_preferred_proxy(client, live_server):
8	+	time.sleep(1)
9	+	live_server_setup(live_server)
10	+	time.sleep(1)
11	+	url = "http://chosen.changedetection.io"
12	+
13	+	res = client.post(
14	+	url_for("import_page"),
15	+	# Because a URL wont show in squid/proxy logs due it being SSLed
16	+	# Use plain HTTP or a specific domain-name here
17	+	data={"urls": url},
18	+	follow_redirects=True
19	+	)
20	+
21	+	assert b"1 Imported" in res.data
22	+
23	+	time.sleep(2)
24	+	res = client.post(
25	+	url_for("edit_page", uuid="first"),
26	+	data={
27	+	"css_filter": "",
28	+	"fetch_backend": "html_requests",
29	+	"headers": "",
30	+	"proxy": "proxy-two",
31	+	"tag": "",
32	+	"url": url,
33	+	},
34	+	follow_redirects=True
35	+	)
36	+	assert b"Updated watch." in res.data
37	+	time.sleep(2)
38	+	# Now the request should appear in the second-squid logs
39	+

■ ■ ■ ■ ■ ■

changedetectionio/tests/proxy_list/test_proxy.py

1	+	#!/usr/bin/python3
2	+
3	+	import time
4	+	from flask import url_for
5	+	from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
6	+
7	+	# just make a request, we will grep in the docker logs to see it actually got called
8	+	def test_check_basic_change_detection_functionality(client, live_server):
9	+	live_server_setup(live_server)
10	+	res = client.post(
11	+	url_for("import_page"),
12	+	# Because a URL wont show in squid/proxy logs due it being SSLed
13	+	# Use plain HTTP or a specific domain-name here
14	+	data={"urls": "http://one.changedetection.io"},
15	+	follow_redirects=True
16	+	)
17	+
18	+	assert b"1 Imported" in res.data
19	+	time.sleep(3)
20	+

■ ■ ■ ■ ■ ■

docker-compose.yml

		skipped 5 lines
6	6		hostname: changedetection
7	7		volumes:
8	8		- changedetection-data:/datastore
	9	+	# Configurable proxy list support, see https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#proxy-list-support
	10	+	# - ./proxies.json:/datastore/proxies.json
9	11
10	12		# environment:
11	13		# Default listening port, can also be changed with the -p option
		skipped 80 lines

Tidy up proxies.json logic, adding tests (#955)