STRLCPY/changedetection.io

Bug fix: Filter failure detection notification was interfering with change-detection results, added test case (#786)
dgtlmoon committed with GitHub 2 years ago

daae43e9

1 parent cdeedaa6

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

changedetectionio/tests/test_filter_exist_changes.py

1	+	#!/usr/bin/python3
2	+
3	+	# https://www.reddit.com/r/selfhosted/comments/wa89kp/comment/ii3a4g7/?context=3
4	+	import os
5	+	import time
6	+	from flask import url_for
7	+	from .util import set_original_response, live_server_setup
8	+	from changedetectionio.model import App
9	+
10	+
11	+	def set_response_without_filter():
12	+	test_return_data = """<html>
13	+	<body>
14	+	Some initial text</br>
15	+	<p>Which is across multiple lines</p>
16	+	</br>
17	+	So let's see what happens. </br>
18	+	<div id="nope-doesnt-exist">Some text thats the same</div>
19	+	</body>
20	+	</html>
21	+	"""
22	+
23	+	with open("test-datastore/endpoint-content.txt", "w") as f:
24	+	f.write(test_return_data)
25	+	return None
26	+
27	+
28	+	def set_response_with_filter():
29	+	test_return_data = """<html>
30	+	<body>
31	+	Some initial text</br>
32	+	<p>Which is across multiple lines</p>
33	+	</br>
34	+	So let's see what happens. </br>
35	+	<div class="ticket-available">Ticket now on sale!</div>
36	+	</body>
37	+	</html>
38	+	"""
39	+
40	+	with open("test-datastore/endpoint-content.txt", "w") as f:
41	+	f.write(test_return_data)
42	+	return None
43	+
44	+	def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server):
45	+	# Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
46	+	# And the page has that filter available
47	+	# Then I should get a notification
48	+
49	+	live_server_setup(live_server)
50	+
51	+	# Give the endpoint time to spin up
52	+	time.sleep(1)
53	+	set_response_without_filter()
54	+
55	+	# Add our URL to the import page
56	+	test_url = url_for('test_endpoint', _external=True)
57	+	res = client.post(
58	+	url_for("form_quick_watch_add"),
59	+	data={"url": test_url, "tag": 'cinema'},
60	+	follow_redirects=True
61	+	)
62	+	assert b"Watch added" in res.data
63	+
64	+	# Give the thread time to pick up the first version
65	+	time.sleep(3)
66	+
67	+	# Goto the edit page, add our ignore text
68	+	# Add our URL to the import page
69	+	url = url_for('test_notification_endpoint', _external=True)
70	+	notification_url = url.replace('http', 'json')
71	+
72	+	print(">>>> Notification URL: " + notification_url)
73	+
74	+	# Just a regular notification setting, this will be used by the special 'filter not found' notification
75	+	notification_form_data = {"notification_urls": notification_url,
76	+	"notification_title": "New ChangeDetection.io Notification - {watch_url}",
77	+	"notification_body": "BASE URL: {base_url}\n"
78	+	"Watch URL: {watch_url}\n"
79	+	"Watch UUID: {watch_uuid}\n"
80	+	"Watch title: {watch_title}\n"
81	+	"Watch tag: {watch_tag}\n"
82	+	"Preview: {preview_url}\n"
83	+	"Diff URL: {diff_url}\n"
84	+	"Snapshot: {current_snapshot}\n"
85	+	"Diff: {diff}\n"
86	+	"Diff Full: {diff_full}\n"
87	+	":-)",
88	+	"notification_format": "Text"}
89	+
90	+	notification_form_data.update({
91	+	"url": test_url,
92	+	"tag": "my tag",
93	+	"title": "my title",
94	+	"headers": "",
95	+	"css_filter": '.ticket-available',
96	+	"fetch_backend": "html_requests"})
97	+
98	+	res = client.post(
99	+	url_for("edit_page", uuid="first"),
100	+	data=notification_form_data,
101	+	follow_redirects=True
102	+	)
103	+	assert b"Updated watch." in res.data
104	+	time.sleep(3)
105	+
106	+	# Shouldn't exist, shouldn't have fired
107	+	assert not os.path.isfile("test-datastore/notification.txt")
108	+	# Now the filter should exist
109	+	set_response_with_filter()
110	+	client.get(url_for("form_watch_checknow"), follow_redirects=True)
111	+	time.sleep(3)
112	+
113	+	assert os.path.isfile("test-datastore/notification.txt")
114	+
115	+	with open("test-datastore/notification.txt", 'r') as f:
116	+	notification = f.read()
117	+
118	+	assert 'Ticket now on sale' in notification
119	+	os.unlink("test-datastore/notification.txt")
120	+
121	+
122	+	# Test that if it gets removed, then re-added, we get a notification
123	+	# Remove the target and re-add it, we should get a new notification
124	+	set_response_without_filter()
125	+	client.get(url_for("form_watch_checknow"), follow_redirects=True)
126	+	time.sleep(3)
127	+	assert not os.path.isfile("test-datastore/notification.txt")
128	+
129	+	set_response_with_filter()
130	+	client.get(url_for("form_watch_checknow"), follow_redirects=True)
131	+	time.sleep(3)
132	+	assert os.path.isfile("test-datastore/notification.txt")
133	+
134	+	# Also test that the filter was updated after the first one was requested
135	+

■ ■ ■ ■ ■ ■

changedetectionio/tests/test_filter_failure_notification.py

		skipped 25 lines
26	26
27	27		# Give the endpoint time to spin up
28	28		time.sleep(1)
	29	+	# cleanup for the next
	30	+	client.get(
	31	+	url_for("form_delete", uuid="all"),
	32	+	follow_redirects=True
	33	+	)
	34	+	if os.path.isfile("test-datastore/notification.txt"):
	35	+	os.unlink("test-datastore/notification.txt")
29	36
30	37		# Add our URL to the import page
31	38		test_url = url_for('test_endpoint', _external=True)
		skipped 2 lines
34	41		data={"url": test_url, "tag": ''},
35	42		follow_redirects=True
36	43		)
	44	+
37	45		assert b"Watch added" in res.data
38	46
39	47		# Give the thread time to pick up the first version
		skipped 27 lines
67	75		"tag": "my tag",
68	76		"title": "my title",
69	77		"headers": "",
	78	+	"filter_failure_notification_send": 'y',
70	79		"css_filter": content_filter,
71	80		"fetch_backend": "html_requests"})
72	81
		skipped 13 lines
86	95		time.sleep(3)
87	96
88	97		# We should see something in the frontend
89		-	assert b'Did the page change its layout' in res.data
	98	+	assert b'Warning, filter' in res.data
90	99
91	100		# Now it should exist and contain our "filter not found" alert
92	101		assert os.path.isfile("test-datastore/notification.txt")
		skipped 39 lines
132	141		time.sleep(1)
133	142		run_filter_test(client, '//*[@id="nope-doesnt-exist"]')
134	143
135		-
	144	+	# Test that notification is never sent

■ ■ ■ ■ ■ ■

changedetectionio/update_worker.py

		skipped 64 lines
65	65		if uuid in list(self.datastore.data['watching'].keys()):
66	66
67	67		changed_detected = False
68		-	contents = ""
	68	+	contents = b''
69	69		screenshot = False
70	70		update_obj= {}
71	71		xpath_data = False
	72	+	process_changedetection_results = True
	73	+
72	74		now = time.time()
73	75
74	76		try:
		skipped 5 lines
80	82		raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
81	83		except PermissionError as e:
82	84		self.app.logger.error("File permission error updating", uuid, str(e))
	85	+	process_changedetection_results = False
83	86		except content_fetcher.ReplyWithContentButNoText as e:
84	87		# Totally fine, it's by choice - just continue on, nothing more to care about
85	88		# Page had elements/content but no renderable text
	89	+	# Backend (not filters) gave zero output
86	90		self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found."})
	91	+	process_changedetection_results = False
	92	+
87	93		except FilterNotFoundInResponse as e:
88		-	err_text = "Filter '{}' not found - Did the page change its layout?".format(str(e))
89		-	c = 0
90		-	if self.datastore.data['watching'].get(uuid, False):
	94	+	err_text = "Warning, filter '{}' not found".format(str(e))
	95	+	self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
	96	+	# So that we get a trigger when the content is added again
	97	+	'previous_md5': ''})
	98	+
	99	+	# Only when enabled, send the notification
	100	+	if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
91	101		c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
92		-	c += 1
	102	+	c += 1
	103	+	# Send notification if we reached the threshold?
	104	+	threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
	105	+	0)
	106	+	print("Filter for {} not found, consecutive_filter_failures: {}".format(uuid, c))
	107	+	if threshold > 0 and c >= threshold:
	108	+	self.send_filter_failure_notification(uuid)
	109	+	c = 0
	110	+	self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
93	111
94		-	# Send notification if we reached the threshold?
95		-	threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
96		-	print("Filter for {} not found, consecutive_filter_failures: {}".format(uuid, c))
97		-	if threshold >0 and c >= threshold:
98		-	self.send_filter_failure_notification(uuid)
99		-	c = 0
	112	+	process_changedetection_results = True
100	113
101		-	self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
102		-	'consecutive_filter_failures': c})
103	114		except content_fetcher.EmptyReply as e:
104	115		# Some kind of custom to-str handler in the exception handler that does this?
105	116		err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
		skipped 3 lines
109	120		err_text = "Screenshot unavailable, page did not render fully in the expected time - try increasing 'Wait seconds before extracting text'"
110	121		self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
111	122		'last_check_status': e.status_code})
	123	+	process_changedetection_results = False
112	124		except content_fetcher.PageUnloadable as e:
113	125		err_text = "Page request from server didnt respond correctly"
114	126		self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
		skipped 1 lines
116	128		except Exception as e:
117	129		self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
118	130		self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
	131	+	# Other serious error
	132	+	process_changedetection_results = False
	133	+	else:
	134	+	# Mark that we never had any failures
	135	+	update_obj['consecutive_filter_failures'] = 0
119	136
120		-	else:
	137	+	# Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
	138	+	if process_changedetection_results:
121	139		try:
122	140		watch = self.datastore.data['watching'][uuid]
123	141		fname = "" # Saved history text filename
		skipped 3 lines
127	145		# A change was detected
128	146		fname = watch.save_history_text(contents=contents, timestamp=str(round(time.time())))
129	147
130		-	# Generally update anything interesting returned
131		-	update_obj['consecutive_filter_failures'] = 0
132	148		self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
133	149
134	150		# A change was detected
		skipped 59 lines
194	210		self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
195	211		self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
196	212
197		-	finally:
	213	+
198	214		# Always record that we atleast tried
199	215		self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
200	216		'last_checked': round(time.time())})
		skipped 16 lines

Bug fix: Filter failure detection notification was interfering with change-detection results, added test case (#786)