STRLCPY/snscrape

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 1 files

■ ■ ■ ■ ■ ■

socialmediascraper/base.py

		skipped 48 lines
49	49		'''Iterator yielding Items.'''
50	50		pass
51	51
52		-	def _get(self, url, params = None, headers = None, timeout = 10, responseOkCallback = None):
	52	+	def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None):
53	53		for attempt in range(self._retries + 1):
54	54		# The request is newly prepared on each retry because of potential cookie updates.
55		-	req = self._session.prepare_request(requests.Request('GET', url, params = params, headers = headers))
	55	+	req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers))
56	56		logger.info(f'Retrieving {req.url}')
57	57		logger.debug(f'... with headers: {headers!r}')
	58	+	if data:
	59	+	logger.debug(f'... with data: {data!r}')
58	60		try:
59	61		r = self._session.send(req, timeout = timeout)
60	62		if responseOkCallback is None or responseOkCallback(r):
		skipped 10 lines
71	73		logger.fatal(msg)
72	74		raise ScraperException(msg)
73	75		raise RuntimeError('Reached unreachable code')
	76	+
	77	+	def _get(self, args, *kwargs):
	78	+	return self._request('GET', args, *kwargs)
	79	+
	80	+	def _post(self, args, *kwargs):
	81	+	return self._request('POST', args, *kwargs)
74	82
75	83		@classmethod
76	84		@abc.abstractmethod
		skipped 8 lines