STRLCPY/Maryam

■ ■ ■ ■ ■ ■

core/core.py

		skipped 47 lines
48	48		_loaded_modules = {}
49	49		_cat_module_names = {}
50	50		_module_names = []
	51	+	_error_stack = []
51	52		_history_file = ''
52	53		workspace = ''
53	54		variables = {}
		skipped 84 lines
138	139		# OUTPUT //
139	140		# ////////////////////////////////
140	141
141		-	def print_exception(self, line=''):
	142	+	def print_exception(self, line='', where='nil', which_func='nil'):
142	143		stack_list = [x.strip() for x in traceback.format_exc().strip().splitlines()]
143	144		message = stack_list[-1]
144	145		if self._global_options['verbosity'] == 0:
145	146		return
146	147		elif self._global_options['verbosity'] == 1:
147	148		line = ' '.join([x for x in [message, line] if x])
148		-	self.error(stack_list[-3])
149		-	self.error(line)
150		-	elif self._global_options['verbosity'] == 2:
	149	+	self.error(stack_list[-3], where, which_func)
	150	+	self.error(line, where, which_func)
	151	+	elif self._global_options['verbosity'] >= 2:
151	152		print(f"{Colors.R}{'-'*60}")
152	153		traceback.print_exc()
153	154		print(f"{'-'*60}{Colors.N}")
		skipped 6 lines
160	161		size -= 1
161	162		return '\n'.join([prefix + line for line in wrap(string, size)])
162	163
163		-	def error(self, line):
	164	+	def error(self, line, where='nil', which_func='nil'):
164	165		'''Formats and presents errors.'''
165	166		if not re.search(r'[.,;!?]$', line):
166	167		line += '.'
167	168		line = line[:1].upper() + line[1:]
168		-	print(f"{Colors.R}[!] {line}{Colors.N}")
	169	+	error = f"[{where}:{which_func}] {line}"
	170	+	print(f"{Colors.O}[!] {error}{Colors.N}")
	171	+	if error not in self._error_stack:
	172	+	self._error_stack.append(error)
169	173
170	174		def output(self, line, color='N', end='', prep='', linesep=True):
171	175		'''Formats and presents normal output.'''
		skipped 200 lines
372	376		file.close()
373	377		return filename
374	378		return False
	379	+
	380	+	# ////////////////////////////////
	381	+	# HISTORY //
	382	+	# ////////////////////////////////
	383	+
	384	+	def _reset_error_stack(self):
	385	+	self._error_stack.clear()
375	386
376	387		# ////////////////////////////////
377	388		# HISTORY //
		skipped 614 lines

■ ■ ■ ■ ■ ■

core/initial.py

		skipped 14 lines
15	15		along with this program. If not, see <http://www.gnu.org/licenses/>.
16	16		"""
17	17
18		-	__version__ = 'v2.0.0'
	18	+	__version__ = 'v2.0.5'
19	19		import argparse
20	20		import imp
21	21		import os
		skipped 242 lines
264	264		output = output.strip().replace('\n', ' ').replace('\\x', ' ')
265	265		self.output(f"{prefix*depth}{output}", color)
266	266
	267	+	def search_engine_results(self, output):
	268	+	for i in output['results']:
	269	+	self.output(i['title'])
	270	+	self.output(i['a'])
	271	+	self.output(i['cite'])
	272	+	self.output(i['content'])
	273	+	print()
	274	+
267	275		def thread(self, *args):
268	276		""" self, function, thread_count, engines, {...all args}, sources"""
269	277		with concurrent.futures.ThreadPoolExecutor(max_workers=args[1]) as executor:
		skipped 67 lines
337	345		# Turn off framework prints till the executing of module
338	346		with turn_off():
339	347		results = mod.module_api(self)
	348	+	results['errors'] = self._error_stack
	349	+	self._reset_error_stack()
340	350		if self.options['format']:
341	351		print(json.dumps(results, indent=4))
342	352		else:
		skipped 3 lines
346	356		except KeyboardInterrupt:
347	357		pass
348	358		except Exception as e:
349		-	self.error(f"{tool_name.title()}CodeError: {e}")
	359	+	self.print_exception(where=tool_name, which_func='running')
350	360
351	361		def run_tool(self, func, tool_name, args, output=None):
352	362		try:
		skipped 13 lines

■ ■ ■ ■ ■ ■

core/util/activesearch.py

		skipped 24 lines
25	25		def __init__(self, q, limit=15):
26	26		""" activesearchresults.com search engine
27	27
28		-	q : query for search
29		-	limit : maximum result count
	28	+	q : Query for search
	29	+	limit : Maximum result count
30	30		"""
31	31		self.framework = main.framework
32	32		self.q = q
		skipped 34 lines
67	67		headers=header
68	68		)
69	69		except:
70		-	self.framework.error('[ACTIVESEARCH] ConnectionError')
71		-	self.framework.error('activesearchresults is missed!')
	70	+	self.framework.error('ConnectionError', 'util/activesearch', 'run_crawl')
	71	+	self.framework.error('activesearchresults is missed!', 'util/activesearch', 'run_crawl')
72	72		return
73	73
74	74		self._rawhtml += req.text
		skipped 60 lines

■ ■ ■ ■ ■ ■

core/util/ahmia.py

		skipped 35 lines
36	36		req = self.framework.request(url=url)
37	37		except:
38	38		self.framework.print_exception()
39		-	self.framework.error('Ahmia is missed!')
	39	+	self.framework.error('Ahmia is missed!', 'util/ahmia', 'run_crawl')
40	40		self._pages = req.text
41	41
42	42		@property
		skipped 7 lines

■ ■ ■ ■ ■ ■

core/util/arxiv.py

		skipped 38 lines
39	39		self.q = urllib.parse.quote_plus(self.q)
40	40		url = f'https://export.arxiv.org/api/query?search_query=all:'\
41	41		+ f'{self.q}&start=0&max_results={self.max}'
42		-	self.framework.verbose('Searching the arxiv.org domain...')
	42	+	self.framework.verbose('[ARXIV] Searching the arxiv.org domain...')
43	43		try:
44	44		req = self.framework.request(url=url)
45	45		except:
46		-	self.framework.error('[ARXIV] ConnectionError')
47		-	self.framework.error('ArXiv is missed!')
	46	+	self.framework.error('ConnectionError', 'arxiv', 'run_crawl')
	47	+	self.framework.error('ArXiv is missed!', 'arxiv', 'run_crawl')
48	48		return
49	49		self._rawxml = req.text
50	50		self._articles = list(re.findall(r'<entry>(.*?)</entry>',
		skipped 31 lines

■ ■ ■ ■ ■ ■

core/util/ask.py

		skipped 19 lines
20	20		def __init__(self, q, limit=2):
21	21		""" ask.com search engine
22	22
23		-	q : query for search
24		-	limit : count of pages
	23	+	q : Query for search
	24	+	limit : Number of pages
25	25		"""
26	26		self.framework = main.framework
27	27		self.q = self.framework.urlib(q).quote
		skipped 9 lines
37	37		try:
38	38		req = self.framework.request(url=urls[url])
39	39		except:
40		-	self.framework.error('[ASK] ConnectionError')
	40	+	self.framework.error('ConnectionError', 'util/ask', 'run_crawl')
41	41		max_attempt -= 1
42	42		if max_attempt == 0:
43		-	self.framework.error('Ask is missed!')
	43	+	self.framework.error('Ask is missed!', 'util/ask', 'run_crawl')
44	44		break
45	45		else:
46	46		page = req.text
		skipped 21 lines

■ ■ ■ ■ ■ ■

core/util/baidu.py

		skipped 21 lines
22	22		def __init__(self, q, limit=2):
23	23		""" baidu.com search engine
24	24
25		-	q : query for search
26		-	limit : count of pages
	25	+	q : Query for search
	26	+	limit : Number of pages
27	27		"""
28	28		self.framework = main.framework
29	29		self.q = self.framework.urlib(q).quote
		skipped 10 lines
40	40		try:
41	41		req = self.framework.request(url=urls[url])
42	42		except:
43		-	self.framework.error('[BAIDU] ConnectionError')
	43	+	self.framework.error('ConnectionError', 'util/baidu', 'run_crawl')
44	44		max_attempt -= 1
45	45		if max_attempt == 0:
46		-	self.framework.error('Baidu is missed!')
	46	+	self.framework.error('Baidu is missed!', 'util/baidu', 'run_crawl')
47	47		break
48	48		else:
49	49		self._pages += req.text
		skipped 17 lines

■ ■ ■ ■ ■ ■

core/util/bing.py

		skipped 40 lines
41	41		try:
42	42		req = self.framework.request(url=urls[url], allow_redirects=True)
43	43		except:
44		-	self.framework.error('[BING] ConnectionError')
	44	+	self.framework.error('ConnectionError', 'util/bing', 'run_crawl')
45	45		max_attempt -= 1
46	46		if max_attempt == 0:
47		-	self.framework.error('Bing is missed!')
	47	+	self.framework.error('Bing is missed!', 'util/bing', 'run_crawl')
48	48		break
49	49		else:
50	50		page = req.text
		skipped 47 lines

■ ■ ■ ■ ■ ■

core/util/bing_mobile_view.py

		skipped 30 lines
31	31		def screenshot(self):
32	32		self.framework.verbose('[Bing Mobile View] Fetching mobile view of the URL...')
33	33		bing_api_url = 'https://www.bing.com/webmaster/tools/mobile-friendliness-result'
34		-
35	34		self.framework._global_options['rand_agent'] = True
36		-
37	35		try:
38	36		response = self.framework.request(url=bing_api_url, method='POST', \
39	37		data={'url': self.url, 'retry': '0'}, timeout=40).text # Setting high timeout of as some req take long
40	38		except:
41		-	self.framework.error('[Bing Mobile View] ConnectionError.')
	39	+	self.framework.error('ConnectionError.', 'util/bing_mobile_view', 'screenshot')
42	40		return False
43	41		else:
44	42
45	43		if 'Bing Webmaster services could not be reached' in response :
46		-	self.framework.error('[Bing Mobile View] Bing Webmaster services could not be reached')
	44	+	self.framework.error('Bing Webmaster services could not be reached', 'util/bing_mobile_view', 'screenshot')
47	45		return False
48	46
49	47		self._raw_image_data = re.findall(r'data\:image.*"', response) # Regex for base64 encoded image
		skipped 8 lines

■ ■ ■ ■ ■ ■

core/util/carrot2.py

		skipped 22 lines
23	23		def __init__(self, q):
24	24		""" carrot2.org search engine
25	25
26		-	q : query for search
	26	+	q : Query for search
27	27		"""
28	28		self.framework = main.framework
29	29		self.q = q
		skipped 13 lines
43	43		'Upgrade-Insecure-Requests': '1', 'Cache-Control': 'max-age=0'}
44	44		self.framework.debug(f"[eTOOLS] Searching in 'etools.ch'...")
45	45		try:
46		-	req = self.framework.request(url=self.etools, params=params, headers=headers,allow_redirects=True)
	46	+	req = self.framework.request(url=self.etools, params=params, headers=headers, allow_redirects=True)
47	47		except:
48		-	self.framework.error('[eTOOLS] ConnectionError')
49		-	self.framework.error('eTOOLS is missed!')
	48	+	self.framework.error('ConnectionError.', 'util/carrot2', 'run_crawl')
	49	+	self.framework.error('eTOOLS is missed!', 'util/carrot2', 'run_crawl')
50	50		else:
51	51		self._pages = req.text
52	52		try:
53	53		self._json = req.json()
54	54		except:
55		-	self.framework.error('eTOOLS is missed!')
	55	+	self.framework.error('eTOOLS is missed!', 'util/carrot2', 'run_crawl')
56	56
57	57		@property
58	58		def pages(self):
		skipped 38 lines

■ ■ ■ ■ ■ ■

core/util/cms_identify.py

		skipped 21 lines
22	22		def __init__(self, content, headers):
23	23		""" baidu.com search engine
24	24
25		-	content : web content
26		-	headers : web headers
	25	+	content : Web content
	26	+	headers : Web headers
27	27		"""
28	28		self.content = content
29	29		self.headers = headers
		skipped 184 lines

■ ■ ■ ■ ■ ■

core/util/crt.py

		skipped 19 lines
20	20		def __init__(self, q):
21	21		""" crt.sh search engine
22	22
23		-	q : query for search
	23	+	q : Query for search
24	24		"""
25	25		self.framework = main.framework
26	26		self.q = q
		skipped 7 lines
34	34		req = self.framework.request(self.crt)
35	35		except:
36	36		self.framework.debug('[CRT] ConnectionError')
37		-	self.framework.error('CRT is missed!')
	37	+	self.framework.error('CRT is missed!', 'util/crt', 'run_crawl')
38	38		return
39	39		self._pages = req.text
40	40		try:
		skipped 19 lines

■ ■ ■ ■ ■ ■

core/util/darksearch.py

		skipped 23 lines
24	24		"""
25	25		darksearch.io search engine
26	26
27		-	q : query for search
28		-	limit : count of pages
	27	+	q : Query for search
	28	+	limit : Number of pages
29	29		"""
30	30
31	31		self.framework = main.framework
		skipped 6 lines
38	38		self._json_links = []
39	39
40	40		def run_crawl(self):
41		-
42	41		urls = [f"https://{self.darksearch}/api/search?query={self.q}&page={i}" for i in range(1, self.limit+1)]
43	42		max_attempt = len(urls)
44	43		for url in range(max_attempt):
		skipped 1 lines
46	45		try:
47	46		req = self.framework.request(url=urls[url], allow_redirects=True)
48	47		except Exception as e:
49		-	self.framework.error('[DARKSEARCH] ConnectionError')
	48	+	self.framework.error('ConnectionError', 'util/darksearch', 'run_crawl')
50	49		max_attempt -= 1
51	50		if max_attempt == 0:
52		-	self.framework.error('Darksearch is missed!')
	51	+	self.framework.error('Darksearch is missed!', 'util/darksearch', 'run_crawl')
53	52		break
54	53
55	54		if req.json()['data'] is None:
		skipped 30 lines

■ ■ ■ ■ ■ ■

core/util/discord.py

		skipped 20 lines
21	21		"""
22	22		discord users and servers search
23	23
24		-	q : query for search
25		-	limit : count of pages
	24	+	q : Query for search
	25	+	limit : Number of pages
26	26		"""
27	27
28	28		self.framework = main.framework
		skipped 19 lines
48	48		try:
49	49		req = self.framework.request(url=urls[url], allow_redirects=True)
50	50		except Exception as e:
51		-	self.framework.error('[DISCORD] ConnectionError')
	51	+	self.framework.error('ConnectionError', 'util/discord', 'run_crawl')
52	52		max_attempt -= 1
53	53		if max_attempt == 0:
54		-	self.framework.error('Discord is missed!')
	54	+	self.framework.error('Discord is missed!', 'util/discord', 'run_crawl')
55	55		break
56	56		self._pages += req.text
57	57		@property
		skipped 3 lines

■ ■ ■ ■ ■ ■

core/util/duckduckgo.py

		skipped 40 lines
41	41		params=payload,
42	42		allow_redirects=False)
43	43		except:
44		-	self.framework.error('[DUCKDUCKGO] ConnectionError')
	44	+	self.framework.error('ConnectionError', 'util/duckduckgo', 'run_crawl')
45	45		return
46	46		if req.status_code == 403:
47		-	self.framework.error('[DUCKDUCKGO] 403 Forbidden (Too many requests.)')
48		-	return
	47	+	self.framework.error('403 Forbidden (Too many requests.)', 'util/duckduckgo', 'run_crawl')
	48	+	break
49	49		self._pages += req.text
50	50		# setting next page offset
51	51		if payload['s'] == 0:
		skipped 44 lines

■ ■ ■ ■ ■ ■

core/util/exalead.py

		skipped 19 lines
20	20		def __init__(self, q, limit):
21	21		""" exalead.com search engine
22	22
23		-	q : query for search
24		-	limit : count of pages
	23	+	q : Query for search
	24	+	limit : Number of pages
25	25		"""
26	26		self.framework = main.framework
27	27		self.q = q
		skipped 11 lines
39	39		try:
40	40		req = self.framework.request(url=urls[url])
41	41		except:
42		-	self.framework.error('[EXALEAD] ConnectionError')
	42	+	self.framework.error('ConnectionError.', 'util/exalead', 'run_crawl')
43	43		max_attempt -= 1
44	44		if max_attempt == 0:
45		-	self.framework.error('Exalead is missed!')
	45	+	self.framework.error('Exalead is missed!', 'util/exalead', 'run_crawl')
46	46		break
47	47		else:
48	48		self._pages += req.text
		skipped 17 lines

■ ■ ■ ■ ■ ■

core/util/gigablast.py

		skipped 21 lines
22	22
23	23		def __init__(self, q, limit=15):
24	24		""" Gigablast search engine
25		-	q : query to search
	25	+	q : Query to search
	26	+	limit : Number of pages
26	27		"""
27	28		self.framework = main.framework
28	29		self.q = q
		skipped 12 lines
41	42		script = soup.find('body')['onload']
42	43		rand = re.findall(r'rand=(\d+[^&]+)&x', script)[0]
43	44		xkhh = re.findall(r"khh=(\d+[^']+)'", script)[0]
44		-
45	45		final_url = ''.join([f'https://www.gigablast.com/search?',
46	46		f'q={self.q}&format=json&rand={rand}&xkhh={xkhh}'])
47	47
48	48		req = self.framework.request(url=final_url)
49	49		self._json = req.json()['results']
50	50		except:
51		-	self.framework.error('[GIGABLAST] ConnectionError')
52		-	self.framework.error('Gigablast is missed!')
	51	+	self.framework.error('ConnectionError.', 'util/gigablast', 'run_crawl')
	52	+	self.framework.error('Gigablast is missed!', 'util/gigablast', 'run_crawl')
53	53
54	54		@property
55	55		def json(self):
		skipped 20 lines
76	76		for count, result in enumerate(self._json):
77	77		if count>=self._max:
78	78		break
79		-
80	79		self._links_with_data.append({
81	80		'title': result['title'],
82	81		'subtitle': result.get('subTitle'),
		skipped 2 lines
85	84		})
86	85		return self._links_with_data
87	86
88		-

■ ■ ■ ■ ■ ■

core/util/github.py

		skipped 46 lines
47	47		self.framework.verbose(f"[GITHUB] Searching in {page} page...")
48	48		req = self.framework.request(url=self.urls[page], headers={'Cookie': self.cookie, 'Accept': 'application/vnd.github.v3.text-match+json'}, allow_redirects=True)
49	49		result_json = req.json()
50		-	except:
51		-	self.framework.error('[GITHUB] ConnectionError')
52		-	return
	50	+	except Exception as e:
	51	+	self.framework.error(f"ConnectionError {e}.", 'util/github', 'run_crawl')
53	52		else:
54	53		for results in result_json['items']:
55	54		if 'fork' not in results:
		skipped 18 lines

■ ■ ■ ■ ■ ■

core/util/google.py

		skipped 14 lines
15	15		along with this program. If not, see <http://www.gnu.org/licenses/>.
16	16		"""
17	17
18		-	import re
19		-	import os
20	18
21	19		class main:
22	20		# framework = None
23		-	def __init__(self, q, limit=1, count=10, google_api=None, google_cx=None):
	21	+	def __init__(self, q, limit=1, count=10):
24	22		""" google.com search engine
25	23
26		-	q : Query for search
27		-	limit : Number of pages
28		-	google_api : Google api(if you need to use api_run_crawl)
29		-	google_cx : Google cx(if you need to use api_run_crawl)
	24	+	q : Query for search
	25	+	limit : Number of pages
	26	+	count : Number of results
30	27		"""
31	28		self.framework = main.framework
32	29		self.q = q
33	30		self.agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0'
	31	+	self.url = 'https://www.google.com/search'
34	32		self._pages = ''
35	33		self.limit = limit + 1
36		-	self.num = count
37		-	self.google_api = google_api
38		-	self.google_cx = google_cx
39		-	self._links = []
	34	+	self.count = count
	35	+	self.xpath_name = {
	36	+	'results': '//div[@class="g"]',
	37	+	'results_content': './/div[@class="IsZvec"]',
	38	+	'results_title': './/h3[1]',
	39	+	'results_a': './/div[@class="yuRUbf"]/a',
	40	+	'results_cite': './/div[@class="yuRUbf"]/a//cite'
	41	+	}
	42	+	self.xpath = {
	43	+	self.xpath_name['results']: [
	44	+	self.xpath_name['results_content'],
	45	+	self.xpath_name['results_title'],
	46	+	self.xpath_name['results_a'],
	47	+	self.xpath_name['results_cite']
	48	+	]
	49	+	}
40	50
41	51		def run_crawl(self):
42	52		page = 1
43		-	url = 'https://www.google.com/search'
44		-	set_page = lambda x: (x - 1) * self.num
45		-	payload = {'num': self.num, 'start': set_page(page), 'ie': 'utf-8', 'oe': 'utf-8', 'q': self.q, 'filter': '0'}
	53	+	set_page = lambda x: (x - 1) * self.count
	54	+	payload = {'num': self.count, 'start': set_page(page), 'ie': 'utf-8', 'oe': 'utf-8', 'q': self.q, 'filter': '0'}
46	55		while True:
47	56		self.framework.verbose(f"[GOOGLE] Searching in {page} page...", end='\r')
48	57		try:
49	58		req = self.framework.request(
50		-	url=url,
	59	+	url=self.url,
51	60		params=payload,
52	61		headers={'user-agent': self.agent},
53	62		allow_redirects=True)
54	63		except Exception as e:
55		-	self.framework.error(f"[GOOGLE] ConnectionError: {e}")
56		-	return
57		-	if req.status_code in (503, 429):
58		-	self.framework.error('[GOOGLE] Google CAPTCHA triggered.')
59		-	break
	64	+	self.framework.error(f"ConnectionError: {e}", 'util/google', 'run_crawl')
	65	+	else:
	66	+	if req.status_code in (503, 429):
	67	+	self.framework.error('Google CAPTCHA triggered.', 'util/google', 'run_crawl')
	68	+	break
60	69
61		-	if req.status_code in (301, 302):
62		-	redirect = req.headers['location']
63		-	req = self.framework.request(url=redirect, allow_redirects=False)
64		-
65		-	self._pages += req.text
66		-	page += 1
67		-	payload['start'] = set_page(page)
68		-	if page >= self.limit:
69		-	break
70		-	parser = self.framework.page_parse(self._pages)
71		-	links = parser.findall(r'a href="([^"]+)"')
72		-	links += parser.findall(r'href="/url\?q=([^"]+)&sa=U&ved=')
73		-
74		-	for link in links:
75		-	inside_links = ('https://accounts.google.com/ServiceLogin?continue', 'https://support.google.com/', \
76		-	'https://www.google.com/webhp', 'https://www.google.com/search?', 'https://support.google.com/websearch?')
77		-	if '://' in link:
78		-	url = self.framework.urlib(link).unquote_plus
79		-	url = re.sub(r"^\/url\?q=", '', url)
80		-	url = re.sub(r"\&amp.+", '', url)
81		-	cond2 = [x in url for x in inside_links]
82		-	if not any(cond2) and '://' in url:
83		-	self._links.append(url)
	70	+	if req.status_code in (301, 302):
	71	+	redirect = req.headers['location']
	72	+	req = self.framework.request(url=redirect, allow_redirects=False)
84	73
85		-	def api_run_crawl(self):
86		-	if not (self.google_api and self.google_cx):
87		-	self.framework.error('[GOOGLEAPI] google api needs google_api and google_cx variable')
88		-	return
	74	+	self._pages += req.text
	75	+	page += 1
	76	+	payload['start'] = set_page(page)
	77	+	if page >= self.limit:
	78	+	break
89	79
90		-	url = 'https://www.googleapis.com/customsearch/v1'
91		-	payload = {'alt': 'json', 'prettyPrint': 'false', 'key': self.google_api, 'cx': self.google_cx, 'q': self.q}
92		-	page = 0
93		-	self.framework.verbose(f"[GOOGLEAPI] Searching Google API for: {self.q}")
94		-	while True:
95		-	self.framework.verbose(f'[GOOGLEAPI] Searching in {page} page...', end='\r')
96		-	resp = self.framework.request(url, params=payload)
97		-	if resp.json() is None:
98		-	raise self.framework.FrameworkException(f"Invalid JSON response.{os.linesep}{resp.text}")
99		-	# add new results
100		-	if 'items' in resp.json():
101		-	self._links.extend(resp.json()['items'])
102		-	# increment and check the limit
103		-	page += 1
104		-	if self.limit == page:
105		-	break
106		-	# check for more pages
107		-	if not 'nextPage' in resp.json()['queries']:
108		-	break
109		-	payload['start'] = resp.json()['queries']['nextPage'][0]['startIndex']
	80	+	@property
	81	+	def results(self):
	82	+	parser = self.framework.page_parse(self._pages)
	83	+	xpath_results = parser.html_fromstring(self.xpath)
	84	+	results = []
	85	+	if not xpath_results:
	86	+	return results
	87	+	root = xpath_results[self.xpath_name['results']]
	88	+	for i in range(len(root[self.xpath_name['results_a']])):
	89	+	result = {
	90	+	'title': root[self.xpath_name['results_title']][i].text_content(),
	91	+	'a': root[self.xpath_name['results_a']][i].get('href'),
	92	+	'cite': root[self.xpath_name['results_cite']][i].text_content(),
	93	+	'content': root[self.xpath_name['results_content']][i].text_content(),
	94	+	}
	95	+	results.append(result)
	96	+	return results
110	97
111	98		@property
112	99		def pages(self):
		skipped 1 lines
114	101
115	102		@property
116	103		def links(self):
117		-	return self._links
	104	+	links = [x['a'] for x in self.results]
	105	+	return links
118	106
119	107		@property
120	108		def dns(self):
121		-	return self.framework.page_parse(self._pages).get_dns(self.q)
	109	+	return self.framework.page_parse(self._pages).get_dns(self.q, self.links)
122	110
123	111		@property
124	112		def emails(self):
		skipped 6 lines

■ ■ ■ ■ ■ ■

core/util/hunter.py

		skipped 36 lines
37	37		try:
38	38		req = self.framework.request(self.hunter_api)
39	39		except:
40		-	self.framework.debug('[HUNTER] ConnectionError')
41		-	self.framework.error('Hunter is missed!')
	40	+	self.framework.debug('ConnectionError', 'util/hunter', 'run_crawl')
	41	+	self.framework.error('Hunter is missed!', 'util/hunter', 'run_crawl')
42	42		return
43	43		self._pages = req.text
44	44		self._json_pages = req.json()
		skipped 1 lines
46	46		# Key validation
47	47		if 'errors' in self._json_pages:
48	48		code = self._json_pages.get('errors')[0]['details']
49		-	self.framework.error(f"[HUNTER] failed with error: {code}")
	49	+	self.framework.error(f"failed with error: {code}", 'util/hunter', 'run_crawl')
50	50		self.acceptable = False
51	51		return
52	52
		skipped 34 lines

■ ■ ■ ■ ■ ■

core/util/instagram.py

		skipped 55 lines
56	56		req = self.get_user_account_info()
57	57		if req.status_code == 404 or self.data == {}:
58	58		# user not found - show similar account names
59		-	self.framework.error('[INSTAGRAM] Account not Found.')
	59	+	self.framework.error('Account not Found.', 'util/instagram', 'run_crawl')
60	60		similar_users = self.get_similar_users()
61	61		if similar_users :
62	62		self.framework.heading('showing similar users', 0)
		skipped 5 lines
68	68		if self.is_private or req.status_code != 200:
69	69		# return if account is private or request fails
70	70		if self.is_private:
71		-	self.framework.error('[INSTAGRAM] Account is Private.')
	71	+	self.framework.error('Account is Private.', 'util/instagram', 'run_crawl')
72	72		if self.session_id == '':
73	73		return
74	74
75	75		else:
76		-	self.framework.error(f"[INSTAGRAM] Request Fails, Url: {req.url}")
	76	+	self.framework.error(f"Request Fails, Url: {req.url}", 'util/instagram', 'run_crawl')
77	77		return
78	78
79	79		# 2. get post and follwers and following
		skipped 16 lines
96	96		try:
97	97		self.data = req.json()
98	98		except:
99		-	self.framework.error('[INSTAGRAM] Request Fail!! Too many tries')
	99	+	self.framework.error('Request Fail!! Too many tries', 'util/instagram', 'get_user_account_info')
100	100		return
101	101		if self.data:
102	102		# id
		skipped 52 lines
155	155		self.posts_next_page = None
156	156		return False
157	157		else:
158		-	self.framework.error('[INSTAGRAM] Request For collecting User POST is unsuccessful.')
	158	+	self.framework.error('Request For collecting User POST is unsuccessful.', 'util/instagram', 'get_user_posts')
159	159		return False
160	160
161	161		def get_user_followers(self):
		skipped 31 lines
193	193		return False
194	194
195	195		else:
196		-	self.framework.error('[INSTAGRAM] Request For collecting User Followers is unsuccessful.')
	196	+	self.framework.error('Request For collecting User Followers is unsuccessful.', 'util/instagram', 'get_user_followers')
197	197		return False
198	198
199	199		def get_user_following(self):
		skipped 54 lines

■ ■ ■ ■ ■ ■

core/util/keywords.py

		skipped 58 lines
59	59		try:
60	60		req = self.framework.request(url, method=method, data=data)
61	61		except Exception as e:
62		-	self.framework.error('[KEYWORDS] connection error!')
	62	+	self.framework.error('ConnectionError', 'util/keywords', 'run_crawl')
63	63		self.framework.print_exception()
64	64		keys[source] = req
65	65
		skipped 28 lines

■ ■ ■ ■ ■ ■

core/util/metacrawler.py

		skipped 37 lines
38	38		try:
39	39		req = self.framework.request(url=urls[url])
40	40		except:
41		-	self.framework.error('[METACRAWLER] ConnectionError')
	41	+	self.framework.error('ConnectionError', 'util/metacrawler', 'run_crawl')
42	42		max_attempt -= 1
43	43		if max_attempt == 0:
44	44		self.framework.error('Metacrawler is missed!')
		skipped 1 lines
46	46		else:
47	47		page = self.framework.to_str(req.text)
48	48		if 'To continue, please respond below:' in page:
49		-	self.framework.error("[METACRAWLER] Google CAPTCHA triggered.")
	49	+	self.framework.error('CaptchaError', 'util/metacrawler', 'run_crawl')
50	50		return
51	51		if url > 0:
52	52		if 'Next »' not in page:
		skipped 26 lines

■ ■ ■ ■ ■ ■

core/util/millionshort.py

		skipped 22 lines
23	23		""" millionshort.com search engine
24	24
25	25		q : query for search
26		-	limit : count of pages
	26	+	limit : Number of pages
27	27		"""
28	28		self.framework = main.framework
29	29		self.q = q
		skipped 15 lines
45	45		try:
46	46		req = self.framework.request(url=urls[url])
47	47		except:
48		-	self.framework.error('[MILLIONSHORT] ConnectionError')
49		-	self.framework.error('Millionshort is missed!')
50		-	return
51		-	if 'captcha' in req.json():
52		-	self.framework.error('Captcha restriction!')
53		-	return
54		-	self._pages += req.text
55		-	self._json.append(req.json())
	48	+	self.framework.error('ConnectionError')
	49	+	else:
	50	+	if 'captcha' in req.json():
	51	+	self.framework.error('CaptchaError', 'util/millionshort', 'run_crawl')
	52	+	return
	53	+	self._pages += req.text
	54	+	self._json.append(req.json())
56	55
57	56		@property
58	57		def pages(self):
		skipped 35 lines

■ ■ ■ ■ ■ ■

core/util/netcraft.py

		skipped 34 lines
35	35		cookies = cookies or {}
36	36		try:
37	37		req = self.framework.request(url=url, cookies=cookies)
38		-	except:
39		-	self.framework.error('[NETCRAFT] ConnectionError')
40		-	self.framework.error('Netcraft is missed!')
	38	+	except Exception as e:
	39	+	self.framework.error(f"ConnectionError {e}.", 'util/netcraft', 'request')
	40	+	self.framework.error('Netcraft is missed!', 'util/netcraft', 'request')
41	41		return False
42	42		return req
43	43
		skipped 53 lines

■ ■ ■ ■ ■ ■

core/util/numverify.py

		skipped 50 lines
51	51
52	52		req = self.framework.request(url=url)
53	53		self._json = req.json()
54		-	except:
55		-	self.framework.error('[NUMVERIFY] ConnectionError')
56		-	self.framework.error('Numverify is missed!')
	54	+	except Exception as e:
	55	+	self.framework.error(f"ConnectionError {e}.", 'util/numverify', 'run_crawl')
	56	+	self.framework.error('Numverify is missed!', 'util/numverify', 'run_crawl')
57	57		return
58	58
59	59		@property
		skipped 3 lines

■ ■ ■ ■ ■ ■

core/util/onionland.py

		skipped 20 lines
21	21		def __init__(self, q, limit):
22	22		""" use onionlandsearchengine.com
23	23
24		-	q : query for search
25		-	limit : count of pages
	24	+	q : Query for search
	25	+	limit : Number of pages
26	26		"""
27	27		self.framework = main.framework
28	28		self.q = self.framework.urlib(q).quote
		skipped 14 lines
43	43		self.framework.print_exception()
44	44		max_attempt -= 1
45	45		if max_attempt == 0:
46		-	self.framework.error('Onionland is missed!')
	46	+	self.framework.error('Onionland is missed!', 'util/onionland', 'run_crawl')
47	47		break
48	48		else:
49	49		self._pages += req.text
		skipped 15 lines

■ ■ ■ ■ ■ ■

core/util/os_identify.py

		skipped 21 lines
22	22		def __init__(self, content, headers):
23	23		""" Detect operation system
24	24
25		-	content : web page content
26		-	headers : web page headers
	25	+	content : Web page content
	26	+	headers : Web page headers
27	27		"""
28	28		self.content = content
29	29		self.headers = headers
		skipped 59 lines

■ ■ ■ ■ ■ ■

core/util/page_parse.py

		skipped 15 lines
16	16		"""
17	17
18	18		import re
	19	+	from lxml import html
19	20
20	21		class main:
21	22		def __init__(self, page):
		skipped 10 lines
32	33		\|%22\|<span dir="[\w]+">\|</span>\|<h\d>\|</h\d>\|<q>\|</q>'
33	34		self.page = self.remove_comments(self.page)
34	35		self.page = re.sub(subs, '', self.page)
35		-	self.page = re.sub(r'%3a', ' ', self.page)
36		-	self.page = re.sub(r'%2f', ' ', self.page)
37		-	self.page = re.sub(r'%2f', ' ', self.page)
	36	+	self.page = re.sub(r"%3a", ' ', self.page)
	37	+	self.page = re.sub(r"%2f", ' ', self.page)
	38	+	self.page = re.sub(r"%2f", ' ', self.page)
	39	+
	40	+	def html_fromstring(self, xpath, parent=None, results={}):
	41	+	if self.page == '':
	42	+	self.framework.error(f"document is nil", 'util/page_parse', 'html_fromstring')
	43	+	return False
	44	+
	45	+	if isinstance(xpath, dict):
	46	+	for root in xpath:
	47	+	results[root] = self.html_fromstring(xpath[root], root)
	48	+	elif isinstance(xpath, list):
	49	+	results = {}
	50	+	for x in xpath:
	51	+	results[x] = self.html_fromstring(x, parent)
	52	+	else:
	53	+	tree = html.fromstring(self.page)
	54	+	results = []
	55	+	try:
	56	+	if parent:
	57	+	for node in tree.xpath(parent):
	58	+	results += node.xpath(xpath)
	59	+	else:
	60	+	results = tree.xpath(xpath)
	61	+	except:
	62	+	self.framework.error(f"invalid xpath: {xpath} or {parent}", 'util/page_parse', 'html_fromstring')
	63	+	return results
38	64
39	65		def dork_clean(self, host):
40		-	# Clear Dork's footprints
	66	+	# Clear dork's fingerprints
41	67		host = re.sub(r"(['\"]+)\|(%40\|@)", '', host)
42	68		return host
43	69
		skipped 251 lines

■ ■ ■ ■ ■ ■

core/util/pastebin.py

		skipped 28 lines
29	29		self.thread = 3
30	30		self._links = []
31	31		self.q_formats = {
32		-	'default_q': f'site:pastebin.com "{self.q}"',
33		-	'yippy_q': f'"pastebin.com" {self.q}',
34		-	'qwant_q': f'site:pastebin.com {self.q}'
	32	+	'default_q': f'site:pastebin.com "{self.q}"',
	33	+	'yippy_q': f'"pastebin.com" {self.q}',
	34	+	'qwant_q': f'site:pastebin.com {self.q}'
35	35		}
36	36
37	37
		skipped 35 lines

■ ■ ■ ■ ■ ■

core/util/piratebay.py

		skipped 39 lines
40	40		try:
41	41		req = self.framework.request(url=url)
42	42		except:
43		-	self.framework.error('[PIRATEBAY] ConnectionError')
44		-	self.framework.error('Piratebay is missed!')
45		-	self.framework.error('[PIRATEBAY] Try again after a few seconds!')
	43	+	self.framework.error('ConnectionError.', 'util/piratebay', 'run_crawl')
	44	+	self.framework.error('Piratebay is missed!', 'util/piratebay', 'run_crawl')
	45	+	self.framework.error('Try again after a few seconds!', 'util/piratebay', 'run_crawl')
46	46		else:
47	47		soup = bs(req.text, 'html.parser')
48	48		self._torrents = soup.find_all('tr')[1:-1]
		skipped 35 lines

■ ■ ■ ■ ■ ■

core/util/pubmed.py

		skipped 39 lines
40	40		self.q = urllib.parse.quote_plus(self.q)
41	41		self.framework.verbose('Searching the pubmed domain...')
42	42
43		-	url = f'https://pubmed.ncbi.nlm.nih.gov/?term={self.q}&size=200'
	43	+	url = f"https://pubmed.ncbi.nlm.nih.gov/?term={self.q}&size=200"
44	44		try:
45	45		req = self.framework.request(url=url)
46		-	except:
47		-	self.framework.error('[PUBMED] ConnectionError')
48		-	self.framework.error('Pubmed is missed!')
	46	+	except Exception as e:
	47	+	self.framework.error(f"ConnectionError {e}.", 'util/pubmed', 'run_crawl')
	48	+	self.framework.error('Pubmed is missed!', 'util/pubmed', 'run_crawl')
49	49		return
50	50		self._rawhtml += req.text
51	51		self._articles.extend(re.findall(r'<article .?>(.?)</article>',
		skipped 30 lines

■ ■ ■ ■ ■ ■

core/util/qwant.py

		skipped 21 lines
22	22		def __init__(self, q, limit=2):
23	23		""" qwant.com search engine
24	24
25		-	q : query for search
26		-	limit : count of pages
	25	+	q : Query for search
	26	+	limit : Number of pages
27	27		"""
28	28		self.framework = main.framework
29	29		self.q = q
		skipped 28 lines
58	58		self.framework.verbose(f"[QWANT] Searching in {url} page...")
59	59		try:
60	60		req = self.framework.request(url=urls[url], headers=headers)
61		-	except:
62		-	self.framework.error('[QWANT] ConnectionError')
63		-	self.framework.error('Qwant is missed!')
64		-	return
65		-	if req.status_code == 429 and "I can't let you do that..." in req.text and '<div class="error-code">' in req.text:
66		-	self.framework.error('429 Too Many Requests')
67		-	return
68		-	self._pages += req.text
69		-	try:
70		-	self._json.append(req.json())
71	61		except Exception as e:
72		-	self.framework.error('429 Too Many Requests')
73		-	return
	62	+	self.framework.error(f"ConnectionError {e}.", 'util/qwant', 'name_crawl')
	63	+	self.framework.error('Qwant is missed!', 'util/qwant', 'name_crawl')
74	64		else:
75		-	if req.json() == {"status": "error", "error": 22}:
	65	+	if req.status_code == 429 and "I can't let you do that..." in req.text and '<div class="error-code">' in req.text:
	66	+	self.framework.error('429 Too Many Requests')
	67	+	return
	68	+	self._pages += req.text
	69	+	try:
	70	+	self._json.append(req.json())
	71	+	except Exception as e:
76	72		self.framework.error('429 Too Many Requests')
	73	+	return
	74	+	else:
	75	+	if req.json() == {"status": "error", "error": 22}:
	76	+	self.framework.error('429 Too Many Requests')
77	77
78	78		@property
79	79		def pages(self):
		skipped 35 lines

■ ■ ■ ■ ■ ■

core/util/sanctionsearch.py

		skipped 13 lines
14	14		You should have received a copy of the GNU General Public License
15	15		along with this program. If not, see <http://www.gnu.org/licenses/>.
16	16		"""
	17	+
17	18		from bs4 import BeautifulSoup as bs
	19	+
18	20		class main:
19	21
20	22		def __init__(self, query=None, id=None, limit=15):
21	23		""" Sanctionsearch.ofac.treas.gov search engine
22		-	name : name to search
23		-	id : id to search
24		-	limit : number of results (only applicable for name search)
	24	+	name : Mame to search
	25	+	id : Id to search
	26	+	limit : Number of results (only applicable for name search)
25	27		"""
26	28		self.framework = main.framework
27	29		self._query = query
		skipped 15 lines
43	45		try:
44	46		# First request to get viewstate from input tag
45	47		req = self.framework.request(url=self.sanctionsearch)
46		-	except:
47		-	self.framework.error('[SANCTIONSEARCH] ConnectionError')
48		-	self.framework.error('Sanctionsearch is missed!')
	48	+	except Exception as e:
	49	+	self.framework.error(f"ConnectionError {e}.", 'util/sanctionsearch', 'name_crawl')
	50	+	self.framework.error('Sanctionsearch is missed!', 'util/sanctionsearch', 'name_crawl')
49	51		return
50	52
51	53		soup = bs(req.text, 'html.parser')
		skipped 11 lines
63	65		method='POST',
64	66		data=data,
65	67		headers=headers)
66		-	except:
67		-	self.framework.error('[SANCTIONSEARCH] ConnectionError')
68		-	self.framework.error('Sanctionsearch is missed!')
	68	+	except Exception as e:
	69	+	self.framework.error(f"ConnectionError {e}.", 'util/sanctionsearch', 'name_crawl')
	70	+	self.framework.error('Sanctionsearch is missed!', 'util/sanctionsearch', 'name_crawl')
69	71		return
70	72
71	73		soup = bs(req.text, 'html.parser')
		skipped 71 lines
143	145		address_table = address_table_div.find('table')
144	146		address_th = list(map(lambda x: x.text,
145	147		address_table.find_all('th')))
146		-	address_rows =list(filter(lambda x:len(''.join(x.text).strip()) > 0,
	148	+	address_rows = list(filter(lambda x:len(''.join(x.text).strip()) > 0,
147	149		address_table.find_all('tr')))[1:]
148	150
149	151		self._data['addresses'] = {}
		skipped 13 lines
163	165		def data(self):
164	166		return self._data
165	167
166		-

■ ■ ■ ■ ■ ■

core/util/startpage.py

		skipped 19 lines
20	20		def __init__(self, q, limit=2):
21	21		""" startpage.com search engine
22	22
23		-	q : query for search
24		-	limit : count of pages
	23	+	q : Query for search
	24	+	limit : Number of pages
25	25		"""
26	26		self.framework = main.framework
27	27		self.q = self.framework.urlib(q).quote
		skipped 8 lines
36	36		self.framework.verbose(f"[STARTPAGE] Searching in {url} page...")
37	37		try:
38	38		req = self.framework.request(url=urls[url])
39		-	except:
40		-	self.framework.error('[STARTPAGE] ConnectionError')
	39	+	except Exception as e:
	40	+	self.framework.error(f"ConnectionError {e}.", 'util/startpage', 'run_crawl')
41	41		max_attempt -= 1
42	42		if max_attempt == 0:
43		-	self.framework.error('Startpage is missed!')
	43	+	self.framework.error('Startpage is missed!', 'util/startpage', 'run_crawl')
44	44		break
45	45		else:
46	46		page = req.text
		skipped 21 lines

■ ■ ■ ■ ■ ■

core/util/urlscan.py

		skipped 42 lines
43	43		self.framework.verbose('[URLSCAN] Not yet in our database.')
44	44		return
45	45		elif 'results' not in result_json:
46		-	self.framework.error(f'[URLSCAN] failed with an error: {result_json["description"]}')
	46	+	self.framework.error(f"failed with an error: {result_json['description']}", 'util/urlscan', 'run_crawl')
47	47		return
48	48		else:
49	49		for results in result_json['results']:
		skipped 13 lines

■ ■ ■ ■ ■ ■

core/util/waf_identify.py

		skipped 22 lines
23	23		def __init__(self, req):
24	24		""" detect WAFs(Web Application Firewalls)
25	25
26		-	req : request attribute
	26	+	req : Request attribute
27	27		"""
28	28		self.req = req
29	29		self.content = req.text
		skipped 1513 lines

■ ■ ■ ■ ■ ■

core/util/wapps.py

		skipped 23 lines
24	24		def __init__(self, q, page, headers):
25	25		""" Web application fingerprint for detect apps. like wappalizer
26	26
27		-	q : domain name
28		-	page : web page
29		-	headers : web headers
	27	+	q : Domain name
	28	+	page : Web page
	29	+	headers : Web headers
30	30		"""
31	31		self.framework = main.framework
32	32		self.q = q
		skipped 184 lines

■ ■ ■ ■ ■ ■

core/util/web_scrap.py

		skipped 24 lines
25	25		def __init__(self, url, debug=False, limit=1, thread_count=1):
26	26		""" Web scraper with thread support
27	27
28		-	url : first page address
29		-	debug : show the result at moment
30		-	limit : web scrap level(if it's 1 that's mean just search in first page)
31		-	thread_count : count of links for open at per lap
	28	+	url : First page address
	29	+	debug : Show the result at moment
	30	+	limit : Web scrap level(if it's 1 that's mean just search in first page)
	31	+	thread_count : Count of links for open at per lap
32	32		"""
33	33		self.framework = main.framework
34	34		# ADD http://
		skipped 302 lines

■ ■ ■ ■ ■ ■

core/util/wikileaks.py

		skipped 22 lines
23	23		def __init__(self, q, limit=1):
24	24		""" wikileaks.org search engine
25	25
26		-	q : query for search
	26	+	q : Query for search
27	27		limit : Number of Pages
28	28
29	29		"""
		skipped 20 lines
50	50		headers={'User-Agent': self.agent},
51	51		allow_redirects=True)
52	52		except Exception as e:
53		-	self.framework.error(f"[Wikileaks] ConnectionError: {e}")
	53	+	self.framework.error(f"ConnectionError: {e}", 'util/wikileaks', 'run_crawl')
54	54		max_attempts += 1
55	55		if max_attempts == self.limit:
56		-	self.framework.error('Wikileaks is missed!')
	56	+	self.framework.error('Wikileaks is missed!', 'util/wikileaks', 'run_crawl')
57	57		return
58	58		else:
59	59		self._pages += req.text
		skipped 39 lines

■ ■ ■ ■ ■ ■

core/util/wikipedia.py

		skipped 24 lines
25	25		self._links = []
26	26		self._titles = []
27	27		self._pids = []
28		-	self.url = "https://en.wikipedia.org/w/api.php"
29		-	self.headers = {"User-Agent" : "OWASP Maryam(github.com/saeeddhqan/maryam)"}
	28	+	self.url = 'https://en.wikipedia.org/w/api.php'
	29	+	self.headers = {'User-Agent' : 'OWASP Maryam(github.com/saeeddhqan/maryam)'}
30	30
31	31
32	32		def run_crawl(self):
33		-	self.framework.verbose(f"[WIKIPEDIA] Searching...", end='\r')
	33	+	self.framework.verbose('[WIKIPEDIA] Searching...', end='\r')
34	34		payload = {
35	35		'action': 'query',
36	36		'list': 'search',
		skipped 47 lines
84	84		headers=self.headers,
85	85		allow_redirects=False)
86	86		return req
87		-	except:
88		-	self.framework.error('[WIKIPEDIA] ConnectionError')
	87	+	except Exception as e:
	88	+	self.framework.error(f"ConnectionError: {e}", 'util/wikipedia', 'wiki_request')
89	89		return None
90	90
91	91		@property
		skipped 15 lines

■ ■ ■ ■ ■ ■

core/util/worldcat.py

		skipped 31 lines
32	32		self.order = f"thold {order}"
33	33		self._xml_data = ''
34	34
35		-	def search(self):
	35	+	def run_crawl(self):
36	36		self.framework.verbose('[Worldcat Search] Fetching data for the query...')
37	37		api_url = 'http://classify.oclc.org/classify2/Classify'
38	38		url_with_payload = f'{api_url}?title={self.title}&author={self.author}&maxRecs={self.limit}&orderBy={self.order}'
39	39		try:
40	40		response = self.framework.request(url=url_with_payload)
41	41		except Exception as e:
42		-	self.framework.error('Worldcat is missed!')
	42	+	self.framework.error('Worldcat is missed!', 'util/worldcat', 'search')
	43	+	return False
43	44		else:
44	45		self._xml_data = response.content
45	46
		skipped 4 lines

■ ■ ■ ■ ■ ■

core/util/yahoo.py

		skipped 23 lines
24	24
25	25		framework : core attribute
26	26		q : query for search
27		-	limit : count of pages
28		-	count : count of links
	27	+	limit : Number of pages
	28	+	count : Number of links
29	29		"""
30	30		self.framework = main.framework
31	31		self.q = self.framework.urlib(q).quote
32	32		self.limit = limit
33	33		self.count = count
	34	+	self.agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0'
	35	+	self.url = 'https://search.yahoo.com/search'
34	36		self._pages = ''
35		-	self.yahoo = 'search.yahoo.com'
	37	+	self.xpath_name = {
	38	+	'results': "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]",
	39	+	'results_a': './/h3/a/@href',
	40	+	'results_title': './/h3/a',
	41	+	'results_cite': './/div/span',
	42	+	'results_content': './/div[contains(@class, "compText")]'
	43	+	}
	44	+	self.xpath = {
	45	+	self.xpath_name['results']: [
	46	+	self.xpath_name['results_content'],
	47	+	self.xpath_name['results_title'],
	48	+	self.xpath_name['results_a'],
	49	+	self.xpath_name['results_cite']
	50	+	]
	51	+	}
36	52
37	53		def run_crawl(self):
	54	+	page = 1
38	55		set_page = lambda x: (x*10)+1
39		-	urls = [f'https://{self.yahoo}/search?p={self.q}&b={set_page(i)}&pz={self.count}' for i in range(1, self.limit+1)]
40		-	max_attempt = len(urls)
41		-	for url in range(len(urls)):
42		-	self.framework.verbose(f"[YAHOO] Searching in {url} page...")
	56	+	payload = {'p': self.q, 'b': set_page(page), 'pz': self.count, 'fl': 1}
	57	+	max_attempt = 0
	58	+	while True:
	59	+	self.framework.verbose(f"[YAHOO] Searching in {page} page...")
43	60		try:
44		-	req = self.framework.request(url=urls[url],allow_redirects=True)
	61	+	req = self.framework.request(
	62	+	url=self.url,
	63	+	params=payload,
	64	+	headers={'user-agent': self.agent},
	65	+	allow_redirects=True)
45	66		except:
46		-	self.framework.error('[YAHOO] ConnectionError')
47		-	max_attempt -= 1
48		-	if max_attempt == 0:
49		-	self.framework.error('Yahoo is missed!')
	67	+	self.framework.error('ConnectionError', 'util/yahoo', 'run_crawl')
	68	+	max_attempt += 1
	69	+	if max_attempt == self.limit:
	70	+	self.framework.error('Yahoo is missed!', 'util/yahoo', 'run_crawl')
50	71		break
51	72		else:
52		-	page = req.text
53		-	if '">Next</a>' not in page:
54		-	self._pages += page
	73	+	if 'unsafe-url">Next<' not in req.text or page == self.limit:
	74	+	self._pages += req.text
55	75		break
56		-	self._pages += page
	76	+	page += 1
	77	+	payload['b'] = set_page(page)
	78	+	self._pages += req.text
57	79
58	80		@property
59	81		def pages(self):
60	82		return self._pages
	83	+
	84	+	@property
	85	+	def results(self):
	86	+	parser = self.framework.page_parse(self._pages)
	87	+	xpath_results = parser.html_fromstring(self.xpath)
	88	+	results = []
	89	+	if not xpath_results:
	90	+	return results
	91	+	root = xpath_results[self.xpath_name['results']]
	92	+	for i in range(len(root[self.xpath_name['results_a']])):
	93	+	a = re.search(r"RU=(https?%3a%2f%2f[^/]+)/", root[self.xpath_name['results_a']][i])
	94	+	if a:
	95	+	a = a.group(1)
	96	+	else:
	97	+	a = root[self.xpath_name['results_a']][i]
	98	+	urlib = self.framework.urlib(a)
	99	+	a = urlib.unquote
	100	+	cite = root[self.xpath_name['results_cite']][i].text_content()
	101	+	if len(cite) < 3:
	102	+	cite = urlib.netloc
	103	+	result = {
	104	+	'title': root[self.xpath_name['results_title']][i].text_content(),
	105	+	'a': a,
	106	+	'cite': cite,
	107	+	'content': root[self.xpath_name['results_content']][i].text_content(),
	108	+	}
	109	+	results.append(result)
	110	+	return results
61	111
62	112		@property
63	113		def links(self):
		skipped 20 lines

■ ■ ■ ■ ■ ■

core/util/yandex.py

		skipped 23 lines
24	24		""" yandex.com search engine
25	25
26	26		q : query for search
27		-	limit : count of pages
28		-	count : count of links
	27	+	limit : Number of pages
	28	+	count : Number of links
29	29		"""
30	30		self.framework = main.framework
31	31		self.q = self.framework.urlib(q).quote
		skipped 10 lines
42	42		try:
43	43		req = self.framework.request(url=urls[url], allow_redirects=True)
44	44		except:
45		-	self.framework.error('[YANDEX] ConnectionError')
	45	+	self.framework.error('ConnectionError', 'util/yandex', 'run_crawl')
46	46		max_attempt -= 1
47	47		if max_attempt == 0:
48		-	self.framework.error('Yandex is missed!')
	48	+	self.framework.error('Yandex is missed!', 'util/yandex', 'run_crawl')
49	49		break
50	50		else:
51	51		if '<title>Oops!</title>' in req.text:
52		-	self.framework.error("[YANDEX] Yandex CAPTCHA triggered.")
	52	+	self.framework.error('Yandex CAPTCHA triggered.', 'util/yandex', 'run_crawl')
53	53		return
54	54
55	55		page = req.text
		skipped 21 lines

■ ■ ■ ■ ■ ■

core/util/yippy.py

		skipped 38 lines
39	39		try:
40	40		req = self.framework.request(url=geturl)
41	41		except:
42		-	self.framework.error('[YIPPY] ConnectionError')
43		-	self.framework.error('Yippy is missed!')
	42	+	self.framework.error('ConnectionError', 'util/yippy', 'run_crawl')
	43	+	self.framework.error('Yippy is missed!', 'util/yippy', 'run_crawl')
44	44		return
45	45
46	46		txt = req.text
		skipped 61 lines

■ ■ ■ ■ ■ ■

maryam

		skipped 15 lines
16	16		along with this program. If not, see <http://www.gnu.org/licenses/>.
17	17		"""
18	18
19		-	__VERSION__ = '2.0.0'
	19	+	__VERSION__ = '2.0.5'
20	20
21	21		import sys
22	22		import argparse
		skipped 41 lines

■ ■ ■ ■ ■ ■

modules/footprint/crawl_pages.py

		skipped 40 lines
41	41		try:
42	42		re.compile(regex)
43	43		except Exception as e:
44		-	self.error(e)
	44	+	self.error(e, 'crawl_pages', 'module_api')
45	45		return
46	46		scrap = self.web_scrap(domain, self.options['debug'], self.options['limit'], self.options['thread'])
47	47		scrap.run_crawl()
		skipped 78 lines

■ ■ ■ ■ ■ ■

modules/footprint/dnsbrute.py

		skipped 88 lines
89	89		wordlist.pop(wordlist.index(''))
90	90		return wordlist
91	91		else:
92		-	self.error(f"{addr} value is not text/plain")
	92	+	self.error(f"{addr} value is not text/plain", 'dnsbrute', 'remote_list')
93	93		return []
94	94
95	95		def module_api(self):
		skipped 36 lines

■ ■ ■ ■ ■ ■

modules/footprint/filebrute.py

		skipped 99 lines
100	100		wordlist.pop(wordlist.index(''))
101	101		return wordlist
102	102		else:
103		-	self.error(f"{addr} value is not text/plain")
	103	+	self.error(f"{addr} value is not text/plain", 'filebrute', 'remote_list')
104	104		return []
105	105
106	106		def module_api(self):
		skipped 40 lines

■ ■ ■ ■ ■ ■

modules/osint/bing_mobile_view.py

		skipped 41 lines
42	42		global project_root
43	43		black_img_path = os.path.join(project_root, 'data', 'images', 'blacklist', f'{self.options["blacklist"]}.json')
44	44		if not os.path.isfile(black_img_path) : # Check if blacklisted image exists?
45		-	self.error('[Mobile Screenshot] The blacklisted image name entered by user does not exist.')
	45	+	self.error('The blacklisted image name entered by user does not exist.', 'bing_mobile_view', 'blacklist_check')
46	46		return False
47	47
48	48		blacklisted_image_data = json.load(open(black_img_path)) # Load the data of blacklisted image
		skipped 8 lines
57	57		counter += 1
58	58
59	59		if counter-1 == self.options['retries']: # If all retries are done then return false otherwise image data
60		-	self.error('[Bing Mobile View] Exhausted retries, maybe this is not a public profile.')
	60	+	self.error('Exhausted retries, maybe this is not a public profile.', 'bing_mobile_view', 'blacklist_check')
61	61		return False
62	62		else:
63	63		return image_data
		skipped 13 lines
77	77		image_data = run.raw_image_data
78	78
79	79		if image_data == '':
80		-	self.error('[Bing Mobile View] Unable to detect image in the page')
	80	+	self.error('Unable to detect image in the page', 'bing_mobile_view', 'module_api')
81	81		return output
82	82
83	83		'''
		skipped 42 lines

■ ■ ■ ■ ■ ■

modules/osint/cve_search.py

		skipped 51 lines
52	52		req = self.request(
53	53		f"https://cve.mitre.org/cgi-bin/cvekey.cgi?keyword={q}")
54	54		except Exception as e:
55		-	self.error('Mitre is missed!')
	55	+	self.error('Mitre is missed!', 'cve_search', 'mitre')
56	56		else:
57	57		hrefs = re.findall(
58	58		r'<a href="(/cgi-bin/cvename.cgi\?name=[^"]+)">', req.text)[:count]
		skipped 10 lines
69	69		req = self.request(
70	70		f"https://services.nvd.nist.gov/rest/json/cves/1.0?keyword={q}&resultsPerPage={count}")
71	71		except Exception as e:
72		-	self.error('Nist is missed!')
	72	+	self.error('Nist is missed!', 'cve_search', 'nist')
73	73		else:
74	74		cve_items = req.json()['result']['CVE_Items']
75	75		titles = [cve['cve']['CVE_data_meta']['ID'] for cve in cve_items]
		skipped 8 lines
84	84		req = self.request(
85	85		f"https://packetstormsecurity.com/search/?q={q}", timeout=60)
86	86		except Exception as e:
87		-	self.error('Packetstormsecurity is missed')
	87	+	self.error('Packetstormsecurity is missed', 'cve_search', 'packetstormsecurity')
88	88		else:
89	89		titles = re.findall(r'<a class="ico text-plain"[^>]+>([^<]+)</a>', req.text)[:count]
90	90		links = re.findall(r'<a class="ico text-plain" href="([^"]+)"[^>]+>', req.text)[:count]
		skipped 21 lines

■ ■ ■ ■ ■ ■

modules/osint/dns_search.py

		skipped 94 lines
95	95		req = self.request(
96	96		f"https://riddler.io/search?q=pld:{q}&view_type=data_table")
97	97		except Exception as e:
98		-	self.error('Riddler is missed!')
	98	+	self.error('Riddler is missed!', 'dns_search', 'riddler')
99	99		else:
100	100		j = self.page_parse(req.text).get_dns(q) or []
101	101		set_data(j)
		skipped 6 lines
108	108		req = self.request(
109	109		f"http://www.sitedossier.com/parentdomain/{q}/{next_page}")
110	110		except Exception as e:
111		-	self.error('Sitedossier is missed!')
	111	+	self.error('Sitedossier is missed!', 'dns_search', 'sitedossier')
112	112		break
113	113		else:
114	114		text = req.text
		skipped 16 lines
131	131		req = self.request(
132	132		'https://threatcrowd.org/searchApi/v2/domain/report/?domain=' + q)
133	133		except Exception as e:
134		-	self.error('ThreatCrowd is missed!')
	134	+	self.error('ThreatCrowd is missed!', 'dns_search', 'threatcrowd')
135	135		else:
136	136		txt = re.sub(r'[\t\n ]+', '', req.text)
137	137		txt = re.findall(
		skipped 11 lines
149	149		req = self.request(
150	150		f"https://api.threatminer.org/v2/domain.php?q={q}&rt=5")
151	151		except Exception as e:
152		-	self.error('ThreatMiner is missed!')
	152	+	self.error('ThreatMiner is missed!', 'dns_search', 'threatminer')
153	153		else:
154	154		j = req.json()['results'] or []
155	155		set_data(j)
		skipped 2 lines
158	158		self.verbose('[RAPIDDNS] Searching in rapiddns...')
159	159		try:
160	160		req = self.request(
161		-	'https://rapiddns.io/subdomain/' + q + '?full=1')
	161	+	f"https://rapiddns.io/subdomain/{q}?full=1")
162	162		except Exception as e:
163		-	self.error('Rapiddns is missed!')
	163	+	self.error('Rapiddns is missed!', 'dns_search', 'rapiddns')
164	164		else:
165	165		j = self.page_parse(req.text).get_dns(q) or []
166	166		set_data(j)
		skipped 18 lines
185	185		data = {'csrfmiddlewaretoken': csrf_token, 'targetip': q}
186	186		req = self.request('https://dnsdumpster.com/', method='POST', headers=headers, data=data)
187	187		except Exception:
188		-	self.error('DNSdumpster is missed!')
	188	+	self.error('DNSdumpster is missed!', 'dns_search', 'dnsdumpster')
189	189		else:
190	190		j = self.page_parse(req.text).get_dns(q) or []
191	191		set_data(j)
		skipped 10 lines
202	202		try:
203	203		req = self.request('https://domains.yougetsignal.com/domains.php', method='POST', headers=headers, data=data).json()
204	204		except Exception as e:
205		-	self.error('YouGetSignal is missed!')
	205	+	self.error('YouGetSignal is missed!', 'dns_search', 'yougetsignal')
206	206		else:
207	207		j = [x[0] if len(x)>1 else '' for x in req.get('domainArray')]
208	208		set_data(j)
		skipped 4 lines
213	213		req = self.request(
214	214		f"https://api.certspotter.com/v1/issuances?domain={q}&include_subdomains=true&expand=dns_names")
215	215		except Exception as e:
216		-	self.error('CERTSPOTTER is missed!')
	216	+	self.error('CERTSPOTTER is missed!', 'dns_search', 'certspotter')
217	217		else:
218	218		text = req.text
219	219		if "rate_limit" in text:
220		-	self.error('[CERTSPOTTER] Too many request please try again later')
	220	+	self.error('Too many request please try again later', 'dns_search', 'certspotter')
221	221		return "certspotter"
222	222		j = self.page_parse(req.text).get_dns(q) or []
223	223		set_data(j)
		skipped 5 lines
229	229		req = self.request(
230	230		'https://api.sublist3r.com/search.php?domain=' + q)
231	231		except Exception as e:
232		-	self.error('SUBLIST3R is missed!')
	232	+	self.error('SUBLIST3R is missed!', 'dns_search', 'sublist3r')
233	233		else:
234	234		j = json.loads(req.text)
235	235		set_data(j)
		skipped 2 lines
238	238		self.verbose('[JLDC] Searching in jldc.me...')
239	239		try:
240	240		req = self.request(
241		-	'https://jldc.me/anubis/subdomains/' + q)
	241	+	f"https://jldc.me/anubis/subdomains/{q}")
242	242		except Exception as e:
243		-	self.error('JLDC is missed!')
	243	+	self.error('JLDC is missed!', 'dns_search', 'jldc')
244	244		else:
245	245		if 'Too many request' in req.text:
246		-	self.error('[JLDC] Too many request please try again later')
	246	+	self.error('Too many request please try again later', 'dns_search', 'jldc')
247	247		return 'jldc'
248	248		j = list(req.json()) or []
249	249		set_data(j)
		skipped 4 lines
254	254		req = self.request(
255	255		f'https://dns.bufferover.run/dns?q=.{q}')
256	256		except Exception as e:
257		-	self.error('BufferOver is missed!')
	257	+	self.error('BufferOver is missed!', 'dns_search', 'bufferover')
258	258		else:
259	259		j = list(req.json()['FDNS_A']) or []
260	260		set_data([(x.split(',')[1] if len(x.split(',')) == 2 else x) for x in j])
		skipped 4 lines
265	265		req = self.request(
266	266		f"https://otx.alienvault.com/api/v1/indicators/domain/{q}/passive_dns")
267	267		except Exception as e:
268		-	self.error("OTX is missed!")
	268	+	self.error('OTX is missed!', 'dns_search', 'otx')
269	269		else:
270	270		j = self.page_parse(req.text).get_dns(q)
271	271		set_data(j)
		skipped 7 lines
279	279		try:
280	280		req = self.request(enu_url)
281	281		except Exception as e:
282		-	self.error("Sectrails is missed!")
	282	+	self.error('Sectrails is missed!', 'dns_search', 'securitytrails')
283	283		else:
284	284		result = req.text.split(trim[0])[1].split(trim[1])[0]
285	285		req_json = json.loads(result)
		skipped 78 lines

■ ■ ■ ■ ■ ■

modules/osint/domain_reputation.py

		skipped 62 lines
63	63		req = self.request('https://www.barracudacentral.org/lookups/lookup-reputation', method='POST',
64	64		data=data)
65	65		except Exception as e:
66		-	self.error('Barracuda is missed!')
	66	+	self.error('Barracuda is missed!', 'domain_reputation', 'barracudacentral')
67	67		else:
68	68		reg = re.compile(r"categories: <strong>([\w\d\s-]+)")
69	69		if 'listed as "poor"' not in req.text:
		skipped 22 lines
92	92		blacklist_reg = re.compile(r'<span class="bld_name">([\d\w\s]+)</span>')
93	93		lists = list_reg.findall(j)[:-1]
94	94		except Exception as e:
95		-	self.verbose('Mxtoolbox is missed')
	95	+	self.error('Mxtoolbox is missed!', 'domain_reputation', 'mxtoolbox')
96	96		else:
97	97		for blacklist in lists:
98	98		if 'LISTED' in blacklist:
		skipped 17 lines
116	116		num_test_reg = re.compile(r'nof: ([\d]+),')
117	117		num_test = num_test_reg.findall(text)[0]
118	118		except Exception as e:
119		-	self.verbose('Multirbl is missed')
	119	+	self.error('Multirbl is missed', 'domain_reputation', 'multirbl')
120	120		else:
121	121		thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=2)
122	122		futures = (thread_pool.submit(__multirbl, self, l_id, hash, l_ids, q) for l_id in range(int(num_test)))
		skipped 34 lines

■ ■ ■ ■ ■ ■

modules/osint/email_search.py

		skipped 45 lines
46	46		key = q.split('&api_key=')[1]
47	47		k_q = q.split('&api_key=')[0]
48	48		if key == 'None':
49		-	self.error('-k <API KEY> is required for hunter')
	49	+	self.error('-k <API KEY> is required for hunter', 'email_search', 'search')
50	50		return
51	51		else:
52	52		attr = engine(k_q, key, limit)
		skipped 31 lines

■ ■ ■ ■ ■ ■

modules/osint/phone_number_search.py

		skipped 13 lines
14	14		You should have received a copy of the GNU General Public License
15	15		along with this program. If not, see <http://www.gnu.org/licenses/>.
16	16		"""
17		-	import re
18	17
19	18		meta = {
20	19		'name': 'Phone Number Search',
		skipped 20 lines
41	40		if output['valid']:
42	41		self.alert_results(output)
43	42		else:
44		-	self.error('Invalid Number!')
45		-	self.error("Number must be of the form '+{area code}{ten digit number}'")
	43	+	self.error('Invalid Number!', 'phone_number_search', 'module_run')
	44	+	self.error("Number must be of the form '+{area code}{ten digit number}'", 'phone_number_search', 'module_run')
46	45

■ ■ ■ ■ ■ ■

modules/search/google.py

		skipped 17 lines
18	18		meta = {
19	19		'name': 'Google Search',
20	20		'author': 'Saeed',
21		-	'version': '0.2',
	21	+	'version': '0.5',
22	22		'description': 'Search your query in the google.com and show the results.',
23	23		'sources': ('google',),
24	24		'options': (
		skipped 10 lines
35	35		count = self.options['count']
36	36		run = self.google(query, limit, count)
37	37		run.run_crawl()
38		-	links = run.links
39		-	output = {'links': links}
	38	+	results = run.results
	39	+	output = {'results': results}
40	40		self.save_gather(output, 'search/google', query, output=self.options['output'])
41	41		return output
42	42
43	43		def module_run(self):
44		-	self.alert_results(module_api(self))
	44	+	self.search_engine_results(module_api(self))
45	45

■ ■ ■ ■ ■ ■

modules/search/instagram.py

		skipped 89 lines
90	90		}
91	91		if 'instagram' in engine and not session_id:
92	92		msg = '"sessionid" field from cookies is required for instagram engine.'
93		-	self.error(msg)
94		-	output['error'] = msg
	93	+	self.error(msg, 'instagram', 'module_api')
95	94		return
96	95
97	96		q_formats = {
		skipped 48 lines

■ ■ ■ ■ ■ ■

modules/search/telegram.py

		skipped 60 lines
61	61		try:
62	62		req = self.request(f'https://telegramchannels.me/search?type=channel&page={page_no}&search={query}').text
63	63		except Exception as e:
64		-	self.error('Telegramchannels.me is missed!')
	64	+	self.error('Telegramchannels.me is missed!', 'telegram', 'scrap')
65	65
66	66		if 'There are no media! Try another search!' in req:
67	67		break
		skipped 10 lines
78	78		LINKS += set(re.findall(r"t\.me/[\w]+", req))
79	79		PAGES += req
80	80		except Exception as e:
81		-	self.error(f'Channel {link} is missed!')
	81	+	self.error(f"Channel {link} is missed!", 'telegram', 'scrap')
82	82
83	83		def module_api(self):
84	84		query = self.options['query']
		skipped 36 lines

■ ■ ■ ■ ■ ■

modules/search/worldcat.py

		skipped 44 lines
45	45		output = {'results': []}
46	46
47	47		run = self.worldcat(title, author, limit, order)
48		-	if run.search() == 'False':
	48	+	if run.run_crawl() == False:
49	49		return output
50	50
51	51		xml_root = ElementTree.fromstring(run.xml_data)
		skipped 2 lines
54	54		response_code = xml_root.findall('.//{http://classify.oclc.org}response')[0].attrib['code']
55	55		if response_code != '4': #If not success
56	56		if response_code in error_codes:
57		-	self.error(f'[Worldcat Search] {error_codes[response_code]}')
	57	+	self.error(f"{error_codes[response_code]}", 'worldcat', 'module_api')
58	58		else:
59		-	self.error('[Worldcat Search] Something went wrong!')
	59	+	self.error('Something went wrong!', 'worldcat', 'module_api')
60	60		return output
61	61
62	62		#Parsing total entries available for this query
63	63		workCount = xml_root.findall('.//{http://classify.oclc.org}workCount')
64	64		total_entries = workCount[0].text
65	65		if limit > int(total_entries):
66		-	self.alert(f'[Worldcat Search] Only {total_entries} entries available for this query instead of {limit}')
	66	+	self.alert(f'Only {total_entries} entries available for this query instead of {limit}')
67	67		limit = int(total_entries)
68	68
69	69		#Parsing book data
		skipped 16 lines

■ ■ ■ ■ ■ ■

modules/search/yahoo.py

1	+	"""
2	+	OWASP Maryam!
3	+
4	+	This program is free software: you can redistribute it and/or modify
5	+	it under the terms of the GNU General Public License as published by
6	+	the Free Software Foundation, either version 3 of the License, or
7	+	any later version.
8	+
9	+	This program is distributed in the hope that it will be useful,
10	+	but WITHOUT ANY WARRANTY; without even the implied warranty of
11	+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	+	GNU General Public License for more details.
13	+
14	+	You should have received a copy of the GNU General Public License
15	+	along with this program. If not, see <http://www.gnu.org/licenses/>.
16	+	"""
17	+
18	+	meta = {
19	+	'name': 'Yahoo Search',
20	+	'author': 'Saeed',
21	+	'version': '0.1',
22	+	'description': 'Search your query in the search.yahoo.com and show the results.',
23	+	'sources': ('yahoo',),
24	+	'options': (
25	+	('query', None, True, 'Query string', '-q', 'store', str),
26	+	('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int),
27	+	('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int),
28	+	),
29	+	'examples': ('yahoo -q <QUERY> -l 15 --output --api',)
30	+	}
31	+
32	+	def module_api(self):
33	+	query = self.options['query']
34	+	limit = self.options['limit']
35	+	count = self.options['count']
36	+	run = self.yahoo(query, limit, count)
37	+	run.run_crawl()
38	+	results = run.results
39	+	output = {'results': results}
40	+	self.save_gather(output, 'search/yahoo', query, output=self.options['output'])
41	+	return output
42	+
43	+	def module_run(self):
44	+	self.search_engine_results(module_api(self))
45	+

■ ■ ■ ■ ■ ■

requirements

1	1		requests
2	2		cloudscraper
3	3		bs4
	4	+	lxml

version 2.0.5 release