STRLCPY/Maryam

■ ■ ■ ■ ■ ■

README.md

1	1		[![Build Status](https://travis-ci.com/saeeddhqan/maryam.svg?branch=master)](https://travis-ci.com/saeeddhqan/maryam)
2		-	![Version 2.2.6](https://img.shields.io/badge/Version-2.5.0-green.svg)
	2	+	![Version 2.5.1](https://img.shields.io/badge/Version-2.5.1-green.svg)
3	3		![GPLv3 License](https://img.shields.io/badge/License-GPLv3-green.svg)
4		-	![Python 3.8.x](https://img.shields.io/badge/Python-3.8.x-green.svg)
	4	+	![Python 3.10.x](https://img.shields.io/badge/Python-3.10.x-green.svg)
5	5		[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4577/badge)](https://bestpractices.coreinfrastructure.org/projects/4577)
6	6
7	7		# OWASP Maryam
		skipped 67 lines

■ ■ ■ ■ ■ ■

maryam/core/util/engines/baidu.py

		skipped 32 lines
33	33
34	34		def run_crawl(self):
35	35		set_page = lambda x: (x - 1) * 10 + 1
36		-	urls = [f"http://{self.baidu}/s?wd={self.q}&oq={self.q}&pn={set_page(i)}&ie=utf-8" for i in range(1, self.limit+1)]
	36	+	urls = [f"https://{self.baidu}/s?wd={self.q}&oq={self.q}&pn={set_page(i)}&ie=utf-8" for i in range(1, self.limit+1)]
37	37		max_attempt = len(urls)
38	38		for url in range(len(urls)):
39	39		self.framework.verbose(f"[BAIDU] Searching in {url} page...")
		skipped 27 lines

■ ■ ■ ■ ■ ■

maryam/core/util/engines/bing.py

		skipped 21 lines
22	22		"""
23	23		self.framework = main.framework
24	24		self.q = q
25		-	self.agent = 'Lynx/2.8.5rel.1 libwww-FM/2.15FC SSL-MM/1.4.1c OpenSSL/0.9.7e-dev'
	25	+	self.agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0'
26	26		self.xpath_name = {
27	27		'results': '//li[@class="b_algo"]',
28	28		'results_content_and_cite': './/div[@class="b_caption"]',
		skipped 124 lines

■ ■ ■ ■ ■ ■

maryam/core/util/engines/metacrawler.py

		skipped 61 lines
62	62		else:
63	63		text = self.framework.to_str(req.text)
64	64		if 'To continue, please respond below:' in text:
65		-	self.framework.error('CaptchaError', 'util/enginesmetacrawler', 'run_crawl')
	65	+	self.framework.error('CaptchaError', 'util/engines/metacrawler', 'run_crawl')
66	66		self.framework.error('Metacrawler is missed!', 'util/engines/metacrawler', 'run_crawl')
67	67		break
68	68		self._pages += text
		skipped 47 lines

■ ■ ■ ■ ■ ■

maryam/core/util/engines/qwant.py

		skipped 35 lines
36	36
37	37		def run_crawl(self):
38	38		page = 1
39		-	set_page = lambda x: (x - 1) * 10 + 1
	39	+	set_page = lambda x: (x-1)*10
40	40		payload = {'q': self.q, 'offset': set_page(page), 'count': '10', 'safesearch': '0', 'device': 'desktop', 'locale': 'en_us'}
41		-	headers = {'Host': 'api.qwant.com',
42		-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0',
43		-	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8',
44		-	'Accept-Language': 'en-US,en;q=0.5en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br',
45		-	'Connection': 'keep-alive', 'Cookie': 'JSESSIONID=6120E7C52197190DE5126DCBF47D38B0',
46		-	'Upgrade-Insecure-Requests': '1', 'Cache-Control': 'max-age=0'}
47	41		while True:
48	42		self.framework.verbose(f"[QWANT] Searching in {page+1} page...")
49	43		try:
50		-	# req = self.framework.request(url=self.qwant, headers=headers, params=payload)
51	44		req = self.framework.request(url=self.qwant, params=payload)
52	45		except Exception as e:
53	46		self.framework.error(f"ConnectionError {e}.", 'util/engines/qwant', 'name_crawl')
		skipped 71 lines

■ ■ ■ ■ ■ ■

maryam/core/util/search/pastebin.py

		skipped 11 lines
12	12		along with this program. If not, see <http://www.gnu.org/licenses/>.
13	13		"""
14	14
	15	+	import re
	16	+	import concurrent.futures
	17	+
15	18		class main:
16	19		def __init__(self, q, limit=1, count=10):
17	20		""" pastebin link
18		-	q : The query domain
	21	+	q : The query
19	22		limit : Number of pages
20	23		"""
21	24		self.framework = main.framework
		skipped 6 lines
28	31		self.count = count
29	32		self.thread = 3
30	33		self._links = []
	34	+	self._links_and_titles = []
31	35		self.q_formats = {
32	36		'default_q': f'site:pastebin.com "{self.q}"',
33		-	'yippy_q': f'"pastebin.com" {self.q}',
34	37		'qwant_q': f'site:pastebin.com {self.q}'
35	38		}
36		-
37	39
38	40		def search(self, self2, name, q, q_formats, limit, count):
39	41		engine = getattr(self.framework, name)
		skipped 7 lines
47	49		attr = engine(q)
48	50
49	51		attr.run_crawl()
50		-	self._links += attr.links
51	52		self._pages += attr.pages
	53	+	self._pastebin_pages = ''
	54	+
	55	+	def open_pages(self, link):
	56	+	heading = re.search(r"pastebin\.com/([\w\d]+)", link)
	57	+	title = 'no title'
	58	+	if heading:
	59	+	head_raw = f"https://pastebin.com/raw/{heading.group(1)}"
	60	+	try:
	61	+	head_req = self.framework.request(url=head_raw).text
	62	+	except Exception as e:
	63	+	self.framework.verbose('Pastebin is missed!')
	64	+	else:
	65	+	head_title = f"{self.q} pastes {head_req.splitlines()[0].lstrip()[:30]}...".ljust(10, ' ')
	66	+	title = head_title.title()
	67	+	self._pastebin_pages += head_req
	68	+	self._links_and_titles.append([link, title])
52	69
53	70		def run_crawl(self):
54	71		self.framework.thread(self.search, self.thread, self.sources, self.q, self.q_formats, self.limit, self.count, self.sources)
	72	+	links = list(set(self.framework.reglib(self._pages).search(r"https://pastebin\.com/[\w\d]{2,}")))
	73	+	self.framework.verbose('Rearranging paste links [give it a few seconds]...')
	74	+	with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
	75	+	[executor.submit(self.open_pages, url) for url in links]
55	76
56	77		@property
57	78		def pages(self):
58		-	return self._pages
	79	+	return self._pastebin_pages
59	80
60	81		@property
61	82		def links(self):
62	83		return self._links
	84	+
	85	+	@property
	86	+	def links_and_titles(self):
	87	+	return self._links_and_titles
63	88
64	89		@property
65	90		def dns(self):
66		-	return self.framework.page_parse(self._pages).get_dns(self.q)
	91	+	return self.framework.page_parse(self._pastebin_pages).get_dns(self.q)
67	92
68	93		@property
69	94		def emails(self):
70		-	return self.framework.page_parse(self._pages).get_emails(self.q)
	95	+	return self.framework.page_parse(self._pastebin_pages).get_emails(self.q)
71	96

■ ■ ■ ■ ■ ■

maryam/modules/osint/article_search.py

		skipped 19 lines
20	20		'author': 'Kaushik',
21	21		'version': '0.1',
22	22		'description': 'Search for scientific papers and articles from Google Scholar \
23		-	, Arxiv, Pubmed, and Core.ac.uk',
24		-	'sources': ('scholar', 'arxiv', 'pubmed', 'core_ac'),
	23	+	, Arxiv, and Pubmed',
	24	+	'sources': ('scholar', 'arxiv', 'pubmed'),
25	25		'options': (
26	26		('query', None, True, 'Search query for papers or articles', '-q', 'store', str),
27	27		('limit', 15, False, 'Max result count (default=15)', '-l', 'store', int),
		skipped 35 lines

■ ■ ■ ■ ■ ■

maryam/modules/osint/dns_search.py

		skipped 20 lines
21	21		'author': 'Saeed',
22	22		'version': '2.5',
23	23		'description': 'Search in the open-sources to find subdomans.',
24		-	'sources': ('securitytrails', 'bing', 'google', 'yahoo', 'yandex', 'metacrawler', 'ask', 'baidu', 'startpage',
	24	+	'sources': ('securitytrails', 'bing', 'google', 'yahoo', 'metacrawler', 'ask', 'baidu', 'startpage',
25	25		'netcraft', 'threatcrowd', 'virustotal', 'otx', 'carrot2', 'crt',
26	26		'qwant', 'millionshort', 'threatminer', 'jldc', 'bufferover', 'rapiddns', 'certspotter',
27	27		'sublist3r', 'riddler', 'sitedossier', 'duckduckgo', 'dnsdumpster', 'yougetsignal', 'pastebin',
		skipped 337 lines

■ ■ ■ ■ ■ ■

maryam/modules/osint/docs_search.py

		skipped 16 lines
17	17		'author': 'Saeed',
18	18		'version': '0.8',
19	19		'description': 'Search in open-sources to find relevant documents. filetypes[pdf,doc,docx,ppt,pptx,xlsx,txt,..].',
20		-	'sources': ('bing', 'google', 'yahoo', 'yandex', 'metacrawler', 'ask',
	20	+	'sources': ('bing', 'google', 'yahoo', 'metacrawler', 'ask',
21	21		'startpage', 'exalead', 'carrot2', 'qwant', 'millionshort', 'duckduckgo', 'gigablast', 'dogpile'),
22	22		'options': (
23	23		('query', None, True, 'Host Name, Company Name, keyword, query, etc', '-q', 'store', str),
		skipped 52 lines

■ ■ ■ ■ ■ ■

maryam/modules/osint/email_search.py

		skipped 16 lines
17	17		'author': 'Saeed',
18	18		'version': '1.0',
19	19		'description': 'Search in open-sources to find emails.',
20		-	'sources': ('bing', 'pastebin', 'google', 'yahoo', 'yandex', 'metacrawler',
21		-	'ask', 'baidu', 'startpage', 'qwant', 'duckduckgo', 'hunter', 'gigablast', 'github'),
	20	+	'sources': ('bing', 'pastebin', 'google', 'yahoo', 'metacrawler',
	21	+	'baidu', 'startpage', 'qwant', 'duckduckgo', 'hunter', 'gigablast', 'github'),
22	22		'options': (
23	23		('query', None, True, 'Domain name or company name', '-q', 'store', str),
24	24		('limit', 3, False, 'Search limit(number of pages, default=3)', '-l', 'store', int),
		skipped 60 lines

■ ■ ■ ■ ■ ■

maryam/modules/osint/social_nets.py

		skipped 16 lines
17	17		'author': 'Saeed',
18	18		'version': '1.5',
19	19		'description': 'Search to find Usernames in social networks.',
20		-	'sources': ('bing', 'google', 'yahoo', 'yandex', 'metacrawler', 'ask', 'startpage', 'urlscan'),
	20	+	'sources': ('bing', 'google', 'yahoo', 'metacrawler', 'ask', 'startpage', 'urlscan'),
21	21		'options': (
22	22		('query', None, True, 'Company Name or Query', '-q', 'store', str),
23	23		('engines', 'google,bing', False, 'Search engine names. e.g `bing,google,..`', '-e', 'store', str),
		skipped 68 lines

■ ■ ■ ■ ■ ■

maryam/modules/search/pastebin.py

		skipped 20 lines
21	21		meta = {
22	22		'name': 'Pastebin Search',
23	23		'author': 'Divya Goswami',
24		-	'version': '1.0',
	24	+	'version': '1.1',
25	25		'description': 'Search your query in the pastebin.com and show the results as paste links.',
26	26		'sources': ('pastebin',),
27	27		'options': (
28	28		('query', None, True, 'Query string', '-q', 'store', str),
29	29		('limit', 1, False, 'Search limit(number of pages, default=1)', '-l', 'store', int),
30	30		('count', 50, False, 'Number of results per page(min=10, max=100, default=50)', '-c', 'store', int),
31		-	('thread', 3, False, 'The number of engine that run per round(default=2)', '-t', 'store', int),
32		-	('engine', 'pastebin', False, 'Engine names for search', '-e', 'store', str),
33	31		),
34	32		'examples': ('pastebin -q passwords -l 15 --output',)
35	33		}
36	34
37		-	LINKS = []
38		-	PAGES = ''
39		-
40		-	def search(self, name, q, q_formats, limit, count):
41		-	global PAGES,LINKS
42		-	engine = getattr(self, name)
43		-	q = q_formats[f"{name}_q"] if f"{name}_q" in q_formats else q_formats['default_q']
44		-	varnames = engine.__init__.__code__.co_varnames
45		-	if 'limit' in varnames and 'count' in varnames:
46		-	attr = engine(q, limit, count)
47		-	elif 'limit' in varnames:
48		-	attr = engine(q, limit)
49		-	else:
50		-	attr = engine(q)
51		-
52		-	attr.run_crawl()
53		-	LINKS += attr.links
54		-	PAGES += attr.pages
55	35
56	36		def module_api(self):
57	37		query = self.options['query']
58	38		limit = self.options['limit']
59	39		count = self.options['count']
60		-	engine = self.options['engine'].split(',')
61	40		output = {'pastes': []}
62		-	q_formats = {
63		-	'default_q': f"{query}"
64		-	}
65		-	self.thread(search, self.options['thread'], engine, query, q_formats, limit, count, meta['sources'])
66		-	links = list(self.reglib().filter(r"https?://pastebin\.com/[\w\d]{2,}", list(set(LINKS))))
67		-	self.verbose('Rearranging paste links [give it a few seconds]...')
68		-	for link in links:
69		-	heading = re.search(r"pastebin\.com/([\w\d]+)", link)
70		-	if heading:
71		-	head_raw = f"https://pastebin.com/raw/{heading.group(1)}"
72		-	try:
73		-	head_req = self.request(url=head_raw).text.splitlines()[0].lstrip()
74		-	except Exception as e:
75		-	self.verbose('Pastebin is missed!')
76		-	else:
77		-	head_title = f"{query} pastes {head_req[:30]}...".ljust(10, ' ')
78		-	title = head_title.title()
79		-	output['pastes'].append([link, title])
	41	+	pastebin = self.pastebin(query, limit, count)
	42	+	pastebin.run_crawl()
	43	+	output['pastes'] = pastebin.links_and_titles
80	44		self.save_gather(output, 'search/pastebin', query, output=self.options.get('output'))
81	45		return output
82	46
		skipped 8 lines

■ ■ ■ ■ ■ ■

setup.py

		skipped 24 lines
25	25
26	26		setup(
27	27		name='maryam',
28		-	version='2.5.0-1',
	28	+	version='2.5.1',
29	29		url='https://github.com/saeeddhqan/Maryam',
30	30		author='Saeed Dehqan',
31	31		author_email='[email protected]',
		skipped 8 lines
40	40		scripts=['bin/maryam'],
41	41		install_requires=requirements,
42	42		classifiers=[
43		-	'Programming Language :: Python :: 3.8',
	43	+	'Programming Language :: Python :: 3.10',
44	44		'Development Status :: 5 - Production/Stable',
45	45		'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
46	46		'Natural Language :: English',
		skipped 5 lines

version 1.5.1 release