STRLCPY/snscrape

added capability to scrape multiple videos from a single post
Tristan Lee committed 2 years ago

d32c9add

1 parent fb8d73ac

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

Total 1 files

■ ■ ■ ■ ■ ■

snscrape/modules/telegram.py

		skipped 30 lines
31	31		content: str
32	32		outlinks: list
33	33		images: list
34		-	video: str
	34	+	videos: list
35	35		forwarded: str
36	36		linkPreview: typing.Optional[LinkPreview] = None
37	37
		skipped 56 lines
94	94		url = rawUrl.replace('//t.me/', '//t.me/s/')
95	95		date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z')
96	96		images = []
97		-	video = None
	97	+	videos = []
98	98		forwarded = None
99	99		if (message := post.find('div', class_ = 'tgme_widget_message_text')):
100	100		content = message.get_text(separator="\n")
101	101
102	102		for video_tag in post.find_all('video'):
103		-	video = video_tag['src']
	103	+	videos.append(video_tag['src'])
104	104
105	105		if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')):
106	106		forwarded = forward_tag['href'].split('t.me/')[1].split('/')[0]
		skipped 27 lines
134	134		content = None
135	135		outlinks = []
136	136		images = []
137		-	video = None
	137	+	videos = []
138	138		linkPreview = None
139	139		if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')):
140	140		kwargs = {}
		skipped 10 lines
151	151		else:
152	152		_logger.warning(f'Could not process link preview image on {url}')
153	153		linkPreview = LinkPreview(**kwargs)
154		-	yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, video = video, forwarded = forwarded)
	154	+	yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, videos = videos, forwarded = forwarded)
155	155
156	156		def get_items(self):
157	157		r, soup = self._initial_page()
		skipped 77 lines

added capability to scrape multiple videos from a single post