STRLCPY/snscrape

Fix KeyErrors on display_url and expanded_url for certain users with broken profile links
```
Fixes #480
```
JustAnotherArchivist committed 2 years ago

530f4fa1

1 parent dc6bc9bf

Total 1 files

■ ■ ■ ■ ■ ■

snscrape/modules/twitter.py

		skipped 1348 lines
1349	1349		if not entity or entity['url'] != user['url']:
1350	1350		self.logger.warning(f'Link inconsistency on user {kwargs["id"]}')
1351	1351		if not entity:
1352		-	entity = {'display_url': None, 'expanded_url': user['url'], 'indices': (0, len(user['url']))}
1353		-	kwargs['link'] = TextLink(text = entity['display_url'], url = entity['expanded_url'], tcourl = user['url'], indices = tuple(entity['indices']))
	1352	+	entity = {'indices': (0, len(user['url']))}
	1353	+	kwargs['link'] = TextLink(text = entity.get('display_url'), url = entity.get('expanded_url', user['url']), tcourl = user['url'], indices = tuple(entity['indices']))
1354	1354		kwargs['profileImageUrl'] = user['profile_image_url_https']
1355	1355		kwargs['profileBannerUrl'] = user.get('profile_banner_url')
1356	1356		if 'ext' in user and (label := user['ext']['highlightedLabel']['r']['ok'].get('label')):
		skipped 125 lines
1482	1482		if not entity or entity['url'] != user['legacy']['url']:
1483	1483		self.logger.warning(f'Link inconsistency on user')
1484	1484		if not entity:
1485		-	entity = {'display_url': None, 'expanded_url': user['legacy']['url'], 'indices': (0, len(user['legacy']['url']))}
1486		-	link = TextLink(text = entity['display_url'], url = entity['expanded_url'], tcourl = user['legacy']['url'], indices = tuple(entity['indices']))
	1485	+	entity = {'indices': (0, len(user['legacy']['url']))}
	1486	+	link = TextLink(text = entity.get('display_url'), url = entity.get('expanded_url', user['legacy']['url']), tcourl = user['legacy']['url'], indices = tuple(entity['indices']))
1487	1487		label = None
1488	1488		if (labelO := user['affiliates_highlighted_label'].get('label')):
1489	1489		label = self._user_label_to_user_label(labelO)
		skipped 262 lines

Fix KeyErrors on display_url and expanded_url for certain users with broken profile links