| skipped 16 lines |
| 17 | 17 | | import getpass |
| 18 | 18 | | import click |
| 19 | 19 | | import re |
| 20 | | - | import textwrap |
| 21 | 20 | | import time |
| 22 | | - | import pprint |
| | 21 | + | |
| 23 | 22 | | |
| 24 | | - | from telepathy.utils import ( |
| | 23 | + | from utils import ( |
| 25 | 24 | | print_banner, |
| 26 | 25 | | color_print_green, |
| 27 | 26 | | populate_user, |
| skipped 1 lines |
| 29 | 28 | | process_description, |
| 30 | 29 | | parse_tg_date, |
| 31 | 30 | | parse_html_page, |
| 32 | | - | print_shell |
| | 31 | + | print_shell, |
| | 32 | + | createPlaceholdeCls |
| 33 | 33 | | ) |
| 34 | 34 | | import const as const |
| 35 | 35 | | |
| skipped 13 lines |
| 49 | 49 | | from telethon import TelegramClient, functions, types, utils |
| 50 | 50 | | from telethon.utils import get_display_name, get_message_id |
| 51 | 51 | | from alive_progress import alive_bar |
| 52 | | - | from bs4 import BeautifulSoup |
| 53 | | - | |
| 54 | 52 | | |
| 55 | 53 | | @click.command() |
| 56 | 54 | | @click.option( |
| skipped 152 lines |
| 209 | 207 | | exports = [] |
| 210 | 208 | | |
| 211 | 209 | | print("Exporting...") |
| 212 | | - | |
| 213 | 210 | | # progress bar |
| 214 | | - | |
| 215 | 211 | | for Dialog in await client.get_dialogs(): |
| 216 | 212 | | try: |
| 217 | 213 | | if Dialog.entity.username: |
| skipped 78 lines |
| 296 | 292 | | |
| 297 | 293 | | except AttributeError: |
| 298 | 294 | | pass |
| 299 | | - | |
| 300 | 295 | | else: |
| 301 | | - | |
| 302 | 296 | | for t in target: |
| 303 | 297 | | target_clean = t |
| 304 | 298 | | alphanumeric = "" |
| 305 | 299 | | |
| 306 | | - | |
| 307 | 300 | | for character in target_clean: |
| 308 | 301 | | if character.isalnum(): |
| 309 | 302 | | alphanumeric += character |
| skipped 116 lines |
| 426 | 419 | | ] |
| 427 | 420 | | |
| 428 | 421 | | translated_description = _desc["translated_text"] |
| 429 | | - | |
| 430 | | - | preferredWidth = 70 |
| 431 | | - | descript = Fore.GREEN + "Description: " + Style.RESET_ALL |
| 432 | | - | prefix = descript |
| 433 | | - | wrapper_d = textwrap.TextWrapper( |
| 434 | | - | initial_indent=prefix, |
| 435 | | - | width=preferredWidth, |
| 436 | | - | subsequent_indent=" ", |
| 437 | | - | ) |
| 438 | | - | |
| 439 | | - | trans_descript = Fore.GREEN + "Translated: " + Style.RESET_ALL |
| 440 | | - | prefix = trans_descript |
| 441 | | - | wrapper_td = textwrap.TextWrapper( |
| 442 | | - | initial_indent=prefix, |
| 443 | | - | width=preferredWidth, |
| 444 | | - | subsequent_indent=" ", |
| 445 | | - | ) |
| 446 | | - | |
| 447 | 422 | | group_description = ('"' + group_description + '"') |
| 448 | 423 | | |
| 449 | 424 | | if entity.broadcast is True: |
| skipped 17 lines |
| 467 | 442 | | else: |
| 468 | 443 | | group_status = "None" |
| 469 | 444 | | |
| 470 | | - | restrict = Fore.GREEN + "Restrictions:" + Style.RESET_ALL |
| 471 | | - | prefix = restrict + " " |
| 472 | | - | preferredWidth = 70 |
| 473 | | - | wrapper_r = textwrap.TextWrapper( |
| 474 | | - | initial_indent=prefix, |
| 475 | | - | width=preferredWidth, |
| 476 | | - | subsequent_indent=" ", |
| 477 | | - | ) |
| 478 | 445 | | found_participants = 0 |
| 479 | | - | found_participants = 0 |
| 480 | | - | |
| | 446 | + | found_percentage = 0 |
| 481 | 447 | | if chat_type != "Channel": |
| 482 | 448 | | members = [] |
| 483 | | - | all_participants = [] |
| 484 | 449 | | all_participants = await client.get_participants(t, limit=5000) |
| 485 | | - | |
| 486 | 450 | | members_df = None |
| 487 | 451 | | for user in all_participants: |
| 488 | 452 | | members_df = pd.DataFrame( |
| skipped 225 lines |
| 714 | 678 | | df01 = value_count.rename_axis("unique_values").reset_index( |
| 715 | 679 | | name="counts" |
| 716 | 680 | | ) |
| 717 | | - | |
| 718 | | - | report_forward = object() |
| 719 | | - | setattr(report_forward, "forward_one", ( |
| | 681 | + | report_forward = createPlaceholdeCls() |
| | 682 | + | report_forward.forward_one = ( |
| 720 | 683 | | str(df01.iloc[0]["unique_values"]) |
| 721 | 684 | | + ", " |
| 722 | 685 | | + str(df01.iloc[0]["counts"]) |
| 723 | 686 | | + " forwarded messages" |
| 724 | | - | )) |
| 725 | | - | setattr(report_forward, "forward_two", ( |
| 726 | | - | str(df01.iloc[1]["unique_values"]) |
| 727 | | - | + ", " |
| 728 | | - | + str(df01.iloc[1]["counts"]) |
| 729 | | - | + " forwarded messages" |
| 730 | | - | )) |
| 731 | | - | setattr(report_forward, "forward_three", ( |
| 732 | | - | str(df01.iloc[2]["unique_values"]) |
| 733 | | - | + ", " |
| 734 | | - | + str(df01.iloc[2]["counts"]) |
| 735 | | - | + " forwarded messages" |
| 736 | | - | )) |
| 737 | | - | setattr(report_forward, "forward_four", ( |
| 738 | | - | str(df01.iloc[3]["unique_values"]) |
| 739 | | - | + ", " |
| 740 | | - | + str(df01.iloc[3]["counts"]) |
| 741 | | - | + " forwarded messages" |
| 742 | | - | )) |
| 743 | | - | setattr(report_forward, "forward_five", ( |
| 744 | | - | str(df01.iloc[4]["unique_values"]) |
| 745 | | - | + ", " |
| 746 | | - | + str(df01.iloc[4]["counts"]) |
| 747 | | - | + " forwarded messages" |
| 748 | | - | )) |
| 749 | | - | |
| | 687 | + | ) |
| | 688 | + | report_forward.forward_two = ( |
| | 689 | + | str(df01.iloc[1]["unique_values"]) |
| | 690 | + | + ", " |
| | 691 | + | + str(df01.iloc[1]["counts"]) |
| | 692 | + | + " forwarded messages" |
| | 693 | + | ) |
| | 694 | + | report_forward.forward_three = ( |
| | 695 | + | str(df01.iloc[2]["unique_values"]) |
| | 696 | + | + ", " |
| | 697 | + | + str(df01.iloc[2]["counts"]) |
| | 698 | + | + " forwarded messages" |
| | 699 | + | ) |
| | 700 | + | report_forward.forward_four = ( |
| | 701 | + | str(df01.iloc[3]["unique_values"]) |
| | 702 | + | + ", " |
| | 703 | + | + str(df01.iloc[3]["counts"]) |
| | 704 | + | + " forwarded messages" |
| | 705 | + | ) |
| | 706 | + | report_forward.forward_five = ( |
| | 707 | + | str(df01.iloc[4]["unique_values"]) |
| | 708 | + | + ", " |
| | 709 | + | + str(df01.iloc[4]["counts"]) |
| | 710 | + | + " forwarded messages" |
| | 711 | + | ) |
| 750 | 712 | | df02 = forwards_df.Source.unique() |
| 751 | | - | setattr(report_forward, "unique_forwards", len(df02)) |
| 752 | | - | setattr(report_forward, "edgelist_file", len(df02)) |
| | 713 | + | report_forward.unique_forwards = len(df02) |
| | 714 | + | report_forward.edgelist_file = edgelist_file |
| 753 | 715 | | print_shell("forwarder_stat",report_forward) |
| 754 | 716 | | |
| 755 | 717 | | else: |
| skipped 235 lines |
| 991 | 953 | | else: |
| 992 | 954 | | has_media = 'FALSE' |
| 993 | 955 | | |
| 994 | | - | post_url = "https://t.me/s/" + t + "/" + message.id |
| | 956 | + | post_url = "https://t.me/s/" + t + "/" + str(message.id) |
| 995 | 957 | | |
| 996 | 958 | | message_list.append( |
| 997 | 959 | | [ |
| skipped 191 lines |
| 1189 | 1151 | | pass |
| 1190 | 1152 | | |
| 1191 | 1153 | | messages_found = int(c_archive.To.count()) - 1 |
| 1192 | | - | report_obj = object() |
| 1193 | | - | setattr(report_obj,"messages_found", messages_found) |
| 1194 | | - | setattr(report_obj,"file_archive", file_archive) |
| | 1154 | + | report_obj = createPlaceholdeCls() |
| | 1155 | + | report_obj.messages_found = messages_found |
| | 1156 | + | report_obj.file_archive = file_archive |
| | 1157 | + | |
| 1195 | 1158 | | if chat_type == "Channel": |
| 1196 | 1159 | | print_shell("channel_stat", report_obj) |
| 1197 | 1160 | | else: |
| skipped 11 lines |
| 1209 | 1172 | | ''' |
| 1210 | 1173 | | #message stats, top words |
| 1211 | 1174 | | |
| 1212 | | - | setattr(report_obj, "poster_one", ( |
| | 1175 | + | report_obj.poster_one = ( |
| 1213 | 1176 | | str(df03.iloc[0]["unique_values"]) |
| 1214 | 1177 | | + ", " |
| 1215 | 1178 | | + str(df03.iloc[0]["counts"]) |
| 1216 | 1179 | | + " messages" |
| 1217 | | - | )) |
| 1218 | | - | setattr(report_obj, "poster_two", ( |
| | 1180 | + | ) |
| | 1181 | + | |
| | 1182 | + | report_obj.poster_two = ( |
| 1219 | 1183 | | str(df03.iloc[1]["unique_values"]) |
| 1220 | 1184 | | + ", " |
| 1221 | 1185 | | + str(df03.iloc[2]["counts"]) |
| 1222 | 1186 | | + " messages" |
| 1223 | | - | )) |
| 1224 | | - | setattr(report_obj, "poster_three", ( |
| | 1187 | + | ) |
| | 1188 | + | |
| | 1189 | + | report_obj.poster_three = ( |
| 1225 | 1190 | | str(df03.iloc[2]["unique_values"]) |
| 1226 | 1191 | | + ", " |
| 1227 | 1192 | | + str(df03.iloc[2]["counts"]) |
| 1228 | 1193 | | + " messages" |
| 1229 | | - | )) |
| 1230 | | - | setattr(report_obj, "poster_four", ( |
| | 1194 | + | ) |
| | 1195 | + | |
| | 1196 | + | report_obj.poster_four = ( |
| 1231 | 1197 | | str(df03.iloc[3]["unique_values"]) |
| 1232 | 1198 | | + ", " |
| 1233 | 1199 | | + df03.iloc[3]["counts"] |
| 1234 | 1200 | | + " messages" |
| 1235 | | - | )) |
| 1236 | | - | setattr(report_obj, "poster_five", ( |
| | 1201 | + | ) |
| | 1202 | + | |
| | 1203 | + | report_obj.poster_five = ( |
| 1237 | 1204 | | str(df03.iloc[4]["unique_values"]) |
| 1238 | 1205 | | + ", " |
| 1239 | 1206 | | + str(df03.iloc[4]["counts"]) |
| 1240 | 1207 | | + " messages" |
| 1241 | | - | )) |
| | 1208 | + | ) |
| 1242 | 1209 | | |
| 1243 | 1210 | | df04 = c_archive.Display_name.unique() |
| 1244 | 1211 | | unique_active = len(df04) |
| 1245 | | - | setattr(report_obj, "unique_active", unique_active) |
| | 1212 | + | report_obj.unique_active = unique_active |
| 1246 | 1213 | | print_shell("group_stat", report_obj) |
| 1247 | 1214 | | |
| 1248 | 1215 | | if reply_analysis is True: |
| skipped 4 lines |
| 1253 | 1220 | | "unique_values" |
| 1254 | 1221 | | ).reset_index(name="counts") |
| 1255 | 1222 | | |
| 1256 | | - | repliers = object() |
| 1257 | | - | setattr(repliers, "replier_one", ( |
| | 1223 | + | repliers = createPlaceholdeCls() |
| | 1224 | + | repliers.replier_one = ( |
| 1258 | 1225 | | str(replier_df.iloc[0]["unique_values"]) |
| 1259 | 1226 | | + ", " |
| 1260 | 1227 | | + str(replier_df.iloc[0]["counts"]) |
| 1261 | 1228 | | + " replies" |
| 1262 | | - | )) |
| 1263 | | - | setattr(repliers, "replier_two", ( |
| | 1229 | + | ) |
| | 1230 | + | |
| | 1231 | + | repliers.replier_two = ( |
| 1264 | 1232 | | str(replier_df.iloc[1]["unique_values"]) |
| 1265 | 1233 | | + ", " |
| 1266 | 1234 | | + str(replier_df.iloc[1]["counts"]) |
| 1267 | 1235 | | + " replies" |
| 1268 | | - | )) |
| 1269 | | - | setattr(repliers, "replier_three", ( |
| | 1236 | + | ) |
| | 1237 | + | |
| | 1238 | + | repliers.replier_three = ( |
| 1270 | 1239 | | str(replier_df.iloc[2]["unique_values"]) |
| 1271 | 1240 | | + ", " |
| 1272 | 1241 | | + str(replier_df.iloc[2]["counts"]) |
| 1273 | 1242 | | + " replies" |
| 1274 | | - | )) |
| 1275 | | - | setattr(repliers, "replier_four", ( |
| | 1243 | + | ) |
| | 1244 | + | |
| | 1245 | + | repliers.replier_four = ( |
| 1276 | 1246 | | str(replier_df.iloc[3]["unique_values"]) |
| 1277 | 1247 | | + ", " |
| 1278 | 1248 | | + str(replier_df.iloc[3]["counts"]) |
| 1279 | 1249 | | + " replies" |
| 1280 | | - | )) |
| 1281 | | - | setattr(repliers, "replier_five", ( |
| | 1250 | + | ) |
| | 1251 | + | |
| | 1252 | + | repliers.replier_five = ( |
| 1282 | 1253 | | str(replier_df.iloc[3]["counts"]) |
| 1283 | 1254 | | + ", " |
| 1284 | 1255 | | + str(replier_df.iloc[4]["counts"]) |
| 1285 | 1256 | | + " replies" |
| 1286 | | - | )) |
| | 1257 | + | ) |
| | 1258 | + | |
| 1287 | 1259 | | replier_count_df = c_repliers["User ID"].unique() |
| 1288 | 1260 | | replier_unique = len(replier_count_df) |
| 1289 | | - | setattr(repliers,"user_replier_list_len", len(user_replier_list)) |
| 1290 | | - | setattr(repliers, "reply_file_archive", str(reply_file_archive)) |
| 1291 | | - | setattr(repliers, "reply_memberlist_filename", str(reply_memberlist_filename)) |
| 1292 | | - | setattr(repliers,"replier_unique", str(replier_unique)) |
| 1293 | | - | print_shell("reply_stat",repliers) |
| | 1261 | + | |
| | 1262 | + | repliers.user_replier_list_len = len(user_replier_list) |
| | 1263 | + | repliers.reply_file_archive = str(reply_file_archive) |
| | 1264 | + | repliers.reply_memberlist_filename = str(reply_memberlist_filename) |
| | 1265 | + | repliers.replier_unique = str(replier_unique) |
| | 1266 | + | print_shell("reply_stat", repliers) |
| 1294 | 1267 | | |
| 1295 | 1268 | | if forwards_check is True: |
| 1296 | 1269 | | if forward_count >= 15: |
| skipped 2 lines |
| 1299 | 1272 | | c_f_stats = value_count.rename_axis( |
| 1300 | 1273 | | "unique_values" |
| 1301 | 1274 | | ).reset_index(name="counts") |
| 1302 | | - | report_forward = object() |
| 1303 | | - | setattr(report_forward, "forward_one", ( |
| | 1275 | + | report_forward = createPlaceholdeCls() |
| | 1276 | + | report_forward.forward_one = ( |
| 1304 | 1277 | | str(c_f_stats.iloc[0]["unique_values"]) |
| 1305 | 1278 | | + ", " |
| 1306 | 1279 | | + str(c_f_stats.iloc[0]["counts"]) |
| 1307 | 1280 | | + " forwarded messages" |
| 1308 | | - | )) |
| 1309 | | - | setattr(report_forward, "forward_two", ( |
| | 1281 | + | ) |
| | 1282 | + | report_forward.forward_two = ( |
| 1310 | 1283 | | str(c_f_stats.iloc[1]["unique_values"]) |
| 1311 | 1284 | | + ", " |
| 1312 | 1285 | | + str(c_f_stats.iloc[1]["counts"]) |
| 1313 | 1286 | | + " forwarded messages" |
| 1314 | | - | )) |
| 1315 | | - | setattr(report_forward, "forward_three", ( |
| | 1287 | + | ) |
| | 1288 | + | report_forward.forward_three = ( |
| 1316 | 1289 | | str(c_f_stats.iloc[2]["unique_values"]) |
| 1317 | 1290 | | + ", " |
| 1318 | 1291 | | + str(c_f_stats.iloc[2]["counts"]) |
| 1319 | 1292 | | + " forwarded messages" |
| 1320 | | - | )) |
| 1321 | | - | setattr(report_forward, "forward_four", ( |
| | 1293 | + | ) |
| | 1294 | + | report_forward.forward_four = ( |
| 1322 | 1295 | | str(c_f_stats.iloc[3]["unique_values"]) |
| 1323 | 1296 | | + ", " |
| 1324 | 1297 | | + str(c_f_stats.iloc[3]["counts"]) |
| 1325 | 1298 | | + " forwarded messages" |
| 1326 | | - | )) |
| 1327 | | - | setattr(report_forward, "forward_five", ( |
| | 1299 | + | ) |
| | 1300 | + | report_forward.forward_five = ( |
| 1328 | 1301 | | str(c_f_stats.iloc[4]["unique_values"]) |
| 1329 | 1302 | | + ", " |
| 1330 | 1303 | | + str(c_f_stats.iloc[4]["counts"]) |
| 1331 | 1304 | | + " forwarded messages" |
| 1332 | | - | )) |
| | 1305 | + | ) |
| | 1306 | + | |
| 1333 | 1307 | | c_f_unique = c_forwards.Source.unique() |
| 1334 | | - | setattr(report_forward, "unique_forwards", len(c_f_unique)) |
| 1335 | | - | setattr(report_forward, "edgelist_file", len(df02)) |
| 1336 | | - | setattr(report_forward, "private_count", private_count) |
| | 1308 | + | report_forward.unique_forwards = len(c_f_unique) |
| | 1309 | + | report_forward.edgelist_file = edgelist_file |
| | 1310 | + | report_forward.private_count = private_count |
| 1337 | 1311 | | print_shell("forwarder_stat", report_forward) |
| 1338 | 1312 | | else: |
| 1339 | 1313 | | #print("\n") |
| skipped 119 lines |
| 1459 | 1433 | | except: |
| 1460 | 1434 | | pass |
| 1461 | 1435 | | |
| 1462 | | - | distance_obj = object() |
| 1463 | | - | setattr(distance_obj, "d500", 0) |
| 1464 | | - | setattr(distance_obj, "d1000", 0) |
| 1465 | | - | setattr(distance_obj, "d2000", 0) |
| 1466 | | - | setattr(distance_obj, "d3000", 0) |
| | 1436 | + | distance_obj = createPlaceholdeCls() |
| | 1437 | + | distance_obj.d500 = 0 |
| | 1438 | + | distance_obj.d1000 = 0 |
| | 1439 | + | distance_obj.d2000 = 0 |
| | 1440 | + | distance_obj.d3000 = 0 |
| 1467 | 1441 | | |
| 1468 | 1442 | | for account, distance in user_df.itertuples(index=False): |
| 1469 | 1443 | | account = int(account) |
| skipped 16 lines |
| 1486 | 1460 | | ) as f: # could one day append, including access time to differentiate |
| 1487 | 1461 | | user_df.to_csv(f, sep=";", index=False) |
| 1488 | 1462 | | |
| 1489 | | - | setattr(distance_obj, "save_file", save_file) |
| 1490 | | - | setattr(distance_obj, "total", total) |
| 1491 | 1463 | | total = len(locations_list) |
| | 1464 | + | distance_obj.save_file = save_file |
| | 1465 | + | distance_obj.total = total |
| 1492 | 1466 | | print_shell("location_report",distance_obj) |
| 1493 | 1467 | | # can also do the same for channels with similar output file to users |
| 1494 | 1468 | | # may one day add trilateration to find users closest to exact point |
| skipped 7 lines |
Giacomo Giallombardo
committed
2 years ago
1 parent a5ce5b9b