1 | 1 | | import csv |
2 | | - | import json |
3 | 2 | | import io |
| 3 | + | import json |
4 | 4 | | import logging |
5 | 5 | | import os |
| 6 | + | from argparse import ArgumentTypeError |
| 7 | + | from datetime import datetime |
| 8 | + | |
6 | 9 | | import pycountry |
7 | 10 | | import xmind |
8 | | - | from datetime import datetime |
| 11 | + | from dateutil.parser import parse as parse_datetime_str |
9 | 12 | | from jinja2 import Template |
10 | 13 | | from xhtml2pdf import pisa |
11 | | - | from argparse import ArgumentTypeError |
12 | | - | from dateutil.parser import parse as parse_datetime_str |
13 | 14 | | |
14 | 15 | | from .result import QueryStatus |
15 | 16 | | from .utils import is_country_tag, CaseConverter, enrich_link_str |
| skipped 2 lines |
18 | 19 | | 'simple', |
19 | 20 | | 'ndjson', |
20 | 21 | | ] |
21 | | - | |
22 | 22 | | |
23 | 23 | | ''' |
24 | 24 | | UTILS |
25 | 25 | | ''' |
| 26 | + | |
| 27 | + | |
26 | 28 | | def filter_supposed_data(data): |
27 | 29 | | ### interesting fields |
28 | 30 | | allowed_fields = ['fullname', 'gender', 'location', 'age'] |
| skipped 6 lines |
35 | 37 | | ''' |
36 | 38 | | REPORTS SAVING |
37 | 39 | | ''' |
| 40 | + | |
| 41 | + | |
38 | 42 | | def save_csv_report(filename: str, username: str, results: dict): |
39 | 43 | | with open(filename, 'w', newline='', encoding='utf-8') as f: |
40 | 44 | | generate_csv_report(username, results, f) |
| skipped 16 lines |
57 | 61 | | filled_template = template.render(**context) |
58 | 62 | | with open(filename, 'w+b') as f: |
59 | 63 | | pisa.pisaDocument(io.StringIO(filled_template), dest=f, default_css=css) |
| 64 | + | |
60 | 65 | | |
61 | 66 | | def save_json_report(filename: str, username: str, results: dict, report_type: str): |
62 | 67 | | with open(filename, 'w', encoding='utf-8') as f: |
| skipped 3 lines |
66 | 71 | | ''' |
67 | 72 | | REPORTS GENERATING |
68 | 73 | | ''' |
| 74 | + | |
| 75 | + | |
69 | 76 | | def generate_report_template(is_pdf: bool): |
70 | 77 | | """ |
71 | 78 | | HTML/PDF template generation |
72 | 79 | | """ |
| 80 | + | |
73 | 81 | | def get_resource_content(filename): |
74 | 82 | | return open(os.path.join(maigret_path, 'resources', filename)).read() |
75 | 83 | | |
| skipped 36 lines |
112 | 120 | | continue |
113 | 121 | | |
114 | 122 | | status = dictionary.get('status') |
| 123 | + | if not status: # FIXME: currently in case of timeout |
| 124 | + | continue |
| 125 | + | |
115 | 126 | | if status.ids_data: |
116 | 127 | | dictionary['ids_data'] = status.ids_data |
117 | 128 | | extended_info_count += 1 |
| skipped 48 lines |
166 | 177 | | for t in status.tags: |
167 | 178 | | tags[t] = tags.get(t, 0) + 1 |
168 | 179 | | |
169 | | - | |
170 | 180 | | brief_text.append(f'Search by {id_type} {username} returned {found_accounts} accounts.') |
171 | 181 | | |
172 | 182 | | if new_ids: |
| skipped 4 lines |
177 | 187 | | |
178 | 188 | | brief_text.append(f'Extended info extracted from {extended_info_count} accounts.') |
179 | 189 | | |
180 | | - | |
181 | | - | |
182 | 190 | | brief = ' '.join(brief_text).strip() |
183 | 191 | | tuple_sort = lambda d: sorted(d, key=lambda x: x[1], reverse=True) |
184 | 192 | | |
| skipped 36 lines |
221 | 229 | | results[site]['url_user'], |
222 | 230 | | str(results[site]['status'].status), |
223 | 231 | | results[site]['http_status'], |
224 | | - | ]) |
| 232 | + | ]) |
225 | 233 | | |
226 | 234 | | |
227 | 235 | | def generate_txt_report(username: str, results: dict, file): |
| skipped 25 lines |
253 | 261 | | |
254 | 262 | | if is_report_per_line: |
255 | 263 | | data['sitename'] = sitename |
256 | | - | file.write(json.dumps(data)+'\n') |
| 264 | + | file.write(json.dumps(data) + '\n') |
257 | 265 | | else: |
258 | 266 | | all_json[sitename] = data |
259 | 267 | | |
260 | 268 | | if not is_report_per_line: |
261 | 269 | | file.write(json.dumps(all_json)) |
262 | 270 | | |
| 271 | + | |
263 | 272 | | ''' |
264 | 273 | | XMIND 8 Functions |
265 | 274 | | ''' |
| 275 | + | |
| 276 | + | |
266 | 277 | | def save_xmind_report(filename, username, results): |
267 | 278 | | if os.path.exists(filename): |
268 | 279 | | os.remove(filename) |
| skipped 8 lines |
277 | 288 | | alltags = {} |
278 | 289 | | supposed_data = {} |
279 | 290 | | |
280 | | - | sheet.setTitle("%s Analysis"%(username)) |
| 291 | + | sheet.setTitle("%s Analysis" % (username)) |
281 | 292 | | root_topic1 = sheet.getRootTopic() |
282 | | - | root_topic1.setTitle("%s"%(username)) |
| 293 | + | root_topic1.setTitle("%s" % (username)) |
283 | 294 | | |
284 | 295 | | undefinedsection = root_topic1.addSubTopic() |
285 | 296 | | undefinedsection.setTitle("Undefined") |
| skipped 47 lines |
333 | 344 | | currentsublabel.setTitle("%s: %s" % (k, currentval)) |
334 | 345 | | ### Add Supposed DATA |
335 | 346 | | filterede_supposed_data = filter_supposed_data(supposed_data) |
336 | | - | if(len(filterede_supposed_data) >0): |
| 347 | + | if (len(filterede_supposed_data) > 0): |
337 | 348 | | undefinedsection = root_topic1.addSubTopic() |
338 | 349 | | undefinedsection.setTitle("SUPPOSED DATA") |
339 | 350 | | for k, v in filterede_supposed_data.items(): |
| skipped 4 lines |
344 | 355 | | def check_supported_json_format(value): |
345 | 356 | | if value and not value in SUPPORTED_JSON_REPORT_FORMATS: |
346 | 357 | | raise ArgumentTypeError(f'JSON report type must be one of the following types: ' |
347 | | - | + ', '.join(SUPPORTED_JSON_REPORT_FORMATS)) |
| 358 | + | + ', '.join(SUPPORTED_JSON_REPORT_FORMATS)) |
348 | 359 | | return value |
349 | 360 | | |
350 | | - | |