| skipped 8 lines |
9 | 9 | | from urllib.parse import quote |
10 | 10 | | |
11 | 11 | | import aiohttp |
| 12 | + | import aiodns |
12 | 13 | | import tqdm.asyncio |
13 | 14 | | from aiohttp_socks import ProxyConnector |
14 | 15 | | from python_socks import _errors as proxy_errors |
| skipped 28 lines |
43 | 44 | | BAD_CHARS = "#" |
44 | 45 | | |
45 | 46 | | |
46 | | - | class SimpleAiohttpChecker: |
| 47 | + | class CheckerBase: |
| 48 | + | pass |
| 49 | + | |
| 50 | + | |
| 51 | + | class SimpleAiohttpChecker(CheckerBase): |
47 | 52 | | def __init__(self, *args, **kwargs): |
48 | 53 | | proxy = kwargs.get('proxy') |
49 | 54 | | cookie_jar = kwargs.get('cookie_jar') |
| skipped 85 lines |
135 | 140 | | ) |
136 | 141 | | |
137 | 142 | | |
| 143 | + | class AiodnsDomainResolver(CheckerBase): |
| 144 | + | def __init__(self, *args, **kwargs): |
| 145 | + | loop = asyncio.get_event_loop() |
| 146 | + | self.logger = kwargs.get('logger', Mock()) |
| 147 | + | self.resolver = aiodns.DNSResolver(loop=loop) |
| 148 | + | |
| 149 | + | def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'): |
| 150 | + | return self.resolver.query(url, 'A') |
| 151 | + | |
| 152 | + | async def check(self, future) -> Tuple[str, int, Optional[CheckError]]: |
| 153 | + | status = 404 |
| 154 | + | error = None |
| 155 | + | text = '' |
| 156 | + | |
| 157 | + | try: |
| 158 | + | res = await future |
| 159 | + | text = str(res[0].host) |
| 160 | + | status = 200 |
| 161 | + | except aiodns.error.DNSError: |
| 162 | + | pass |
| 163 | + | except Exception as e: |
| 164 | + | self.logger.error(e, exc_info=True) |
| 165 | + | error = CheckError('DNS resolve error', str(e)) |
| 166 | + | |
| 167 | + | return text, status, error |
| 168 | + | |
| 169 | + | |
| 170 | + | class CheckerMock: |
| 171 | + | def __init__(self, *args, **kwargs): |
| 172 | + | pass |
| 173 | + | |
| 174 | + | def prepare(self, url, headers=None, allow_redirects=True, timeout=0, method='get'): |
| 175 | + | return None |
| 176 | + | |
| 177 | + | async def check(self, future) -> Tuple[str, int, Optional[CheckError]]: |
| 178 | + | await asyncio.sleep(0) |
| 179 | + | return '', 0, None |
| 180 | + | |
| 181 | + | async def close(self): |
| 182 | + | return |
| 183 | + | |
| 184 | + | |
138 | 185 | | # TODO: move to separate class |
139 | 186 | | def detect_error_page( |
140 | 187 | | html_text, status_code, fail_flags, ignore_403 |
| skipped 229 lines |
370 | 417 | | url = re.sub("(?<!:)/+", "/", url) |
371 | 418 | | |
372 | 419 | | # always clearweb_checker for now |
373 | | - | checker = options["checkers"][site.network] |
| 420 | + | checker = options["checkers"][site.protocol] |
374 | 421 | | |
375 | 422 | | # site check is disabled |
376 | 423 | | if site.disabled and not options['forced']: |
| skipped 141 lines |
518 | 565 | | no_progressbar=False, |
519 | 566 | | cookies=None, |
520 | 567 | | retries=0, |
| 568 | + | check_domains=False, |
521 | 569 | | ) -> QueryResultWrapper: |
522 | 570 | | """Main search func |
523 | 571 | | |
| skipped 47 lines |
571 | 619 | | ) |
572 | 620 | | |
573 | 621 | | # TODO |
574 | | - | tor_checker = Mock() |
| 622 | + | tor_checker = CheckerMock() |
575 | 623 | | if tor_proxy: |
576 | 624 | | tor_checker = TorAiohttpChecker( # type: ignore |
577 | 625 | | proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger |
578 | 626 | | ) |
| 627 | + | |
| 628 | + | # TODO |
| 629 | + | dns_checker = CheckerMock() |
| 630 | + | if check_domains: |
| 631 | + | dns_checker = AiodnsDomainResolver(logger=logger) # type: ignore |
579 | 632 | | |
580 | 633 | | if logger.level == logging.DEBUG: |
581 | 634 | | await debug_ip_request(clearweb_checker, logger) |
| skipped 13 lines |
595 | 648 | | options["checkers"] = { |
596 | 649 | | '': clearweb_checker, |
597 | 650 | | 'tor': tor_checker, |
| 651 | + | 'dns': dns_checker, |
598 | 652 | | } |
599 | 653 | | options["parsing"] = is_parsing_enabled |
600 | 654 | | options["timeout"] = timeout |
| skipped 193 lines |