PCQRSCANER/venv/Lib/site-packages/fake_useragent/utils.py

# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import contextlib
import inspect
import io
import json
import os
import re
import ssl

from fake_useragent.log import logger

try:  # Python 2 # pragma: no cover
    from urllib2 import urlopen, Request, URLError
    from urllib import quote_plus

    str_types = (unicode, str)  # noqa
    text = unicode  # noqa
except ImportError:  # Python 3 # pragma: no cover
    from urllib.request import urlopen, Request
    from urllib.parse import quote_plus
    from urllib.error import URLError

    str_types = (str,)
    text = str

# gevent monkey patched environment check
try:  # pragma: no cover
    import socket
    import gevent.socket

    if socket.socket is gevent.socket.socket:
        from gevent import sleep
    else:
        from time import sleep
except (ImportError, AttributeError):  # pragma: no cover
    from time import sleep


try:
    urlopen_args = inspect.getfullargspec(urlopen).kwonlyargs
except AttributeError:
    urlopen_args = inspect.getargspec(urlopen).args

urlopen_has_ssl_context = 'context' in urlopen_args


def get(url, verify_ssl=True):
    attempt = 0

    while True:
        request = Request(url)

        attempt += 1

        try:
            if urlopen_has_ssl_context:
                if not verify_ssl:
                    context = ssl._create_unverified_context()
                else:
                    context = None

                with contextlib.closing(urlopen(
                    request,
                    timeout=settings.HTTP_TIMEOUT,
                    context=context,
                )) as response:
                    return response.read()
            else:  # ssl context is not supported ;(
                with contextlib.closing(urlopen(
                    request,
                    timeout=settings.HTTP_TIMEOUT,
                )) as response:
                    return response.read()
        except (URLError, OSError) as exc:
            logger.debug(
                'Error occurred during fetching %s',
                url,
                exc_info=exc,
            )

            if attempt == settings.HTTP_RETRIES:
                raise FakeUserAgentError('Maximum amount of retries reached')
            else:
                logger.debug(
                    'Sleeping for %s seconds',
                    settings.HTTP_DELAY,
                )
                sleep(settings.HTTP_DELAY)


def get_browsers(verify_ssl=True):
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(settings.BROWSERS_STATS_PAGE, verify_ssl=verify_ssl)
    html = html.decode('utf-8')
    html = html.split('<table class="w3-table-all notranslate">')[1]
    html = html.split('</table>')[0]

    pattern = r'\.asp">(.+?)<'
    browsers = re.findall(pattern, html, re.UNICODE)

    browsers = [
        settings.OVERRIDES.get(browser, browser)
        for browser in browsers
    ]

    pattern = r'td\sclass="right">(.+?)\s'
    browsers_statistics = re.findall(pattern, html, re.UNICODE)

    return list(zip(browsers, browsers_statistics))


def get_browser_versions(browser, verify_ssl=True):
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(
        settings.BROWSER_BASE_PAGE.format(browser=quote_plus(browser)),
        verify_ssl=verify_ssl,
    )
    html = html.decode('iso-8859-1')
    html = html.split('<div id=\'liste\'>')[1]
    html = html.split('</div>')[0]

    pattern = r'\?id=\d+\'>(.+?)</a'
    browsers_iter = re.finditer(pattern, html, re.UNICODE)

    browsers = []

    for browser in browsers_iter:
        if 'more' in browser.group(1).lower():
            continue

        browsers.append(browser.group(1))

        if len(browsers) == settings.BROWSERS_COUNT_LIMIT:
            break

    if not browsers:
        raise FakeUserAgentError(
            'No browsers version found for {browser}'.format(browser=browser))

    return browsers


def load(use_cache_server=True, verify_ssl=True):
    browsers_dict = {}
    randomize_dict = {}

    try:
        for item in get_browsers(verify_ssl=verify_ssl):
            browser, percent = item

            browser_key = browser

            for value, replacement in settings.REPLACEMENTS.items():
                browser_key = browser_key.replace(value, replacement)

            browser_key = browser_key.lower()

            browsers_dict[browser_key] = get_browser_versions(
                browser,
                verify_ssl=verify_ssl,
            )

            # it is actually so bad way for randomizing, simple list with
            # browser_key's is event better
            # I've failed so much a lot of years ago.
            # Ideas for refactoring
            # {'chrome': <percantage|int>, 'firefox': '<percatage|int>'}
            for _ in range(int(float(percent) * 10)):
                randomize_dict[str(len(randomize_dict))] = browser_key
    except Exception as exc:
        if not use_cache_server:
            raise exc

        logger.warning(
            'Error occurred during loading data. '
            'Trying to use cache server %s',
            settings.CACHE_SERVER,
            exc_info=exc,
        )
        try:
            ret = json.loads(get(
                settings.CACHE_SERVER,
                verify_ssl=verify_ssl,
            ).decode('utf-8'))
        except (TypeError, ValueError):
            raise FakeUserAgentError('Can not load data from cache server')
    else:
        ret = {
            'browsers': browsers_dict,
            'randomize': randomize_dict,
        }

    if not isinstance(ret, dict):
        raise FakeUserAgentError('Data is not dictionary ', ret)

    for param in ['browsers', 'randomize']:
        if param not in ret:
            raise FakeUserAgentError('Missing data param: ', param)

        if not isinstance(ret[param], dict):
            raise FakeUserAgentError('Data param is not dictionary', ret[param])  # noqa

        if not ret[param]:
            raise FakeUserAgentError('Data param is empty', ret[param])

    return ret


# TODO: drop these useless functions


def write(path, data):
    with io.open(path, encoding='utf-8', mode='wt') as fp:
        dumped = json.dumps(data)

        if not isinstance(dumped, text):  # Python 2
            dumped = dumped.decode('utf-8')

        fp.write(dumped)


def read(path):
    with io.open(path, encoding='utf-8', mode='rt') as fp:
        return json.loads(fp.read())


def exist(path):
    return os.path.isfile(path)


def rm(path):
    if exist(path):
        os.remove(path)


def update(path, use_cache_server=True, verify_ssl=True):
    rm(path)

    write(path, load(use_cache_server=use_cache_server, verify_ssl=verify_ssl))


def load_cached(path, use_cache_server=True, verify_ssl=True):
    if not exist(path):
        update(path, use_cache_server=use_cache_server, verify_ssl=verify_ssl)

    return read(path)


from fake_useragent import settings  # noqa # isort:skip
from fake_useragent.errors import FakeUserAgentError  # noqa # isort:skip