From e10c5e8fa73b32700b0b6eab163e01664b6d5ff1 Mon Sep 17 00:00:00 2001 From: SuHun Han Date: Sun, 14 Jun 2020 15:42:32 +0900 Subject: [PATCH] chore: v3.0.0 (#181) Co-authored-by: Kirill Bobrov Co-authored-by: Zhuopsticks <36221214+terryyz@users.noreply.github.com> --- .gitignore | 4 ++ .travis.yml | 1 - Pipfile | 13 +----- Pipfile.lock | 98 +++++++++++++++++++++++++++++++++++----- README.rst | 12 ++--- googletrans/__init__.py | 2 +- googletrans/adapters.py | 16 ------- googletrans/client.py | 63 ++++++++++++++++---------- googletrans/constants.py | 3 ++ googletrans/gtoken.py | 8 ++-- googletrans/utils.py | 2 +- setup.py | 6 +-- test-requirements.txt | 2 - tests/test_client.py | 34 +++++++------- tests/test_utils.py | 20 +++++++- tox.ini | 3 +- 16 files changed, 180 insertions(+), 107 deletions(-) delete mode 100644 googletrans/adapters.py diff --git a/.gitignore b/.gitignore index b1b5b23..1792885 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ target/ .DS_Store .python-version + +# IDE +.idea +.vscode diff --git a/.travis.yml b/.travis.yml index d731b3b..2dd70e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - 3.5 - 3.6 - 3.7 - 3.8 diff --git a/Pipfile b/Pipfile index fba95e3..2403c06 100644 --- a/Pipfile +++ b/Pipfile @@ -4,20 +4,9 @@ verify_ssl = true name = "pypi" [packages] -requests = "==2.13.0" - - - - - - +httpx = "==0.13.3" [dev-packages] - coveralls = "*" "pytest-watch" = "*" "pytest-testmon" = "*" - - -[requires] -python_version = "3.7" \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index d0ddcdc..e3e3d84 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,12 +1,10 @@ { "_meta": { "hash": { - "sha256": "bde08e25225ef9645715fdf4240e5d14e20533f2ba2522fe85bb9a1156eaa882" + "sha256": "e3f9ee3c7d118b3905fb3797a8b610e4f0485055c0809c93aba126596529673b" }, "pipfile-spec": 6, - "requires": { - "python_version": "3.7" - }, + "requires": {}, "sources": [ { "name": "pypi", @@ -16,13 +14,90 @@ ] }, "default": { - "requests": { + "certifi": { "hashes": [ - "sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb", - "sha256:5722cd09762faa01276230270ff16af7acf7c5c45d623868d9ba116f15791ce8" + "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1", + "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc" + ], + "version": "==2020.4.5.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "h11": { + "hashes": [ + "sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1", + "sha256:4bc6d6a1238b7615b266ada57e0618568066f57dd6fa967d1290ec9309b2f2f1" + ], + "version": "==0.9.0" + }, + "h2": { + "hashes": [ + "sha256:61e0f6601fa709f35cdb730863b4e5ec7ad449792add80d1410d4174ed139af5", + "sha256:875f41ebd6f2c44781259005b157faed1a5031df3ae5aa7bcb4628a6c0782f14" + ], + "version": "==3.2.0" + }, + "hpack": { + "hashes": [ + "sha256:0edd79eda27a53ba5be2dfabf3b15780928a0dff6eb0c60a3d6767720e970c89", + "sha256:8eec9c1f4bfae3408a3f30500261f7e6a65912dc138526ea054f9ad98892e9d2" + ], + "version": "==3.0.0" + }, + "hstspreload": { + "hashes": [ + "sha256:2858151b4f77322c6a61312abccead20217b1169ae0855753c0da45da2049329", + "sha256:9850f199a4678bbc6a392255948d2435b706bc8a7e40df0f84ac2f7348cf4195" + ], + "version": "==2020.6.5" + }, + "httpcore": { + "hashes": [ + "sha256:9850fe97a166a794d7e920590d5ec49a05488884c9fc8b5dba8561effab0c2a0", + "sha256:ecc5949310d9dae4de64648a4ce529f86df1f232ce23dcfefe737c24d21dfbe9" + ], + "version": "==0.9.1" + }, + "httpx": { + "hashes": [ + "sha256:32d930858eab677bc29a742aaa4f096de259f1c78c68a90ad11f5c3c04f08335", + "sha256:3642bd13e90b80ba8a243a730275eb10a4c26ec96f5fc16b87e458d4ab21efae" ], "index": "pypi", - "version": "==2.13.0" + "version": "==0.13.3" + }, + "hyperframe": { + "hashes": [ + "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40", + "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f" + ], + "version": "==5.2.0" + }, + "idna": { + "hashes": [ + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + ], + "version": "==2.9" + }, + "rfc3986": { + "hashes": [ + "sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d", + "sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50" + ], + "version": "==1.4.0" + }, + "sniffio": { + "hashes": [ + "sha256:20ed6d5b46f8ae136d00b9dcb807615d83ed82ceea6b2058cecb696765246da5", + "sha256:8e3810100f69fe0edd463d02ad407112542a11ffdc29f67db2bf3771afb87a21" + ], + "version": "==1.1.0" } }, "develop": { @@ -184,11 +259,10 @@ }, "requests": { "hashes": [ - "sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb", - "sha256:5722cd09762faa01276230270ff16af7acf7c5c45d623868d9ba116f15791ce8" + "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", + "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" ], - "index": "pypi", - "version": "==2.13.0" + "version": "==2.23.0" }, "six": { "hashes": [ diff --git a/README.rst b/README.rst index 93a0c75..9e44d15 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ implemented Google Translate API. This uses the `Google Translate Ajax API `__ to make calls to such methods as detect and translate. -Compatible with Python 3.5+. +Compatible with Python 3.6+. For details refer to the `API Documentation `__. @@ -22,7 +22,6 @@ Features - Auto language detection - Bulk translations - Customizable service URL -- Connection pooling (the advantage of using requests.Session) - HTTP/2 support TODO @@ -36,11 +35,7 @@ more features are coming soon. HTTP/2 support ~~~~~~~~~~~~~~ -This is a great deal for everyone! (up to 2x times faster in my test) If -you want to get googletrans faster you should install -`hyper `__ package. Googletrans will -automatically detect if hyper is installed and if so, it will be used -for http networking. +This library uses httpx for HTTP requests so HTTP/2 is supported by default. How does this library work ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -64,8 +59,7 @@ Installation To install, either use things like pip with the package "googletrans" or download the package and put the "googletrans" directory into your -python path. Anyway, it is noteworthy that, this just requires two -modules: requests and future. +python path. .. code:: bash diff --git a/googletrans/__init__.py b/googletrans/__init__.py index 0371da3..470e6e4 100644 --- a/googletrans/__init__.py +++ b/googletrans/__init__.py @@ -1,6 +1,6 @@ """Free Google Translate API for Python. Translates totally free of charge.""" __all__ = 'Translator', -__version__ = '2.4.1' +__version__ = '3.0.0' from googletrans.client import Translator diff --git a/googletrans/adapters.py b/googletrans/adapters.py deleted file mode 100644 index f04dc8e..0000000 --- a/googletrans/adapters.py +++ /dev/null @@ -1,16 +0,0 @@ -from requests.adapters import HTTPAdapter - - -class TimeoutAdapter(HTTPAdapter): - """HTTP adapter that adds timeout to each query.""" - def __init__(self, timeout=None, *args, **kwargs): - """HTTP adapter that adds timeout to each query. - - :param timeout: Timeout that will be added to each query - """ - self.timeout = timeout - super(TimeoutAdapter, self).__init__(*args, **kwargs) - - def send(self, *args, **kwargs): - kwargs['timeout'] = self.timeout - return super(TimeoutAdapter, self).send(*args, **kwargs) diff --git a/googletrans/client.py b/googletrans/client.py index 4e29c15..1ef1e24 100644 --- a/googletrans/client.py +++ b/googletrans/client.py @@ -4,16 +4,21 @@ A Translation module. You can translate text using this module. """ -import requests import random +import typing + +import httpcore +import httpx +from httpx import Timeout from googletrans import urls, utils -from googletrans.adapters import TimeoutAdapter from googletrans.gtoken import TokenAcquirer -from googletrans.constants import DEFAULT_USER_AGENT, LANGCODES, LANGUAGES, SPECIAL_CASES +from googletrans.constants import ( + DEFAULT_USER_AGENT, LANGCODES, LANGUAGES, SPECIAL_CASES, + DEFAULT_RAISE_EXCEPTION, DUMMY_DATA +) from googletrans.models import Translated, Detected - EXCLUDES = ('en', 'ca', 'fr') @@ -34,33 +39,35 @@ class Translator: For example ``{'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}`` :type proxies: dictionary - :param timeout: Definition of timeout for Requests library. - Will be used by every request. + :param timeout: Definition of timeout for httpx library. + Will be used for every request. :type timeout: number or a double of numbers +||||||| constructed merge base + :param proxies: proxies configuration. + Dictionary mapping protocol or protocol and host to the URL of the proxy + For example ``{'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}`` + :param raise_exception: if `True` then raise exception if smth will go wrong + :type raise_exception: boolean """ def __init__(self, service_urls=None, user_agent=DEFAULT_USER_AGENT, - proxies=None, timeout=None): + raise_exception=DEFAULT_RAISE_EXCEPTION, + proxies: typing.Dict[str, httpcore.SyncHTTPTransport] = None, timeout: Timeout = None): - self.session = requests.Session() - if proxies is not None: - self.session.proxies = proxies - self.session.headers.update({ + self.client = httpx.Client() + if proxies is not None: # pragma: nocover + self.client.proxies = proxies + + self.client.headers.update({ 'User-Agent': user_agent, }) + if timeout is not None: - self.session.mount('https://', TimeoutAdapter(timeout)) - self.session.mount('http://', TimeoutAdapter(timeout)) + self.client.timeout = timeout self.service_urls = service_urls or ['translate.google.com'] - self.token_acquirer = TokenAcquirer(session=self.session, host=self.service_urls[0]) - - # Use HTTP2 Adapter if hyper is installed - try: # pragma: nocover - from hyper.contrib import HTTP20Adapter - self.session.mount(urls.BASE, HTTP20Adapter()) - except ImportError: # pragma: nocover - pass + self.token_acquirer = TokenAcquirer(client=self.client, host=self.service_urls[0]) + self.raise_exception = raise_exception def _pick_service_url(self): if len(self.service_urls) == 1: @@ -73,10 +80,16 @@ class Translator: token=token, override=override) url = urls.TRANSLATE.format(host=self._pick_service_url()) - r = self.session.get(url, params=params) + r = self.client.get(url, params=params) - data = utils.format_json(r.text) - return data + if r.status_code == 200: + data = utils.format_json(r.text) + return data + else: + if self.raise_exception: + raise Exception('Unexpected status code "{}" from {}'.format(r.status_code, self.service_urls)) + DUMMY_DATA[0][0][0] = text + return DUMMY_DATA def _parse_extra_data(self, data): response_parts_name_mapping = { @@ -189,7 +202,7 @@ class Translator: if pron is None: try: pron = data[0][1][2] - except: # pragma: nocover + except: # pragma: nocover pass if dest in EXCLUDES and pron == origin: diff --git a/googletrans/constants.py b/googletrans/constants.py index 33b73eb..185e4b3 100644 --- a/googletrans/constants.py +++ b/googletrans/constants.py @@ -182,3 +182,6 @@ LANGUAGES = { } LANGCODES = dict(map(reversed, LANGUAGES.items())) +DEFAULT_RAISE_EXCEPTION = False +DUMMY_DATA = [[["", None, None, 0]], None, "en", None, + None, None, 1, None, [["en"], None, [1], ["en"]]] diff --git a/googletrans/gtoken.py b/googletrans/gtoken.py index a9558fe..83bace6 100644 --- a/googletrans/gtoken.py +++ b/googletrans/gtoken.py @@ -4,7 +4,7 @@ import math import re import time -import requests +import httpx from googletrans.utils import rshift @@ -38,8 +38,8 @@ class TokenAcquirer: RE_TKK = re.compile(r'tkk:\'(.+?)\'', re.DOTALL) RE_RAWTKK = re.compile(r'tkk:\'(.+?)\'', re.DOTALL) - def __init__(self, tkk='0', session=None, host='translate.google.com'): - self.session = session or requests.Session() + def __init__(self, tkk='0', client: httpx.Client = None, host='translate.google.com'): + self.client = client or httpx.Client() self.tkk = tkk self.host = host if 'http' in host else 'https://' + host @@ -51,7 +51,7 @@ class TokenAcquirer: if self.tkk and int(self.tkk.split('.')[0]) == now: return - r = self.session.get(self.host) + r = self.client.get(self.host) raw_tkk = self.RE_TKK.search(r.text) if raw_tkk: diff --git a/googletrans/utils.py b/googletrans/utils.py index 0c91727..6f2e9b1 100644 --- a/googletrans/utils.py +++ b/googletrans/utils.py @@ -1,6 +1,6 @@ """A conversion module for googletrans""" -import re import json +import re def build_params(query, src, dest, token, override): diff --git a/setup.py b/setup.py index 2c16385..c80e84e 100644 --- a/setup.py +++ b/setup.py @@ -52,18 +52,14 @@ def install(): 'Operating System :: MacOS :: MacOS X', 'Topic :: Education', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8'], packages=find_packages(exclude=['docs', 'tests']), keywords='google translate translator', install_requires=[ - 'requests', + 'httpx==0.13.3', ], - extras_require={ - 'h2': ['hyper'], - }, tests_require=[ 'pytest', 'coveralls', diff --git a/test-requirements.txt b/test-requirements.txt index 39139ee..357a40a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1 @@ -requests==2.13.0 -future==0.14.3 coveralls==1.1 diff --git a/tests/test_client.py b/tests/test_client.py index 92e2599..e1e7109 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- +from httpcore import TimeoutException +from httpcore._exceptions import ConnectError +from httpx import Timeout, Client +from unittest.mock import patch from pytest import raises -from requests.exceptions import ConnectionError -from requests.exceptions import ReadTimeout from googletrans import Translator @@ -127,20 +128,21 @@ def test_dest_not_in_supported_languages(translator): translator.translate(*args) -def test_connection_timeout(): - # Requests library specifies two timeouts: connection and read - - with raises((ConnectionError, ReadTimeout)): - """If a number is passed to timeout parameter, both connection - and read timeouts will be set to it. - Firstly, the connection timeout will fail. - """ - translator = Translator(timeout=0.00001) +def test_timeout(): + # httpx will raise ConnectError in some conditions + with raises((TimeoutException, ConnectError)): + translator = Translator(timeout=Timeout(0.0001)) translator.translate('안녕하세요.') -def test_read_timeout(): +class MockResponse: + def __init__(self, status_code): + self.status_code = status_code + self.text = 'tkk:\'translation\'' + + +@patch.object(Client, 'get', return_value=MockResponse('403')) +def test_403_error(session_mock): + translator = Translator() + assert translator.translate('test', dest='ko') - with raises(ReadTimeout): - translator = Translator(timeout=(10, 0.00001)) - translator.translate('안녕하세요.') diff --git a/tests/test_utils.py b/tests/test_utils.py index 7bbbf13..b965809 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,16 +5,18 @@ from pytest import raises def test_format_json(): text = '[,,"en",,,,0.96954316,,[["en"],,[0.96954316]]]' - result = utils.legacy_format_json(text) + result = utils.format_json(text) assert result == [None, None, 'en', None, None, None, 0.96954316, None, [['en'], None, [0.96954316]]] + def test_format_malformed_json(): text = '[,,"en",,,,0.96954316,,[["en"],,0.96954316]]]' with raises(ValueError): - utils.legacy_format_json(text) + utils.format_json(text) + def test_rshift(): value, n = 1000, 3 @@ -22,3 +24,17 @@ def test_rshift(): result = utils.rshift(value, n) assert result == 125 + + +def test_build_params_with_override(): + params = utils.build_params( + query='', + src='', + dest='', + token='', + override={ + 'otf': '3', + }, + ) + + assert params['otf'] == '3' diff --git a/tox.ini b/tox.ini index 2a84f20..e5700e0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,11 @@ [tox] -envlist = py35,py36,py37,py38,pypy3 +envlist = py36,py37,py38,pypy3 [testenv] deps= pytest pytest-cov + mock commands= py.test --cov-report= --cov={envsitepackagesdir}/googletrans {posargs:}