From c5eb00d4151e204cab32a3026583326745054419 Mon Sep 17 00:00:00 2001 From: Stefan Tjarks Date: Tue, 17 Jul 2018 13:53:05 -0700 Subject: [PATCH] Drop support for Python 3.4 to support aiohttp>3.3 aiohttp dropped support for Python 3.4 and with it `asyncio.coroutine` decorated generators. To support latest aiohttp Python 3.4 support is dropped and all notion of `asyncio.coroutine` is replaced with `async def`. Signed-off-by: Randy Barlow --- .coveragerc-py34 => .coveragerc-py35 | 0 .travis.yml | 1 - Makefile | 10 +- README.rst | 17 +- backoff/_async.py | 33 ++-- .../test_backoff_async.py | 167 +++++++----------- 6 files changed, 84 insertions(+), 144 deletions(-) rename .coveragerc-py34 => .coveragerc-py35 (100%) rename tests/{python34 => python35}/test_backoff_async.py (82%) diff --git a/.coveragerc-py34 b/.coveragerc-py35 similarity index 100% rename from .coveragerc-py34 rename to .coveragerc-py35 diff --git a/.travis.yml b/.travis.yml index 1c0c7fd..b5cb8de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.6" - "2.7" - - "3.4" - "3.5" - "3.6" matrix: diff --git a/Makefile b/Makefile index 66526bd..eb5f188 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PY_VERSION := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1))) PY_MAJOR := $(word 1,${PY_VERSION}) PY_MINOR := $(word 2,${PY_VERSION}) -PY_GTE_34 = $(shell echo $(PY_MAJOR).$(PY_MINOR)\>=3.4 | bc) +PY_GTE_35 = $(shell echo $(PY_MAJOR).$(PY_MINOR)\>=3.5 | bc) PY_GTE_27 = $(shell echo $(PY_MAJOR).$(PY_MINOR)\>=2.7 | bc) @@ -18,10 +18,10 @@ pep8: @pep8 backoff tests flake8: -ifeq ($(PY_GTE_34),1) +ifeq ($(PY_GTE_35),1) @flake8 backoff tests else ifeq ($(PY_GTE_27),1) - @flake8 --exclude tests/python34,backoff/_async.py backoff tests + @flake8 --exclude tests/python35,backoff/_async.py backoff tests else @echo 'Not running flake8 for Python < 2.7' endif @@ -32,8 +32,8 @@ clean: @rm -rf build dist .coverage MANIFEST test: clean -ifeq ($(PY_GTE_34),1) - @PYTHONPATH=. py.test --cov-config .coveragerc-py34 --cov backoff tests +ifeq ($(PY_GTE_35),1) + @PYTHONPATH=. py.test --cov-config .coveragerc-py35 --cov backoff tests else @PYTHONPATH=. py.test --cov-config .coveragerc-py2 --cov backoff tests/test_*.py endif diff --git a/README.rst b/README.rst index ee535e7..fee5723 100644 --- a/README.rst +++ b/README.rst @@ -284,22 +284,7 @@ On Python 3.5 and above with ``async def`` and ``await`` syntax: async with session.get(url) as response: return await response.text() -In case you use Python 3.4 you can use `@asyncio.coroutine` and `yield from`: - -.. code-block:: python - - @backoff.on_exception(backoff.expo, aiohttp.ClientError, max_time=60) - @asyncio.coroutine - def get_url_py34(url): - with aiohttp.ClientSession() as session: - response = yield from session.get(url) - try: - return (yield from response.text()) - except Exception: - response.close() - raise - finally: - yield from response.release() +Python 3.4 is not supported. Logging configuration --------------------- diff --git a/backoff/_async.py b/backoff/_async.py index e1b70d0..2cd33a8 100644 --- a/backoff/_async.py +++ b/backoff/_async.py @@ -20,8 +20,7 @@ def _ensure_coroutines(coros_or_funcs): return [_ensure_coroutine(f) for f in coros_or_funcs] -@asyncio.coroutine -def _call_handlers(hdlrs, target, args, kwargs, tries, elapsed, **extra): +async def _call_handlers(hdlrs, target, args, kwargs, tries, elapsed, **extra): details = { 'target': target, 'args': args, @@ -31,7 +30,7 @@ def _call_handlers(hdlrs, target, args, kwargs, tries, elapsed, **extra): } details.update(extra) for hdlr in hdlrs: - yield from hdlr(details) + await hdlr(details) def retry_predicate(target, wait_gen, predicate, @@ -49,8 +48,7 @@ def retry_predicate(target, wait_gen, predicate, assert asyncio.iscoroutinefunction(target) @functools.wraps(target) - @asyncio.coroutine - def retry(*args, **kwargs): + async def retry(*args, **kwargs): # change names because python 2.x doesn't have nonlocal max_tries_ = _maybe_call(max_tries) @@ -64,20 +62,20 @@ def retry_predicate(target, wait_gen, predicate, elapsed = _total_seconds(datetime.datetime.now() - start) details = (target, args, kwargs, tries, elapsed) - ret = yield from target(*args, **kwargs) + ret = await target(*args, **kwargs) if predicate(ret): max_tries_exceeded = (tries == max_tries_) max_time_exceeded = (max_time_ is not None and elapsed >= max_time_) if max_tries_exceeded or max_time_exceeded: - yield from _call_handlers( + await _call_handlers( giveup_hdlrs, *details, value=ret) break seconds = _next_wait(wait, jitter, elapsed, max_time_) - yield from _call_handlers( + await _call_handlers( backoff_hdlrs, *details, value=ret, wait=seconds) # Note: there is no convenient way to pass explicit event @@ -89,10 +87,10 @@ def retry_predicate(target, wait_gen, predicate, # See for details: # # - yield from asyncio.sleep(seconds) + await asyncio.sleep(seconds) continue else: - yield from _call_handlers(success_hdlrs, *details, value=ret) + await _call_handlers(success_hdlrs, *details, value=ret) break return ret @@ -114,8 +112,7 @@ def retry_exception(target, wait_gen, exception, assert not asyncio.iscoroutinefunction(jitter) @functools.wraps(target) - @asyncio.coroutine - def retry(*args, **kwargs): + async def retry(*args, **kwargs): # change names because python 2.x doesn't have nonlocal max_tries_ = _maybe_call(max_tries) max_time_ = _maybe_call(max_time) @@ -129,20 +126,20 @@ def retry_exception(target, wait_gen, exception, details = (target, args, kwargs, tries, elapsed) try: - ret = yield from target(*args, **kwargs) + ret = await target(*args, **kwargs) except exception as e: - giveup_result = yield from giveup(e) + giveup_result = await giveup(e) max_tries_exceeded = (tries == max_tries_) max_time_exceeded = (max_time_ is not None and elapsed >= max_time_) if giveup_result or max_tries_exceeded or max_time_exceeded: - yield from _call_handlers(giveup_hdlrs, *details) + await _call_handlers(giveup_hdlrs, *details) raise seconds = _next_wait(wait, jitter, elapsed, max_time_) - yield from _call_handlers( + await _call_handlers( backoff_hdlrs, *details, wait=seconds) # Note: there is no convenient way to pass explicit event @@ -154,9 +151,9 @@ def retry_exception(target, wait_gen, exception, # See for details: # # - yield from asyncio.sleep(seconds) + await asyncio.sleep(seconds) else: - yield from _call_handlers(success_hdlrs, *details) + await _call_handlers(success_hdlrs, *details) return ret return retry diff --git a/tests/python34/test_backoff_async.py b/tests/python35/test_backoff_async.py similarity index 82% rename from tests/python34/test_backoff_async.py rename to tests/python35/test_backoff_async.py index 0917c6b..c1a30f8 100644 --- a/tests/python34/test_backoff_async.py +++ b/tests/python35/test_backoff_async.py @@ -8,47 +8,41 @@ import random from tests.common import _log_hdlrs, _save_target -@pytest.mark.asyncio -def test_on_predicate(monkeypatch): +async def test_on_predicate(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) @backoff.on_predicate(backoff.expo) - @asyncio.coroutine - def return_true(log, n): + async def return_true(log, n): val = (len(log) == n - 1) log.append(val) return val log = [] - ret = yield from return_true(log, 3) + ret = await return_true(log, 3) assert ret is True assert 3 == len(log) -@pytest.mark.asyncio -def test_on_predicate_max_tries(monkeypatch): +async def test_on_predicate_max_tries(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3) - @asyncio.coroutine - def return_true(log, n): + async def return_true(log, n): val = (len(log) == n) log.append(val) return val log = [] - ret = yield from return_true(log, 10) + ret = await return_true(log, 10) assert ret is False assert 3 == len(log) -@pytest.mark.asyncio -def test_on_exception(monkeypatch): +async def test_on_exception(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) @backoff.on_exception(backoff.expo, KeyError) - @asyncio.coroutine - def keyerror_then_true(log, n): + async def keyerror_then_true(log, n): if len(log) == n: return True e = KeyError() @@ -56,17 +50,15 @@ def test_on_exception(monkeypatch): raise e log = [] - assert (yield from keyerror_then_true(log, 3)) is True + assert (await keyerror_then_true(log, 3)) is True assert 3 == len(log) -@pytest.mark.asyncio -def test_on_exception_tuple(monkeypatch): +async def test_on_exception_tuple(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) @backoff.on_exception(backoff.expo, (KeyError, ValueError)) - @asyncio.coroutine - def keyerror_valueerror_then_true(log): + async def keyerror_valueerror_then_true(log): if len(log) == 2: return True if len(log) == 0: @@ -77,19 +69,17 @@ def test_on_exception_tuple(monkeypatch): raise e log = [] - assert (yield from keyerror_valueerror_then_true(log)) is True + assert (await keyerror_valueerror_then_true(log)) is True assert 2 == len(log) assert isinstance(log[0], KeyError) assert isinstance(log[1], ValueError) -@pytest.mark.asyncio -def test_on_exception_max_tries(monkeypatch): +async def test_on_exception_max_tries(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3) - @asyncio.coroutine - def keyerror_then_true(log, n, foo=None): + async def keyerror_then_true(log, n, foo=None): if len(log) == n: return True e = KeyError() @@ -98,13 +88,12 @@ def test_on_exception_max_tries(monkeypatch): log = [] with pytest.raises(KeyError): - yield from keyerror_then_true(log, 10, foo="bar") + await keyerror_then_true(log, 10, foo="bar") assert 3 == len(log) -@pytest.mark.asyncio -def test_on_exception_success_random_jitter(monkeypatch): +async def test_on_exception_success_random_jitter(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) log, log_success, log_backoff, log_giveup = _log_hdlrs() @@ -117,13 +106,12 @@ def test_on_exception_success_random_jitter(monkeypatch): jitter=backoff.random_jitter, factor=0.5) @_save_target - @asyncio.coroutine - def succeeder(*args, **kwargs): + async def succeeder(*args, **kwargs): # succeed after we've backed off twice if len(log['backoff']) < 2: raise ValueError("catch me") - yield from succeeder(1, 2, 3, foo=1, bar=2) + await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -135,8 +123,7 @@ def test_on_exception_success_random_jitter(monkeypatch): assert details['wait'] >= 0.5 * 2 ** i -@pytest.mark.asyncio -def test_on_exception_success_full_jitter(monkeypatch): +async def test_on_exception_success_full_jitter(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) log, log_success, log_backoff, log_giveup = _log_hdlrs() @@ -149,13 +136,12 @@ def test_on_exception_success_full_jitter(monkeypatch): jitter=backoff.full_jitter, factor=0.5) @_save_target - @asyncio.coroutine - def succeeder(*args, **kwargs): + async def succeeder(*args, **kwargs): # succeed after we've backed off twice if len(log['backoff']) < 2: raise ValueError("catch me") - yield from succeeder(1, 2, 3, foo=1, bar=2) + await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -167,8 +153,7 @@ def test_on_exception_success_full_jitter(monkeypatch): assert details['wait'] <= 0.5 * 2 ** i -@pytest.mark.asyncio -def test_on_exception_success(): +async def test_on_exception_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_exception(backoff.constant, @@ -179,13 +164,12 @@ def test_on_exception_success(): jitter=lambda: 0, interval=0) @_save_target - @asyncio.coroutine def succeeder(*args, **kwargs): # succeed after we've backed off twice if len(log['backoff']) < 2: raise ValueError("catch me") - yield from succeeder(1, 2, 3, foo=1, bar=2) + await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -211,8 +195,7 @@ def test_on_exception_success(): 'tries': 3} -@pytest.mark.asyncio -def test_on_exception_giveup(): +async def test_on_exception_giveup(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_exception(backoff.constant, @@ -224,12 +207,11 @@ def test_on_exception_giveup(): jitter=lambda: 0, interval=0) @_save_target - @asyncio.coroutine - def exceptor(*args, **kwargs): + async def exceptor(*args, **kwargs): raise ValueError("catch me") with pytest.raises(ValueError): - yield from exceptor(1, 2, 3, foo=1, bar=2) + await exceptor(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice and giving up once assert len(log['success']) == 0 @@ -245,8 +227,7 @@ def test_on_exception_giveup(): 'tries': 3} -@pytest.mark.asyncio -def test_on_exception_giveup_predicate(monkeypatch): +async def test_on_exception_giveup_predicate(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) def on_baz(e): @@ -257,22 +238,19 @@ def test_on_exception_giveup_predicate(monkeypatch): @backoff.on_exception(backoff.constant, ValueError, giveup=on_baz) - @asyncio.coroutine - def foo_bar_baz(): + async def foo_bar_baz(): raise ValueError(vals.pop()) with pytest.raises(ValueError): - yield from foo_bar_baz() + await foo_bar_baz() assert not vals -@pytest.mark.asyncio -def test_on_exception_giveup_coro(monkeypatch): +async def test_on_exception_giveup_coro(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) - @asyncio.coroutine - def on_baz(e): + async def on_baz(e): return str(e) == "baz" vals = ["baz", "bar", "foo"] @@ -280,18 +258,16 @@ def test_on_exception_giveup_coro(monkeypatch): @backoff.on_exception(backoff.constant, ValueError, giveup=on_baz) - @asyncio.coroutine - def foo_bar_baz(): + async def foo_bar_baz(): raise ValueError(vals.pop()) with pytest.raises(ValueError): - yield from foo_bar_baz() + await foo_bar_baz() assert not vals -@pytest.mark.asyncio -def test_on_predicate_success(): +async def test_on_predicate_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_predicate(backoff.constant, @@ -301,12 +277,11 @@ def test_on_predicate_success(): jitter=lambda: 0, interval=0) @_save_target - @asyncio.coroutine - def success(*args, **kwargs): + async def success(*args, **kwargs): # succeed after we've backed off twice return len(log['backoff']) == 2 - yield from success(1, 2, 3, foo=1, bar=2) + await success(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -334,8 +309,7 @@ def test_on_predicate_success(): 'value': True} -@pytest.mark.asyncio -def test_on_predicate_giveup(): +async def test_on_predicate_giveup(): log, log_success, log_backoff, log_giveup = _log_hdlrs() @backoff.on_predicate(backoff.constant, @@ -346,11 +320,10 @@ def test_on_predicate_giveup(): jitter=lambda: 0, interval=0) @_save_target - @asyncio.coroutine - def emptiness(*args, **kwargs): + async def emptiness(*args, **kwargs): pass - yield from emptiness(1, 2, 3, foo=1, bar=2) + await emptiness(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice and giving up once assert len(log['success']) == 0 @@ -367,8 +340,7 @@ def test_on_predicate_giveup(): 'value': None} -@pytest.mark.asyncio -def test_on_predicate_iterable_handlers(): +async def test_on_predicate_iterable_handlers(): hdlrs = [_log_hdlrs() for _ in range(3)] @backoff.on_predicate(backoff.constant, @@ -379,11 +351,10 @@ def test_on_predicate_iterable_handlers(): jitter=lambda: 0, interval=0) @_save_target - @asyncio.coroutine - def emptiness(*args, **kwargs): + async def emptiness(*args, **kwargs): pass - yield from emptiness(1, 2, 3, foo=1, bar=2) + await emptiness(1, 2, 3, foo=1, bar=2) for i in range(3): assert len(hdlrs[i][0]['success']) == 0 @@ -402,8 +373,7 @@ def test_on_predicate_iterable_handlers(): # To maintain backward compatibility, # on_predicate should support 0-argument jitter function. -@pytest.mark.asyncio -def test_on_exception_success_0_arg_jitter(monkeypatch): +async def test_on_exception_success_0_arg_jitter(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) monkeypatch.setattr('random.random', lambda: 0) @@ -417,13 +387,12 @@ def test_on_exception_success_0_arg_jitter(monkeypatch): jitter=random.random, interval=0) @_save_target - @asyncio.coroutine - def succeeder(*args, **kwargs): + async def succeeder(*args, **kwargs): # succeed after we've backed off twice if len(log['backoff']) < 2: raise ValueError("catch me") - yield from succeeder(1, 2, 3, foo=1, bar=2) + await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -451,8 +420,7 @@ def test_on_exception_success_0_arg_jitter(monkeypatch): # To maintain backward compatibility, # on_predicate should support 0-argument jitter function. -@pytest.mark.asyncio -def test_on_predicate_success_0_arg_jitter(monkeypatch): +async def test_on_predicate_success_0_arg_jitter(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) monkeypatch.setattr('random.random', lambda: 0) @@ -465,12 +433,11 @@ def test_on_predicate_success_0_arg_jitter(monkeypatch): jitter=random.random, interval=0) @_save_target - @asyncio.coroutine - def success(*args, **kwargs): + async def success(*args, **kwargs): # succeed after we've backed off twice return len(log['backoff']) == 2 - yield from success(1, 2, 3, foo=1, bar=2) + await success(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding assert len(log['success']) == 1 @@ -498,8 +465,7 @@ def test_on_predicate_success_0_arg_jitter(monkeypatch): 'value': True} -@pytest.mark.asyncio -def test_on_exception_callable_max_tries(monkeypatch): +async def test_on_exception_callable_max_tries(monkeypatch): monkeypatch.setattr('asyncio.sleep', asyncio.coroutine(lambda x: None)) def lookup_max_tries(): @@ -510,19 +476,17 @@ def test_on_exception_callable_max_tries(monkeypatch): @backoff.on_exception(backoff.constant, ValueError, max_tries=lookup_max_tries) - @asyncio.coroutine - def exceptor(): + async def exceptor(): log.append(True) raise ValueError() with pytest.raises(ValueError): - yield from exceptor() + await exceptor() assert len(log) == 3 -@pytest.mark.asyncio -def test_on_exception_callable_gen_kwargs(): +async def test_on_exception_callable_gen_kwargs(): def lookup_foo(): return "foo" @@ -539,25 +503,22 @@ def test_on_exception_callable_gen_kwargs(): max_tries=2, foo=lookup_foo, bar="bar") - @asyncio.coroutine - def exceptor(): + async def exceptor(): raise ValueError("aah") with pytest.raises(ValueError): - yield from exceptor() + await exceptor() -@pytest.mark.asyncio -def test_on_exception_coro_cancelling(event_loop): +async def test_on_exception_coro_cancelling(event_loop): sleep_started_event = asyncio.Event() @backoff.on_predicate(backoff.expo) - @asyncio.coroutine - def coro(): + async def coro(): sleep_started_event.set() try: - yield from asyncio.sleep(10) + await asyncio.sleep(10) except asyncio.CancelledError: return True @@ -565,17 +526,16 @@ def test_on_exception_coro_cancelling(event_loop): task = event_loop.create_task(coro()) - yield from sleep_started_event.wait() + await sleep_started_event.wait() task.cancel() - assert (yield from task) + assert (await task) -@pytest.mark.asyncio -def test_on_exception_on_regular_function(): +async def test_on_exception_on_regular_function(): # Force this function to be a running coroutine. - yield from asyncio.sleep(0) + await asyncio.sleep(0) with pytest.raises(TypeError) as excinfo: @backoff.on_exception(backoff.expo, ValueError) @@ -584,10 +544,9 @@ def test_on_exception_on_regular_function(): assert "applied to a regular function" in str(excinfo.value) -@pytest.mark.asyncio -def test_on_predicate_on_regular_function(): +async def test_on_predicate_on_regular_function(): # Force this function to be a running coroutine. - yield from asyncio.sleep(0) + await asyncio.sleep(0) with pytest.raises(TypeError) as excinfo: @backoff.on_predicate(backoff.expo) -- 2.18.0