| |
@@ -0,0 +1,188 @@
|
| |
+ From ac974739a53b6e3fc7a0b25d5d1c01cba7c2f6e2 Mon Sep 17 00:00:00 2001
|
| |
+ From: Lukas Slebodnik <lslebodn@redhat.com>
|
| |
+ Date: Tue, 5 Feb 2019 10:42:08 +0100
|
| |
+ Subject: [PATCH] Utils: Add decorator for retry
|
| |
+
|
| |
+ It might be useful in checks which do lot of network communication.
|
| |
+ ---
|
| |
+ colin/utils/cmd_tools.py | 29 +++++++++++
|
| |
+ tests/unit/test_utils.py | 103 +++++++++++++++++++++++++++++++++++++--
|
| |
+ 2 files changed, 129 insertions(+), 3 deletions(-)
|
| |
+
|
| |
+ diff --git a/colin/utils/cmd_tools.py b/colin/utils/cmd_tools.py
|
| |
+ index 2da4118..f3f08a8 100644
|
| |
+ --- a/colin/utils/cmd_tools.py
|
| |
+ +++ b/colin/utils/cmd_tools.py
|
| |
+ @@ -13,9 +13,11 @@
|
| |
+ # You should have received a copy of the GNU General Public License
|
| |
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
|
| |
+ #
|
| |
+ +import functools
|
| |
+ import logging
|
| |
+ import subprocess
|
| |
+ import threading
|
| |
+ +import time
|
| |
+
|
| |
+ try:
|
| |
+ import thread
|
| |
+ @@ -120,3 +122,30 @@ def inner(*args, **kwargs):
|
| |
+ return inner
|
| |
+
|
| |
+ return outer
|
| |
+ +
|
| |
+ +
|
| |
+ +def retry(retry_count=5, delay=2):
|
| |
+ + """
|
| |
+ + Use as decorator to retry functions few times with delays
|
| |
+ +
|
| |
+ + Exception will be raised if last call fails
|
| |
+ +
|
| |
+ + :param retry_count: int could of retries in case of failures. It must be
|
| |
+ + a positive number
|
| |
+ + :param delay: int delay between retries
|
| |
+ + """
|
| |
+ + if retry_count <= 0:
|
| |
+ + raise ValueError("retry_count have to be positive")
|
| |
+ +
|
| |
+ + def decorator(f):
|
| |
+ + @functools.wraps(f)
|
| |
+ + def wrapper(*args, **kwargs):
|
| |
+ + for i in range(retry_count, 0, -1):
|
| |
+ + try:
|
| |
+ + return f(*args, **kwargs)
|
| |
+ + except Exception:
|
| |
+ + if i <= 1:
|
| |
+ + raise
|
| |
+ + time.sleep(delay)
|
| |
+ + return wrapper
|
| |
+ + return decorator
|
| |
+ diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
|
| |
+ index 0d23182..4185a56 100644
|
| |
+ --- a/tests/unit/test_utils.py
|
| |
+ +++ b/tests/unit/test_utils.py
|
| |
+ @@ -13,12 +13,13 @@
|
| |
+ # You should have received a copy of the GNU General Public License
|
| |
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
|
| |
+ #
|
| |
+ -from time import sleep
|
| |
+ +import time
|
| |
+
|
| |
+ import pytest
|
| |
+
|
| |
+ from colin.core.exceptions import ColinException
|
| |
+ from colin.utils.cmd_tools import exit_after
|
| |
+ +from colin.utils.cmd_tools import retry
|
| |
+
|
| |
+
|
| |
+ @exit_after(1)
|
| |
+ @@ -28,7 +29,7 @@ def fast_fce():
|
| |
+
|
| |
+ @exit_after(1)
|
| |
+ def slow_fce():
|
| |
+ - sleep(2)
|
| |
+ + time.sleep(2)
|
| |
+
|
| |
+
|
| |
+ @exit_after(1)
|
| |
+ @@ -52,4 +53,100 @@ def test_timeout_bad_fce():
|
| |
+
|
| |
+ def test_timeout_dirrect():
|
| |
+ with pytest.raises(TimeoutError):
|
| |
+ - exit_after(1)(sleep)(2)
|
| |
+ + exit_after(1)(time.sleep)(2)
|
| |
+ +
|
| |
+ +
|
| |
+ +COUNTER = 0
|
| |
+ +
|
| |
+ +
|
| |
+ +def raise_exception():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER += 1
|
| |
+ +
|
| |
+ + raise Exception('I am bad function')
|
| |
+ +
|
| |
+ +
|
| |
+ +def test_no_retry_for_success():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER = 0
|
| |
+ +
|
| |
+ + @retry(5, 0)
|
| |
+ + def always_success():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER = COUNTER + 1
|
| |
+ +
|
| |
+ + return 42
|
| |
+ +
|
| |
+ + assert always_success() == 42
|
| |
+ + assert COUNTER == 1
|
| |
+ +
|
| |
+ +
|
| |
+ +def test_retry_with_exception():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER = 0
|
| |
+ +
|
| |
+ + @retry(5, 0)
|
| |
+ + def always_raise_exception():
|
| |
+ + raise_exception()
|
| |
+ +
|
| |
+ + with pytest.raises(Exception) as ex:
|
| |
+ + always_raise_exception()
|
| |
+ +
|
| |
+ + assert str(ex.value) == 'I am bad function'
|
| |
+ + assert COUNTER == 5
|
| |
+ +
|
| |
+ +
|
| |
+ +def test_wrong_parameter():
|
| |
+ + with pytest.raises(ValueError) as ex:
|
| |
+ + retry(-1, 1)
|
| |
+ + assert str(ex.value) == 'retry_count have to be positive'
|
| |
+ +
|
| |
+ + with pytest.raises(ValueError) as ex:
|
| |
+ + retry(0, 1)
|
| |
+ + assert str(ex.value) == 'retry_count have to be positive'
|
| |
+ +
|
| |
+ + @retry(5, -1)
|
| |
+ + def fail_negative_sleep():
|
| |
+ + raise_exception()
|
| |
+ +
|
| |
+ + with pytest.raises(ValueError) as ex:
|
| |
+ + fail_negative_sleep()
|
| |
+ + assert str(ex.value) == 'sleep length must be non-negative'
|
| |
+ +
|
| |
+ +
|
| |
+ +def test_retry_with_sleep():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER = 0
|
| |
+ +
|
| |
+ + @retry(4, .5)
|
| |
+ + def fail_and_sleep():
|
| |
+ + raise_exception()
|
| |
+ +
|
| |
+ + time_start = time.time()
|
| |
+ + with pytest.raises(Exception) as ex:
|
| |
+ + fail_and_sleep()
|
| |
+ + time_end = time.time()
|
| |
+ +
|
| |
+ + assert str(ex.value) == 'I am bad function'
|
| |
+ + assert COUNTER == 4
|
| |
+ +
|
| |
+ + # there are 3 sleeps between 4 delays
|
| |
+ + assert time_end - time_start >= 1.5
|
| |
+ + # there were not 4 sleeps
|
| |
+ + assert time_end - time_start < 4
|
| |
+ +
|
| |
+ +
|
| |
+ +def test_recover_after_few_failures():
|
| |
+ + global COUNTER
|
| |
+ + COUNTER = 0
|
| |
+ +
|
| |
+ + @retry(5, 0)
|
| |
+ + def sleep_like_a_baby():
|
| |
+ + global COUNTER
|
| |
+ + if COUNTER < 3:
|
| |
+ + COUNTER += 1
|
| |
+ + raise Exception("sleeping")
|
| |
+ + return []
|
| |
+ +
|
| |
+ + assert sleep_like_a_baby() == []
|
| |
+ + assert COUNTER == 3
|
| |
lslebodn
commented 5 years ago