Fix fast upload avg calculation Upstream verigak/progress is broken for very fast uploads; e.g. with 'copr build' command, the next() call is called so often that the avg() calculation probably suffers from some small floating-point numbers problems: With 15MB/s => next() called for each 8096B => ~2000 calls/s Since the upstream default window size is only of size 10 items (by default), it calculates the average speed only for the last ~0.005s. We could enlarge the size of window (sma_window param), but the algorithm is so naive that it would decrease the performance. This has been discussed very extensively with upstream (PR 24 and friends) but I neither was not able to explain the problem, nor I was able to convince upstream to accept my patches. This downstream patch - while it keeps the backward API compatibility - changes the algorithm so the average speed is calculation is fast enough, and much more stable (by default it calculates speed for window of 2 seconds). We also don't seem to have the monotonic() mess, since we don't seem to suffer from the same issues. Fork with this patch backported is maintained in https://github.com/python-progress/python-progress diff --git a/MANIFEST.in b/MANIFEST.in index ef7c4cb..0c73842 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1 @@ include README.rst LICENSE -include test_*.py diff --git a/progress/__init__.py b/progress/__init__.py index e434c25..1cbdce6 100644 --- a/progress/__init__.py +++ b/progress/__init__.py @@ -18,10 +18,7 @@ from collections import deque from datetime import timedelta from math import ceil from sys import stderr -try: - from time import monotonic -except ImportError: - from time import time as monotonic +from time import time __version__ = '1.5' @@ -30,19 +27,55 @@ HIDE_CURSOR = '\x1b[?25l' SHOW_CURSOR = '\x1b[?25h' +class _Window(object): + max_seconds = 2 + max_items = None + + def __init__(self, max_seconds=2, max_items=None): + self.max_seconds = max_seconds + self.max_items = max_items + + stamp = time() + self.last = stamp - 0.001 + self.counter = 0 + self.deque = deque() + self.next(0, stamp) + + def pop(self): + item = self.deque.popleft() + self.counter -= item[1] + + def clean(self): + if self.max_items: + while len(self.deque) > self.max_items: + self.pop() + while len(self.deque) > 2 and self.last - self.deque[0][0] > float(self.max_seconds): + self.pop() + + def next(self, n, t): + self.clean() + self.deque.append((self.last, n)) + self.last = t + self.counter += n + + @property + def avg(self): + return self.counter / (self.last - self.deque[0][0]) + + class Infinite(object): file = stderr - sma_window = 10 # Simple Moving Average window + # Maximum number of next() calls to be held in Simple Moving Average + # window structure (in memory), default is unlimited. + sma_window_seconds = 2 + sma_window = None check_tty = True hide_cursor = True def __init__(self, message='', **kwargs): self.index = 0 - self.start_ts = monotonic() - self.avg = 0 - self._avg_update_ts = self.start_ts - self._ts = self.start_ts - self._xput = deque(maxlen=self.sma_window) + self.start_ts = time() + self.window = _Window(self.sma_window_seconds, self.sma_window) for key, val in kwargs.items(): setattr(self, key, val) @@ -62,23 +95,19 @@ class Infinite(object): @property def elapsed(self): - return int(monotonic() - self.start_ts) + return int(time() - self.start_ts) + + @property + def avg(self): + speed = self.window.avg + if speed: + return 1/speed + return 3600 # better constant? @property def elapsed_td(self): return timedelta(seconds=self.elapsed) - def update_avg(self, n, dt): - if n > 0: - xput_len = len(self._xput) - self._xput.append(dt / n) - now = monotonic() - # update when we're still filling _xput, then after every second - if (xput_len < self.sma_window or - now - self._avg_update_ts > 1): - self.avg = sum(self._xput) / len(self._xput) - self._avg_update_ts = now - def update(self): pass @@ -112,10 +141,7 @@ class Infinite(object): return self.file.isatty() if self.check_tty else True def next(self, n=1): - now = monotonic() - dt = now - self._ts - self.update_avg(n, dt) - self._ts = now + self.window.next(n, time()) self.index = self.index + n self.update()