Blob Blame History Raw
Fix fast upload avg calculation

Upstream verigak/progress is broken for very fast uploads; e.g.
with 'copr build' command, the next() call is called so often that
the avg() calculation probably suffers from some small
floating-point numbers problems:

  With 15MB/s => next() called for each 8096B => ~2000 calls/s

Since the upstream default window size is only of size 10 items
(by default), it calculates the average speed only for the last
~0.005s.  We could enlarge the size of window (sma_window param),
but the algorithm is so naive that it would decrease the
performance (O(N): sum(queue)/len(queue)).

This has been discussed very extensively with upstream (PR 24 and
friends) but I neither was not able to explain the problem, nor I
was able to convince upstream to accept my patches.

This downstream patch - while it keeps the backward API
compatibility - changes the algorithm so the average speed is
calculation is fast enough, and much more stable (by default it
calculates speed for window of 2 seconds).

Fork with this patch backported is maintained in
https://github.com/python-progress/python-progress

diff --git a/progress/__init__.py b/progress/__init__.py
index b434b30..f26056d 100644
--- a/progress/__init__.py
+++ b/progress/__init__.py
@@ -30,19 +30,55 @@ HIDE_CURSOR = '\x1b[?25l'
 SHOW_CURSOR = '\x1b[?25h'
 
 
+class _Window(object):
+    max_seconds = 2
+    max_items = None
+
+    def __init__(self, max_seconds=2, max_items=None):
+        self.max_seconds = max_seconds
+        self.max_items = max_items
+
+        stamp = monotonic()
+        self.last = stamp - 0.001
+        self.counter = 0
+        self.deque = deque()
+        self.next(0, stamp)
+
+    def pop(self):
+        item = self.deque.popleft()
+        self.counter -= item[1]
+
+    def clean(self):
+        if self.max_items:
+            while len(self.deque) > self.max_items:
+                self.pop()
+        while len(self.deque) > 2 and self.last - self.deque[1][0] > float(self.max_seconds):
+            self.pop()
+
+    def next(self, n, t):
+        self.clean()
+        self.deque.append((self.last, n))
+        self.last = t
+        self.counter += n
+
+    @property
+    def avg(self):
+        return self.counter / (self.last - self.deque[0][0])
+
+
 class Infinite(object):
     file = stderr
-    sma_window = 10         # Simple Moving Average window
+    # Maximum number of next() calls to be held in Simple Moving Average
+    # window structure (in memory), default is unlimited.
+    sma_window_seconds = 2
+    sma_window = None
     check_tty = True
     hide_cursor = True
 
     def __init__(self, message='', **kwargs):
         self.index = 0
         self.start_ts = monotonic()
-        self.avg = 0
-        self._avg_update_ts = self.start_ts
-        self._ts = self.start_ts
-        self._xput = deque(maxlen=self.sma_window)
+        self.window = _Window(self.sma_window_seconds, self.sma_window)
         for key, val in kwargs.items():
             setattr(self, key, val)
 
@@ -69,21 +105,17 @@ class Infinite(object):
     def elapsed(self):
         return int(monotonic() - self.start_ts)
 
+    @property
+    def avg(self):
+        speed = self.window.avg
+        if speed:
+            return 1/speed
+        return float("inf")
+
     @property
     def elapsed_td(self):
         return timedelta(seconds=self.elapsed)
 
-    def update_avg(self, n, dt):
-        if n > 0:
-            xput_len = len(self._xput)
-            self._xput.append(dt / n)
-            now = monotonic()
-            # update when we're still filling _xput, then after every second
-            if (xput_len < self.sma_window or
-                    now - self._avg_update_ts > 1):
-                self.avg = sum(self._xput) / len(self._xput)
-                self._avg_update_ts = now
-
     def update(self):
         pass
 
@@ -112,14 +144,10 @@ class Infinite(object):
         try:
             return self.file.isatty() if self.check_tty else True
         except AttributeError:
-            msg = "%s has no attribute 'isatty'. Try setting check_tty=False." % self
-            raise AttributeError(msg)
+            raise AttributeError('\'{}\' object has no attribute \'isatty\'. Try setting parameter check_tty=False.'.format(self))
 
     def next(self, n=1):
-        now = monotonic()
-        dt = now - self._ts
-        self.update_avg(n, dt)
-        self._ts = now
+        self.window.next(n, monotonic())
         self.index = self.index + n
         self.update()