From 9b23da47de17c9cc39f2918ed6be88846a5abf09 Mon Sep 17 00:00:00 2001 From: Zdenek Pavlas Date: Dec 18 2013 12:27:31 +0000 Subject: Update to latest HEAD. - Decrease the default_speed value. BZ 1043177 - Added client-side range support. BZ 435076 --- diff --git a/.gitignore b/.gitignore index 2e1fe04..5e327c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ urlgrabber-3.9.1.tar.gz /urlgrabber-3.10.tar.gz +/urlgrabber-3.10.1.tar.gz diff --git a/python-urlgrabber.spec b/python-urlgrabber.spec index f026a56..52583d3 100644 --- a/python-urlgrabber.spec +++ b/python-urlgrabber.spec @@ -2,8 +2,8 @@ Summary: A high-level cross-protocol url-grabber Name: python-urlgrabber -Version: 3.10 -Release: 1%{?dist} +Version: 3.10.1 +Release: 0%{?dist} Source0: http://urlgrabber.baseurl.org/download/urlgrabber-%{version}.tar.gz Patch1: urlgrabber-HEAD.patch @@ -44,6 +44,11 @@ rm -rf $RPM_BUILD_ROOT %attr(0755,root,root) %{_libexecdir}/urlgrabber-ext-down %changelog +* Wed Dec 18 2013 Zdenek Pavlas - 3.10.1-0 +- Update to latest HEAD. +- Decrease the default_speed value. BZ 1043177 +- Added client-side range support. BZ 435076 + * Mon Dec 9 2013 Zdenek Pavlas - 3.10-1 - Process mirror retries before other queued requests. - Tell curl to return immediately on ctrl-c. BZ 1017491 diff --git a/sources b/sources index 5e28765..350d00c 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -7cff064649619355d329b26d75872f60 urlgrabber-3.10.tar.gz +1f5dc63805623cc473e06204fd240bb2 urlgrabber-3.10.1.tar.gz diff --git a/urlgrabber-HEAD.patch b/urlgrabber-HEAD.patch index e2203c4..e69de29 100644 --- a/urlgrabber-HEAD.patch +++ b/urlgrabber-HEAD.patch @@ -1,71 +0,0 @@ -diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py -index b004f4d..d19ca01 100644 ---- a/urlgrabber/grabber.py -+++ b/urlgrabber/grabber.py -@@ -1893,6 +1893,8 @@ class PyCurlFileObject(object): - urllib.addinfourl, via. urllib.URLopener.* """ - return self.url - -+# tell curl to return immediately on ctrl-c -+pycurl.global_init(pycurl.GLOBAL_DEFAULT | pycurl.GLOBAL_ACK_EINTR) - _curl_cache = pycurl.Curl() # make one and reuse it over and over and over - - def reset_curl_obj(): -@@ -2170,6 +2172,7 @@ def parallel_wait(meter=None): - dl = _ExternalDownloaderPool() - host_con = {} # current host connection counts - single = set() # hosts in single connection mode -+ retry_queue = [] - - def start(opts, tries): - opts.tries = tries -@@ -2221,6 +2224,9 @@ def parallel_wait(meter=None): - # Turn on the max_connections=1 override. BZ 853432 - if DEBUG: DEBUG.info('max_connections(%s) %s => 1', key, limit) - single.add(key) -+ # When using multi-downloader the parent's _curl_cache -+ # object is idle. Kill it, as it might use keepalive=1. -+ reset_curl_obj() - - retry = opts.retry or 0 - if opts.failure_callback: -@@ -2247,7 +2253,7 @@ def parallel_wait(meter=None): - # mask this mirror and retry - if action.get('remove', 1): - removed.add(key) -- _async_queue.append(opts) -+ retry_queue.append(opts) - continue - # fail=1 from callback - ug_err.errors = errors -@@ -2257,19 +2263,22 @@ def parallel_wait(meter=None): - _run_callback(opts.failfunc, opts) - - try: -- idx = 0 -+ retry_idx = idx = 0 - while True: -- if idx >= len(_async_queue): -- # the queue is empty -+ if retry_idx < len(retry_queue): -+ # retries first -+ opts = retry_queue[retry_idx] -+ retry_idx += 1 -+ elif idx < len(_async_queue): -+ # handle next request -+ opts = _async_queue[idx] -+ idx += 1 -+ else: -+ # both queues are empty - if not dl.running: break -- # pending dl may extend it - perform() - continue - -- # handle next request -- opts = _async_queue[idx] -- idx += 1 -- - # check global limit - while len(dl.running) >= default_grabber.opts.max_connections: - perform()