commit 83fcbe745c3ee8f5f0fa29626a86c824db059b22
Author: James Antill <james@and.org>
Date: Thu Feb 7 12:58:07 2013 -0500
Fix problems with mirrors like wtfnix.com, delete bad MD files.
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index dfcf8f9..efbc42a 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -940,6 +941,7 @@ Insufficient space in download directory %s
range=(start, end),
)
except URLGrabError, e:
+ self._del_dl_file(local, size)
errstr = "failed to retrieve %s from %s\nerror was %s" % (relative, self, e)
if self.mirrorurls:
errstr +="\n You could try running: yum clean expire-cache"
@@ -961,6 +963,7 @@ Insufficient space in download directory %s
**kwargs
)
except URLGrabError, e:
+ self._del_dl_file(local, size)
errstr = "failure: %s from %s: %s" % (relative, self, e)
errors = getattr(e, 'errors', None)
raise Errors.NoMoreMirrorsRepoError(errstr, errors)
@@ -1652,6 +1655,18 @@ Insufficient space in download directory %s
raise URLGrabError(-1, 'repomd.xml does not match metalink for %s' %
self)
+ def _del_dl_file(self, local, size):
+ """ Delete a downloaded file if it's the correct size. """
+
+ sd = misc.stat_f(local)
+ if not sd: # File doesn't exist...
+ return
+
+ if size and sd.st_size < size:
+ return # Still more to get...
+
+ # Is the correct size, or too big ... delete it so we'll try again.
+ misc.unlink_f(local)
def checkMD(self, fn, mdtype, openchecksum=False):
"""check the metadata type against its checksum"""
@@ -1681,7 +1696,7 @@ Insufficient space in download directory %s
if size is not None:
size = int(size)
- if fast:
+ if fast and skip_old_DBMD_check:
fsize = misc.stat_f(file)
if fsize is None: # File doesn't exist...
return None
@@ -1756,16 +1771,21 @@ Insufficient space in download directory %s
try:
def checkfunc(obj):
- self.checkMD(obj, mdtype)
+ try:
+ self.checkMD(obj, mdtype)
+ except URLGrabError:
+ # Don't share MD among mirrors, in theory we could use:
+ # self._del_dl_file(local, int(thisdata.size))
+ # ...but this is safer.
+ misc.unlink_f(obj.filename)
+ raise
self.retrieved[mdtype] = 1
text = "%s/%s" % (self, mdtype)
if thisdata.size is None:
reget = None
else:
reget = 'simple'
- if os.path.exists(local):
- if os.stat(local).st_size >= int(thisdata.size):
- misc.unlink_f(local)
+ self._del_dl_file(local, int(thisdata.size))
local = self._getFile(relative=remote,
local=local,
copy_local=1,
commit c148eb10b798270b3d15087433c8efb2a79a69d0
Author: James Antill <james@and.org>
Date: Mon Feb 18 16:17:06 2013 -0500
Use xattr data as well as file size for "fast checksumming".
diff --git a/yum/yumRepo.py b/yum/yumRepo.py
index efbc42a..8c38093 100644
--- a/yum/yumRepo.py
+++ b/yum/yumRepo.py
@@ -52,15 +52,54 @@ import stat
import errno
import tempfile
-# If you want yum to _always_ check the MD .sqlite files then set this to
-# False (this doesn't affect .xml files or .sqilte files derived from them).
-# With this as True yum will only check when a new repomd.xml or
-# new MD is downloaded.
-# Note that with atomic MD, we can't have old MD lying around anymore so
-# the only way we need this check is if someone does something like:
-# cp primary.sqlite /var/cache/yum/blah
-# ...at which point you lose.
-skip_old_DBMD_check = True
+# This is unused now, probably nothing uses it but it was global/public.
+skip_old_DBMD_check = False
+
+try:
+ import xattr
+ if not hasattr(xattr, 'get') or not hasattr(xattr, 'set'):
+ xattr = None # This is a "newer" API.
+except ImportError:
+ xattr = None
+
+# The problem we are trying to solve here is that:
+#
+# 1. We rarely want to be downloading MD/pkgs/etc.
+# 2. We want to check those files are valid (match checksums) when we do
+# download them.
+# 3. We _really_ don't want to checksum all the files everytime we
+# run (100s of MBs).
+# 4. We can continue to download files from bad mirrors, or retry files due to
+# C-c etc.
+#
+# ...we used to solve this by just checking the file size, and assuming the
+# files had been downloaded and checksumed as correct if that matched. But that
+# was error prone on bad mirrors, so now we store the checksum in an
+# xattr ... this does mean that if you can't store xattrs (Eg. NFS) you will
+# rechecksum everything constantly.
+
+def _xattr_get_chksum(filename, chktype):
+ if not xattr:
+ return None
+
+ try:
+ ret = xattr.get(filename, 'user.yum.checksum.' + chktype)
+ except: # Documented to be "EnvironmentError", but make sure
+ return None
+
+ return ret
+
+def _xattr_set_chksum(filename, chktype, chksum):
+ if not xattr:
+ return None
+
+ try:
+ xattr.set(filename, 'user.yum.checksum.' + chktype, chksum)
+ except:
+ return False # Data too long. = IOError ... ignore everything.
+
+ return True
+
warnings.simplefilter("ignore", Errors.YumFutureDeprecationWarning)
@@ -228,7 +267,7 @@ class YumPackageSack(packageSack.PackageSack):
# get rid of all this stuff we don't need now
del repo.cacheHandler
- def _check_uncompressed_db_gen(self, repo, mdtype, fast=True):
+ def _check_uncompressed_db_gen(self, repo, mdtype):
"""return file name of db in gen/ dir if good, None if not"""
mydbdata = repo.repoXML.getData(mdtype)
@@ -238,7 +277,7 @@ class YumPackageSack(packageSack.PackageSack):
db_un_fn = mdtype + '.sqlite'
if not repo._checkMD(compressed_fn, mdtype, data=mydbdata,
- check_can_fail=fast, fast=fast):
+ check_can_fail=True):
return None
ret = misc.repo_gen_decompress(compressed_fn, db_un_fn,
@@ -261,8 +300,7 @@ class YumPackageSack(packageSack.PackageSack):
result = None
if os.path.exists(db_un_fn):
- if skip_old_DBMD_check and repo._using_old_MD:
- return db_un_fn
+
try:
repo.checkMD(db_un_fn, mdtype, openchecksum=True)
@@ -296,7 +334,6 @@ class YumRepository(Repository, config.RepoConf):
# eventually want
self.repoMDFile = 'repodata/repomd.xml'
self._repoXML = None
- self._using_old_MD = None
self._oldRepoMDData = {}
self.cache = 0
self.mirrorlistparsed = 0
@@ -1407,7 +1444,6 @@ Insufficient space in download directory %s
self._revertOldRepoXML()
return False
- self._using_old_MD = caching
if caching:
return False # Skip any work.
@@ -1673,7 +1709,7 @@ Insufficient space in download directory %s
return self._checkMD(fn, mdtype, openchecksum)
def _checkMD(self, fn, mdtype, openchecksum=False,
- data=None, check_can_fail=False, fast=False):
+ data=None, check_can_fail=False):
""" Internal function, use .checkMD() from outside yum. """
thisdata = data # So the argument name is nicer
@@ -1696,17 +1732,15 @@ Insufficient space in download directory %s
if size is not None:
size = int(size)
- if fast and skip_old_DBMD_check:
+ l_csum = _xattr_get_chksum(file, r_ctype)
+ if l_csum:
fsize = misc.stat_f(file)
- if fsize is None: # File doesn't exist...
- return None
- if size is None:
- return 1
- if size == fsize.st_size:
- return 1
- if check_can_fail:
- return None
- raise URLGrabError(-1, 'Metadata file does not match size')
+ if fsize is not None: # We just got an xattr, so it should be there
+ if size is None and l_csum == r_csum:
+ return 1
+ if size == fsize.st_size and l_csum == r_csum:
+ return 1
+ # Anything goes wrong, run the checksums as normal...
try: # get the local checksum
l_csum = self._checksum(r_ctype, file, datasize=size)
@@ -1716,6 +1750,7 @@ Insufficient space in download directory %s
raise URLGrabError(-3, 'Error performing checksum')
if l_csum == r_csum:
+ _xattr_set_chksum(file, r_ctype, l_csum)
return 1
else:
if check_can_fail: