diff --git a/createrepo-head.patch b/createrepo-head.patch index 5518870..b7858e5 100644 --- a/createrepo-head.patch +++ b/createrepo-head.patch @@ -1,1313 +1,94 @@ -diff --git a/Makefile b/Makefile -index facbc5b..c821ae4 100644 ---- a/Makefile -+++ b/Makefile -@@ -95,6 +95,12 @@ distclean: clean - $(MAKE) -C $$subdir distclean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ - done - -+pylint: -+ @pylint --rcfile=test/createrepo-pylintrc *.py createrepo -+ -+pylint-short: -+ @pylint -r n --rcfile=test/createrepo-pylintrc *.py createrepo -+ - mostlyclean: - $(MAKE) clean - +diff --git a/createrepo.spec b/createrepo.spec +index 0d4bbd8..57a0196 100644 +--- a/createrepo.spec ++++ b/createrepo.spec +@@ -11,7 +11,7 @@ URL: http://createrepo.baseurl.org/ + BuildRoot: %{_tmppath}/%{name}-%{version}root + BuildArchitectures: noarch + Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python +-Requires: yum-metadata-parser, yum >= 3.2.23 ++Requires: yum-metadata-parser, yum >= 3.2.23, python-deltarpm + + %description + This utility will generate a common metadata repository from a directory of diff --git a/createrepo/__init__.py b/createrepo/__init__.py -index 2a880a8..609569a 100644 +index 4601a15..77a03ee 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py -@@ -11,12 +11,12 @@ - # You should have received a copy of the GNU General Public License - # along with this program; if not, write to the Free Software - # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. --# Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org -+# Copyright 2009 Red Hat, Inc - -+# written by seth vidal skvidal at fedoraproject.org - - import os - import sys - import libxml2 --import string - import fnmatch - import time - import yumbased -@@ -24,6 +24,7 @@ import shutil - from bz2 import BZ2File - from urlgrabber import grabber - import tempfile -+import stat - - from yum import misc, Errors, to_unicode - from yum.sqlutils import executeSQL -@@ -43,7 +44,8 @@ try: - except ImportError: - pass - --from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename -+from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \ -+ checksum_and_rename - import deltarpms - - __version__ = '0.9.7' -@@ -66,11 +68,13 @@ class MetaDataConfig(object): - self.split = False - self.update = False - self.deltas = False # do the deltarpm thing -- self.deltadir = None # where to put the .drpms - defaults to 'drpms' inside 'repodata' -+ # where to put the .drpms - defaults to 'drpms' inside 'repodata' -+ self.deltadir = None - self.delta_relative = 'drpms/' - self.oldpackage_paths = [] # where to look for the old packages - - self.deltafile = 'prestodelta.xml.gz' - self.num_deltas = 1 # number of older versions to delta (max) -+ self.max_delta_rpm_size = 100000000 - self.update_md_path = None - self.skip_stat = False - self.database = False -@@ -153,10 +157,12 @@ class MetaDataGenerator: - self.conf.basedir = os.path.realpath(self.conf.basedir) - self.conf.relative_dir = self.conf.directory - -- self.package_dir = os.path.join(self.conf.basedir, self.conf.relative_dir) -+ self.package_dir = os.path.join(self.conf.basedir, -+ self.conf.relative_dir) - - if not self.conf.outputdir: -- self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir) -+ self.conf.outputdir = os.path.join(self.conf.basedir, -+ self.conf.relative_dir) - - def _test_setup_dirs(self): - # start the sanity/stupidity checks -@@ -187,7 +193,8 @@ class MetaDataGenerator: - raise MDError, _('Cannot create/verify %s') % temp_final - - if self.conf.deltas: -- temp_delta = os.path.join(self.conf.outputdir, self.conf.delta_relative) -+ temp_delta = os.path.join(self.conf.outputdir, -+ self.conf.delta_relative) - if not checkAndMakeDir(temp_delta): - raise MDError, _('Cannot create/verify %s') % temp_delta - self.conf.deltadir = temp_delta -@@ -202,15 +209,19 @@ class MetaDataGenerator: - direcs.append('deltadir') - - for direc in direcs: -- filepath = os.path.join(self.conf.outputdir, getattr(self.conf, direc)) -+ filepath = os.path.join(self.conf.outputdir, getattr(self.conf, -+ direc)) - if os.path.exists(filepath): - if not os.access(filepath, os.W_OK): - raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath - - if self.conf.checkts: -- timestamp = os.path.getctime(filepath) -- if timestamp > self.conf.mdtimestamp: -- self.conf.mdtimestamp = timestamp -+ # checking for repodata/repomd.xml - not just the data dir -+ rxml = filepath + '/repomd.xml' -+ if os.path.exists(rxml): -+ timestamp = os.path.getctime(rxml) -+ if timestamp > self.conf.mdtimestamp: -+ self.conf.mdtimestamp = timestamp - - if self.conf.groupfile: - a = self.conf.groupfile -@@ -227,7 +238,7 @@ class MetaDataGenerator: - if self.conf.cachedir: - a = self.conf.cachedir - if not os.path.isabs(a): -- a = os.path.join(self.conf.outputdir ,a) -+ a = os.path.join(self.conf.outputdir, a) - if not checkAndMakeDir(a): - raise MDError, _('Error: cannot open/write to cache dir %s' % a) - -@@ -248,7 +259,6 @@ class MetaDataGenerator: - name = os.path.join(top, name) - if os.path.isdir(name): - self._os_path_walk(name, func, arg) -- # module - def getFileList(self, directory, ext): - """Return all files in path matching ext, store them in filelist, - recurse dirs. Returns a list object""" -@@ -264,7 +274,7 @@ class MetaDataGenerator: - elif fn[-extlen:].lower() == '%s' % (ext): - relativepath = dirname.replace(startdir, "", 1) - relativepath = relativepath.lstrip("/") -- filelist.append(os.path.join(relativepath,fn)) -+ filelist.append(os.path.join(relativepath, fn)) - - filelist = [] - startdir = directory + '/' -@@ -276,8 +286,8 @@ class MetaDataGenerator: - errorprint(thing) - - def checkTimeStamps(self): -- """check the timestamp of our target dir. If it is not newer than the repodata -- return False, else True""" -+ """check the timestamp of our target dir. If it is not newer than -+ the repodata return False, else True""" - if self.conf.checkts: - dn = os.path.join(self.conf.basedir, self.conf.directory) - files = self.getFileList(dn, '.rpm') -@@ -295,14 +305,14 @@ class MetaDataGenerator: - - def trimRpms(self, files): - badrpms = [] -- for file in files: -+ for rpm_file in files: - for glob in self.conf.excludes: -- if fnmatch.fnmatch(file, glob): -- if file not in badrpms: -- badrpms.append(file) -- for file in badrpms: -- if file in files: -- files.remove(file) -+ if fnmatch.fnmatch(rpm_file, glob): -+ if rpm_file not in badrpms: -+ badrpms.append(rpm_file) -+ for rpm_file in badrpms: -+ if rpm_file in files: -+ files.remove(rpm_file) - return files - - def _setup_old_metadata_lookup(self): -@@ -356,7 +366,6 @@ class MetaDataGenerator: - self.writeMetadataDocs(packages) - self.closeMetadataDocs() - -- # module - def openMetadataDocs(self): - if self.conf.database_only: - self.setup_sqlite_dbs() -@@ -369,34 +378,40 @@ class MetaDataGenerator: - - def _setupPrimary(self): - # setup the primary metadata file -- primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.primaryfile) -+ primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, -+ self.conf.primaryfile) - fo = _gzipOpen(primaryfilepath, 'w') - fo.write('\n') -- fo.write('' % -+ fo.write('' % - self.pkgcount) - return fo - - def _setupFilelists(self): - # setup the filelist file -- filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.filelistsfile) -+ filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, -+ self.conf.filelistsfile) - fo = _gzipOpen(filelistpath, 'w') - fo.write('\n') -- fo.write('' % -- self.pkgcount) -+ fo.write('' % self.pkgcount) - return fo - - def _setupOther(self): - # setup the other file -- otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.otherfile) -+ otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, -+ self.conf.otherfile) - fo = _gzipOpen(otherfilepath, 'w') - fo.write('\n') -- fo.write('' % -+ fo.write('' % - self.pkgcount) - return fo - - def _setupDelta(self): - # setup the other file -- deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.deltafile) -+ deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, -+ self.conf.deltafile) - fo = _gzipOpen(deltafilepath, 'w') - fo.write('\n') - fo.write('\n') -@@ -405,7 +420,6 @@ class MetaDataGenerator: - - def read_in_package(self, rpmfile, pkgpath=None, reldir=None): - """rpmfile == relative path to file from self.packge_dir""" -- remote_package = False - baseurl = self.conf.baseurl - - if not pkgpath: -@@ -415,7 +429,6 @@ class MetaDataGenerator: - raise MDError, "Blank filename passed in, skipping" - - if rpmfile.find("://") != -1: -- remote_package = True - - if not hasattr(self, 'tempdir'): - self.tempdir = tempfile.mkdtemp() -@@ -425,18 +438,20 @@ class MetaDataGenerator: - reldir = self.tempdir - dest = os.path.join(self.tempdir, pkgname) - if not self.conf.quiet: -- self.callback.log('\nDownloading %s' % rpmfile) -+ self.callback.log('\nDownloading %s' % rpmfile) - try: - rpmfile = self.grabber.urlgrab(rpmfile, dest) - except grabber.URLGrabError, e: -- raise MDError, "Unable to retrieve remote package %s: %s" %(rpmfile, e) -+ raise MDError, "Unable to retrieve remote package %s: %s" % ( -+ rpmfile, e) - - - else: - rpmfile = '%s/%s' % (pkgpath, rpmfile) - - try: -- po = yumbased.CreateRepoPackage(self.ts, rpmfile) -+ po = yumbased.CreateRepoPackage(self.ts, rpmfile, -+ sumtype=self.conf.sumtype) - except Errors.MiscError, e: - raise MDError, "Unable to open package: %s" % e - # external info we need -@@ -449,7 +464,8 @@ class MetaDataGenerator: - self.rpmlib_reqs[r] = 1 - - if po.checksum in (None, ""): -- raise MDError, "No Package ID found for package %s, not going to add it" % po -+ raise MDError, "No Package ID found for package %s, not going to" \ -+ " add it" % po - - return po - -@@ -459,9 +475,9 @@ class MetaDataGenerator: - pkglist = self.conf.pkglist - - if not pkgpath: -- directory=self.conf.directory -+ directory = self.conf.directory - else: -- directory=pkgpath -+ directory = pkgpath - - for pkg in pkglist: - self.current_pkg += 1 -@@ -492,13 +508,15 @@ class MetaDataGenerator: - if not isinstance(pkg, YumAvailablePackage): - - try: -- po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir) -+ po = self.read_in_package(pkg, pkgpath=pkgpath, -+ reldir=reldir) - except MDError, e: - # need to say something here - self.callback.errorlog("\nError %s: %s\n" % (pkg, e)) - continue - # we can use deltas: -- self._do_delta_rpm_package(po) -+ if self.conf.deltas: -+ self._do_delta_rpm_package(po) - - else: - po = pkg -@@ -513,12 +531,13 @@ class MetaDataGenerator: - clog_limit=self.conf.changelog_limit)) - else: - if self.conf.verbose: -- self.callback.log(_("Using data from old metadata for %s") % pkg) -+ self.callback.log(_("Using data from old metadata for %s") -+ % pkg) - (primarynode, filenode, othernode) = nodes - -- for node, outfile in ((primarynode,self.primaryfile), -- (filenode,self.flfile), -- (othernode,self.otherfile)): -+ for node, outfile in ((primarynode, self.primaryfile), -+ (filenode, self.flfile), -+ (othernode, self.otherfile)): - if node is None: - break - output = node.serialize('UTF-8', self.conf.pretty) -@@ -526,17 +545,19 @@ class MetaDataGenerator: - outfile.write(output) - else: - if self.conf.verbose: -- self.callback.log(_("empty serialize on write to %s in %s") % (outfile, pkg)) -+ self.callback.log(_("empty serialize on write to" \ -+ "%s in %s") % (outfile, pkg)) - outfile.write('\n') - - self.oldData.freeNodes(pkg) - #FIXME - if we're in update and we have deltas enabled -- # check the presto data for this pkg and write its info back out -- # to our deltafile -+ # check the presto data for this pkg and write its info back out -+ # to our deltafile - - if not self.conf.quiet: - if self.conf.verbose: -- self.callback.log('%d/%d - %s' % (self.current_pkg, self.pkgcount, pkg)) -+ self.callback.log('%d/%d - %s' % (self.current_pkg, -+ self.pkgcount, pkg)) - else: - self.callback.progress(pkg, self.current_pkg, self.pkgcount) - -@@ -574,24 +595,36 @@ class MetaDataGenerator: - self.otherfile.close() - - if self.conf.deltas: -+ deltam_st = time.time() - if not self.conf.quiet: - self.callback.log(_('Saving delta metadata')) - self.deltafile.write(self.generate_delta_xml()) - self.deltafile.write('\n') - self.deltafile.close() -+ if self.conf.profile: -+ self.callback.log('deltam time: %0.3f' % (time.time() - deltam_st)) - - def _do_delta_rpm_package(self, pkg): - """makes the drpms, if possible, for this package object. - returns the presto/delta xml metadata as a string - """ -+ drpm_pkg_time = time.time() -+ # duck and cover if the pkg.size is > whatever -+ if int(pkg.size) > self.conf.max_delta_rpm_size: -+ if not self.conf.quiet: -+ self.callback.log("Skipping %s package " \ -+ "that is > max_delta_rpm_size" % pkg) -+ return - - # generate a list of all the potential 'old rpms' -- opd = self._get_old_package_dict() # yes I could make this a property but -- -- # for each of our old_package_paths - make a drpm from the newest of that pkg -+ opd = self._get_old_package_dict() -+ # for each of our old_package_paths - -+ # make a drpm from the newest of that pkg - # get list of potential candidates which are likely to match - for d in self.conf.oldpackage_paths: - pot_cand = [] -+ if d not in opd: -+ continue - for fn in opd[d]: - if os.path.basename(fn).startswith(pkg.name): - pot_cand.append(fn) -@@ -599,7 +632,8 @@ class MetaDataGenerator: - candidates = [] - for fn in pot_cand: - try: -- thispo = yumbased.CreateRepoPackage(self.ts, fn) -+ thispo = yumbased.CreateRepoPackage(self.ts, fn, -+ sumtype=self.conf.sumtype) - except Errors.MiscError, e: - continue - if (thispo.name, thispo.arch) != (pkg.name, pkg.arch): -@@ -615,9 +649,14 @@ class MetaDataGenerator: - - for delta_p in candidates[0:self.conf.num_deltas]: - #make drpm of pkg and delta_p -+ dt_st = time.time() - drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir) -- self.callback.log('created drpm from %s to %s: %s' % ( -- delta_p, pkg, drpmfn)) -+ if not self.conf.quiet or self.conf.profile: -+ self.callback.log('created drpm from %s to %s: %s in %0.3f' % ( -+ delta_p, pkg, drpmfn, (time.time() - dt_st))) -+ if self.conf.profile: -+ self.callback.log('total drpm time for %s: %0.3f' % (pkg, -+ (time.time() - drpm_pkg_time))) - - def _get_old_package_dict(self): - if hasattr(self, '_old_package_dict'): -@@ -627,6 +666,12 @@ class MetaDataGenerator: - opl = [] - for d in self.conf.oldpackage_paths: - for f in self.getFileList(d, 'rpm'): -+ fp = d + '/' + f -+ fpstat = os.stat(fp) -+ if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: -+ self.callback.log("Skipping %s package " \ -+ "that is > max_delta_rpm_size" % f) -+ continue - if not self._old_package_dict.has_key(d): - self._old_package_dict[d] = [] - self._old_package_dict[d].append(d + '/' + f) -@@ -642,24 +687,29 @@ class MetaDataGenerator: - # the tag for the target + each of the drpm infos + closure for the target - # tag - targets = {} -- result = u'' -+ results = [] - for drpm_fn in self.getFileList(self.conf.deltadir, 'drpm'): -- drpm_rel_fn = os.path.normpath(self.conf.delta_relative + '/' + drpm_fn) # this is annoying -- drpm_po = yumbased.CreateRepoPackage(self.ts, self.conf.deltadir + '/' + drpm_fn) -+ drpm_rel_fn = os.path.normpath(self.conf.delta_relative + -+ '/' + drpm_fn) # this is annoying -+ drpm_po = yumbased.CreateRepoPackage(self.ts, -+ self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype) - -- drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, drpm_rel_fn) -+ drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, -+ drpm_rel_fn) - if not targets.has_key(drpm_po.pkgtup): -- targets[drpm_po.pkgtup] = u'' -- targets[drpm_po.pkgtup] += drpm.xml_dump_metadata() -+ targets[drpm_po.pkgtup] = [] -+ targets[drpm_po.pkgtup].append(drpm.xml_dump_metadata()) - -- for (n,e,v,r,a) in targets.keys(): -- result += """ \n""" -+ for (n, a, e, v, r) in targets.keys(): -+ results.append(""" \n""" % ( -+ n, e, v, r, a)) -+ results.extend(targets[(n,a,e,v,r)]) -+# for src in targets[(n, a, e, v, r)]: -+# results.append(src) -+ -+ results.append(" \n") - -- return result -+ return ' '.join(results) - - def addArbitraryMetadata(self, mdfile, mdtype, xml_node, compress=True, - compress_type='gzip', attribs={}): -@@ -695,7 +745,7 @@ class MetaDataGenerator: - - - if self.conf.unique_md_filenames: -- (csum, outfn) = checksum_and_rename(outfn) -+ (csum, outfn) = checksum_and_rename(outfn, self.conf.sumtype) - sfile = os.path.basename(outfn) - else: - if compress: -@@ -721,12 +771,13 @@ class MetaDataGenerator: - timestamp = data.newChild(None, 'timestamp', str(timest)) - - # add the random stuff -- for (k,v) in attribs.items(): -+ for (k, v) in attribs.items(): - data.newChild(None, k, str(v)) - - - def doRepoMetadata(self): -- """wrapper to generate the repomd.xml file that stores the info on the other files""" -+ """wrapper to generate the repomd.xml file that stores the info -+ on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) -@@ -740,7 +791,7 @@ class MetaDataGenerator: - tags = reporoot.newChild(None, 'tags', None) - for item in self.conf.content_tags: - c_tags = tags.newChild(None, 'content', item) -- for (cpeid,item) in self.conf.distro_tags: -+ for (cpeid, item) in self.conf.distro_tags: - d_tags = tags.newChild(None, 'distro', item) - if cpeid: - d_tags.newProp('cpeid', cpeid) -@@ -757,10 +808,10 @@ class MetaDataGenerator: - (self.conf.filelistsfile, 'filelists'), - (self.conf.primaryfile, 'primary')] - db_workfiles = [] -- repoid='garbageid' -+ repoid = 'garbageid' - - if self.conf.deltas: -- workfiles.append((self.conf.deltafile, 'deltainfo')) -+ workfiles.append((self.conf.deltafile, 'prestodelta')) - if self.conf.database: - if not self.conf.quiet: self.callback.log('Generating sqlite DBs') - try: -@@ -769,11 +820,16 @@ class MetaDataGenerator: - dbversion = '9' - rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None) - -- for (file, ftype) in workfiles: -- complete_path = os.path.join(repopath, file) -+ for (rpm_file, ftype) in workfiles: -+ complete_path = os.path.join(repopath, rpm_file) - - zfo = _gzipOpen(complete_path) -- uncsum = misc.checksum(sumtype, zfo) -+ # This is misc.checksum() done locally so we can get the size too. -+ data = misc.Checksums([sumtype]) -+ while data.read(zfo, 2**16): -+ pass -+ uncsum = data.hexdigest(sumtype) -+ unsize = len(data) - zfo.close() - csum = misc.checksum(sumtype, complete_path) - timestamp = os.stat(complete_path)[8] -@@ -782,8 +838,10 @@ class MetaDataGenerator: - db_compressed_sums = {} - - if self.conf.database: -- if self.conf.verbose: -- self.callback.log("Starting %s db creation: %s" % (ftype, time.ctime())) -+ if ftype in ['primary', 'filelists', 'other']: -+ if self.conf.verbose: -+ self.callback.log("Starting %s db creation: %s" % (ftype, -+ time.ctime())) - - if ftype == 'primary': - rp.getPrimary(complete_path, csum) -@@ -793,54 +851,67 @@ class MetaDataGenerator: - - elif ftype == 'other': - rp.getOtherdata(complete_path, csum) -+ -+ if ftype in ['primary', 'filelists', 'other']: -+ tmp_result_name = '%s.xml.gz.sqlite' % ftype -+ tmp_result_path = os.path.join(repopath, tmp_result_name) -+ good_name = '%s.sqlite' % ftype -+ resultpath = os.path.join(repopath, good_name) - -- -- -- tmp_result_name = '%s.xml.gz.sqlite' % ftype -- tmp_result_path = os.path.join(repopath, tmp_result_name) -- good_name = '%s.sqlite' % ftype -- resultpath = os.path.join(repopath, good_name) -- -- # rename from silly name to not silly name -- os.rename(tmp_result_path, resultpath) -- compressed_name = '%s.bz2' % good_name -- result_compressed = os.path.join(repopath, compressed_name) -- db_csums[ftype] = misc.checksum(sumtype, resultpath) -- -- # compress the files -- bzipFile(resultpath, result_compressed) -- # csum the compressed file -- db_compressed_sums[ftype] = misc.checksum(sumtype, result_compressed) -- # remove the uncompressed file -- os.unlink(resultpath) -- -- if self.conf.unique_md_filenames: -- csum_compressed_name = '%s-%s.bz2' % (db_compressed_sums[ftype], good_name) -- csum_result_compressed = os.path.join(repopath, csum_compressed_name) -- os.rename(result_compressed, csum_result_compressed) -- result_compressed = csum_result_compressed -- compressed_name = csum_compressed_name -+ # rename from silly name to not silly name -+ os.rename(tmp_result_path, resultpath) -+ compressed_name = '%s.bz2' % good_name -+ result_compressed = os.path.join(repopath, compressed_name) -+ db_csums[ftype] = misc.checksum(sumtype, resultpath) -+ -+ # compress the files -+ bzipFile(resultpath, result_compressed) -+ # csum the compressed file -+ db_compressed_sums[ftype] = misc.checksum(sumtype, -+ result_compressed) -+ # timestamp+size the uncompressed file -+ un_stat = os.stat(resultpath) -+ # remove the uncompressed file -+ os.unlink(resultpath) -+ -+ if self.conf.unique_md_filenames: -+ csum_compressed_name = '%s-%s.bz2' % ( -+ db_compressed_sums[ftype], good_name) -+ csum_result_compressed = os.path.join(repopath, -+ csum_compressed_name) -+ os.rename(result_compressed, csum_result_compressed) -+ result_compressed = csum_result_compressed -+ compressed_name = csum_compressed_name - -- # timestamp the compressed file -- db_timestamp = os.stat(result_compressed)[8] -+ # timestamp+size the compressed file -+ db_stat = os.stat(result_compressed) +@@ -298,8 +298,8 @@ class MetaDataGenerator: + self.callback.errorlog(_('cannot get to file: %s') % fn) + if os.path.getctime(fn) > self.conf.mdtimestamp: + return False +- else: +- return True ++ ++ return True -- # add this data as a section to the repomdxml -- db_data_type = '%s_db' % ftype -- data = reporoot.newChild(None, 'data', None) -- data.newProp('type', db_data_type) -- location = data.newChild(None, 'location', None) -- if self.conf.baseurl is not None: -- location.newProp('xml:base', self.conf.baseurl) -+ # add this data as a section to the repomdxml -+ db_data_type = '%s_db' % ftype -+ data = reporoot.newChild(None, 'data', None) -+ data.newProp('type', db_data_type) -+ location = data.newChild(None, 'location', None) - -- location.newProp('href', os.path.join(self.conf.finaldir, compressed_name)) -- checksum = data.newChild(None, 'checksum', db_compressed_sums[ftype]) -- checksum.newProp('type', sumtype) -- db_tstamp = data.newChild(None, 'timestamp', str(db_timestamp)) -- unchecksum = data.newChild(None, 'open-checksum', db_csums[ftype]) -- unchecksum.newProp('type', sumtype) -- database_version = data.newChild(None, 'database_version', dbversion) -- if self.conf.verbose: -- self.callback.log("Ending %s db creation: %s" % (ftype, time.ctime())) -+ if self.conf.baseurl is not None: -+ location.newProp('xml:base', self.conf.baseurl) -+ -+ location.newProp('href', os.path.join(self.conf.finaldir, -+ compressed_name)) -+ checksum = data.newChild(None, 'checksum', -+ db_compressed_sums[ftype]) -+ checksum.newProp('type', sumtype) -+ db_tstamp = data.newChild(None, 'timestamp', -+ str(db_stat.st_mtime)) -+ data.newChild(None, 'size', str(db_stat.st_size)) -+ data.newChild(None, 'open-size', str(un_stat.st_size)) -+ unchecksum = data.newChild(None, 'open-checksum', -+ db_csums[ftype]) -+ unchecksum.newProp('type', sumtype) -+ database_version = data.newChild(None, 'database_version', -+ dbversion) -+ if self.conf.verbose: -+ self.callback.log("Ending %s db creation: %s" % (ftype, -+ time.ctime())) - - - -@@ -850,6 +921,9 @@ class MetaDataGenerator: - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', sumtype) - timestamp = data.newChild(None, 'timestamp', str(timestamp)) -+ size = os.stat(os.path.join(repopath, rpm_file)) -+ data.newChild(None, 'size', str(size.st_size)) -+ data.newChild(None, 'open-size', str(unsize)) - unchecksum = data.newChild(None, 'open-checksum', uncsum) - unchecksum.newProp('type', sumtype) - location = data.newChild(None, 'location', None) -@@ -857,19 +931,20 @@ class MetaDataGenerator: - location.newProp('xml:base', self.conf.baseurl) - if self.conf.unique_md_filenames: - res_file = '%s-%s.xml.gz' % (csum, ftype) -- orig_file = os.path.join(repopath, file) -+ orig_file = os.path.join(repopath, rpm_file) - dest_file = os.path.join(repopath, res_file) - os.rename(orig_file, dest_file) - - else: -- res_file = file -+ res_file = rpm_file - -- file = res_file -+ rpm_file = res_file - -- location.newProp('href', os.path.join(self.conf.finaldir, file)) -+ location.newProp('href', os.path.join(self.conf.finaldir, rpm_file)) - - -- if not self.conf.quiet and self.conf.database: self.callback.log('Sqlite DBs complete') -+ if not self.conf.quiet and self.conf.database: -+ self.callback.log('Sqlite DBs complete') + return False - for (fn, ftype) in db_workfiles: - attribs = {'database_version':dbversion} -@@ -883,7 +958,8 @@ class MetaDataGenerator: - - if self.conf.groupfile is not None: - self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot) -- self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, compress=False) -+ self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, -+ compress=False) - - if self.conf.additional_metadata: - for md_type, mdfile in self.conf.additional_metadata.items(): -@@ -900,7 +976,8 @@ class MetaDataGenerator: - try: - repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) - except: -- self.callback.errorlog(_('Error saving temp file for repomd.xml: %s') % repofilepath) -+ self.callback.errorlog( -+ _('Error saving temp file for repomd.xml: %s') % repofilepath) - raise MDError, 'Could not save temp file: %s' % repofilepath - - del repodoc -@@ -918,8 +995,8 @@ class MetaDataGenerator: - try: - os.rename(output_final_dir, output_old_dir) - except: -- raise MDError, _('Error moving final %s to old dir %s' % (output_final_dir, -- output_old_dir)) -+ raise MDError, _('Error moving final %s to old dir %s' % ( -+ output_final_dir, output_old_dir)) - - output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) - -@@ -930,7 +1007,8 @@ class MetaDataGenerator: - os.rename(output_old_dir, output_final_dir) - raise MDError, _('Error moving final metadata into place') - -- for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']: -+ for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', -+ 'groupfile']: - if getattr(self.conf, f): - fn = os.path.basename(getattr(self.conf, f)) - else: -@@ -941,7 +1019,8 @@ class MetaDataGenerator: - try: - os.remove(oldfile) - except OSError, e: -- raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e) -+ raise MDError, _( -+ 'Could not remove old metadata file: %s: %s') % (oldfile, e) - - # Move everything else back from olddir (eg. repoview files) - for f in os.listdir(output_old_dir): -@@ -952,7 +1031,8 @@ class MetaDataGenerator: - 'other.xml.gz','filelists.xml.gz'): - os.remove(oldfile) # kill off the old ones - continue -- if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', 'primary.sqlite.bz2'): -+ if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', -+ 'primary.sqlite.bz2'): - os.remove(oldfile) - continue - -@@ -964,7 +1044,8 @@ class MetaDataGenerator: - else: - os.remove(oldfile) - except OSError, e: -- raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e) -+ raise MDError, _( -+ 'Could not remove old metadata file: %s: %s') % (oldfile, e) - else: - try: - os.rename(oldfile, finalfile) -@@ -976,7 +1057,8 @@ class MetaDataGenerator: - try: - os.rmdir(output_old_dir) - except OSError, e: -- self.errorlog(_('Could not remove old metadata dir: %s') % self.conf.olddir) -+ self.errorlog(_('Could not remove old metadata dir: %s') -+ % self.conf.olddir) - self.errorlog(_('Error was %s') % e) - self.errorlog(_('Please clean up this directory manually.')) - -@@ -986,7 +1068,8 @@ class MetaDataGenerator: - try: - self.md_sqlite = MetaDataSqlite(destdir) - except sqlite.OperationalError, e: -- raise MDError, _('Cannot create sqlite databases: %s.\nMaybe you need to clean up a .repodata dir?') % e -+ raise MDError, _('Cannot create sqlite databases: %s.\n'\ -+ 'Maybe you need to clean up a .repodata dir?') % e - - - -@@ -1014,11 +1097,11 @@ class SplitMetaDataGenerator(MetaDataGenerator): - for fn in names: - if os.path.isdir(fn): - continue -- elif string.lower(fn[-extlen:]) == '%s' % (ext): -+ elif fn[-extlen:].lower() == '%s' % (ext): - reldir = os.path.basename(dirname) - if reldir == os.path.basename(directory): - reldir = "" -- arg.append(os.path.join(reldir,fn)) -+ arg.append(os.path.join(reldir, fn)) - - rpmlist = [] - os.path.walk(directory, extension_visitor, rpmlist) -@@ -1116,7 +1199,7 @@ class MetaDataSqlite(object): - - def create_filelists_db(self): - schema = [ -- """PRAGMA synchronous="0FF";""", -+ """PRAGMA synchronous="OFF";""", - """pragma locking_mode="EXCLUSIVE";""", - """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", - """CREATE TABLE filelist ( pkgKey INTEGER, dirname TEXT, filenames TEXT, filetypes TEXT);""", diff --git a/createrepo/deltarpms.py b/createrepo/deltarpms.py -index f201ab7..26a6715 100644 +index 26a6715..d14d3fa 100644 --- a/createrepo/deltarpms.py +++ b/createrepo/deltarpms.py -@@ -19,8 +19,7 @@ import os.path +@@ -18,7 +18,7 @@ + import os.path import commands from yum import misc - import gzip --import yumbased --from utils import _, errorprint, MDError -+from utils import MDError +-import gzip ++import deltarpm + from utils import MDError class DeltaRPMPackage: - """each drpm is one object, you pass it a drpm file -diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py -index 7ed79e2..285e54d 100644 ---- a/createrepo/readMetadata.py -+++ b/createrepo/readMetadata.py -@@ -210,10 +210,10 @@ if __name__ == "__main__": - - idx = MetadataIndex(cwd, opts) - for fn in idx.basenodes.keys(): -- a,b,c, = idx.getNodes(fn) -- a.serialize() -- b.serialize() -- c.serialize() -- idx.freeNodes(fn) -+ a,b,c, = idx.getNodes(fn) -+ a.serialize() -+ b.serialize() -+ c.serialize() -+ idx.freeNodes(fn) - - -diff --git a/createrepo/utils.py b/createrepo/utils.py -index 3f6414b..fb23964 100644 ---- a/createrepo/utils.py -+++ b/createrepo/utils.py -@@ -76,34 +76,34 @@ def returnFD(filename): - raise MDError, "Error opening file" - return fdno - --def checkAndMakeDir(dir): -+def checkAndMakeDir(directory): - """ -- check out the dir and make it, if possible, return 1 if done, else return 0 -+ check out the directory and make it, if possible, return 1 if done, else return 0 - """ -- if os.path.exists(dir): -- if not os.path.isdir(dir): -- #errorprint(_('%s is not a dir') % dir) -+ if os.path.exists(directory): -+ if not os.path.isdir(directory): -+ #errorprint(_('%s is not a dir') % directory) - result = False - else: -- if not os.access(dir, os.W_OK): -- #errorprint(_('%s is not writable') % dir) -+ if not os.access(directory, os.W_OK): -+ #errorprint(_('%s is not writable') % directory) - result = False - else: - result = True - else: - try: -- os.mkdir(dir) -+ os.mkdir(directory) - except OSError, e: -- #errorprint(_('Error creating dir %s: %s') % (dir, e)) -+ #errorprint(_('Error creating dir %s: %s') % (directory, e)) - result = False - else: - result = True - return result - --def checksum_and_rename(fn_path): -+def checksum_and_rename(fn_path, sumtype='sha256'): - """checksum the file rename the file to contain the checksum as a prefix - return the new filename""" -- csum = misc.checksum('sha256', fn_path) -+ csum = misc.checksum(sumtype, fn_path) - fn = os.path.basename(fn_path) - fndir = os.path.dirname(fn_path) - csum_fn = csum + '-' + fn -diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py -index b820fd8..f04f4fb 100644 ---- a/createrepo/yumbased.py -+++ b/createrepo/yumbased.py -@@ -18,21 +18,19 @@ - import os - import rpm - import types --import re - - from yum.packages import YumLocalPackage - from yum.Errors import * - from yum import misc --from yum.sqlutils import executeSQL --from rpmUtils.transaction import initReadOnlyTransaction --from rpmUtils.miscutils import flagToString, stringToVersion - import utils - import tempfile - - class CreateRepoPackage(YumLocalPackage): -- def __init__(self, ts, package): -+ def __init__(self, ts, package, sumtype=None): - YumLocalPackage.__init__(self, ts, package) +@@ -44,17 +44,16 @@ class DeltaRPMPackage: + os.lseek(fd, 0, 0) + fo = os.fdopen(fd, 'rb') + self.csum = misc.checksum(self.csum_type, fo) +- fo.seek(int(self.po.hdrend)) +- self._getOldInfo(fo) + del fo + del fd ++ self._getDRPMInfo(os.path.join(basedir, filename)) + + def _stringToNEVR(self, string): + i = string.rfind("-", 0, string.rfind("-")-1) + name = string[:i] + (epoch, ver, rel) = self._stringToVersion(string[i+1:]) + return (name, epoch, ver, rel) - -+ if sumtype: -+ self.checksum_type = sumtype -+ - def _do_checksum(self): - """return a checksum for a package: - - check if the checksum cache is enabled -@@ -60,7 +58,7 @@ class CreateRepoPackage(YumLocalPackage): - if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType: - t.append("".join(self.hdr[rpm.RPMTAG_HDRID])) - -- kcsum = misc.Checksums() -+ kcsum = misc.Checksums(checksums=[self.checksum_type]) - kcsum.update("".join(t)) - key = kcsum.hexdigest() - -diff --git a/docs/createrepo.8 b/docs/createrepo.8 -index e7cce4f..44318ef 100644 ---- a/docs/createrepo.8 -+++ b/docs/createrepo.8 -@@ -71,20 +71,29 @@ Generate sqlite databases for use with yum. - Ignore symlinks of packages - .IP "\fB\-s --checksum\fP" - Choose the checksum type used in repomd.xml and for packages in the metadata. --The default is now sha256, the older default was sha which is actually sha1. --.IP "\fB\-profile\fP" -+The default is now "sha256" (if python has hashlib). The older default was -+"sha", which is actually "sha1", however explicitly using "sha1" doesn't work -+on older (3.0.x) versions of yum, you need to specify "sha". -+.IP "\fB\--profile\fP" - Output time based profiling information. - .IP "\fB\--changelog-limit\fP CHANGELOG_LIMIT" - Only import the last N changelog entries, from each rpm, into the metadata - .IP "\fB\--unique-md-filenames\fP" - Include the file's checksum in the metadata filename, helps HTTP caching --.IP "\--distro\fP" -+.IP "\fB\--distro\fP" - Specify distro tags. Can be specified more than once. Optional syntax specifying a - cpeid(http://cpe.mitre.org/) --distro=cpeid,distrotag --.IP "\--content\fP" -+.IP "\fB\--content\fP" - Specify keyword/tags about the content of the repository. Can be specified more than once. --.IP "\--revision\fP" -+.IP "\fB\--revision\fP" - Arbitrary string for a repository revision. -+.IP "\fB\--deltas\fP" -+Tells createrepo to generate deltarpms and the delta metadata -+.IP "\fB\--oldpackagedirs\fP PATH" -+paths to look for older pkgs to delta against. Can be specified multiple times -+.IP "\fB\--num-deltas\fP int" -+the number of older versions to make deltas against. Defaults to 1 -+ - - .SH "EXAMPLES" - Here is an example of a repository with a groups file. Note that the -diff --git a/genpkgmetadata.py b/genpkgmetadata.py -index 041bc02..684417b 100755 ---- a/genpkgmetadata.py -+++ b/genpkgmetadata.py -@@ -16,7 +16,8 @@ - # along with this program; if not, write to the Free Software - # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - # Copyright 2004 Duke University --# Portions Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org -+# Portions Copyright 2009 Red Hat, Inc - -+# written by seth vidal skvidal at fedoraproject.org - - import os - import sys -@@ -27,78 +28,83 @@ import time - import createrepo - from createrepo import MDError - from createrepo.utils import errorprint, _ -+import yum.misc - - --def parseArgs(args, conf): -+def parse_args(args, conf): - """ - Parse the command line args. return a config object. - Sanity check all the things being passed in. - """ - -+ _def = yum.misc._default_checksums[0] -+ _avail = yum.misc._available_checksums - parser = OptionParser(version = "createrepo %s" % createrepo.__version__) - # query options - parser.add_option("-q", "--quiet", default=False, action="store_true", -- help="output nothing except for serious errors") -+ help="output nothing except for serious errors") - parser.add_option("-v", "--verbose", default=False, action="store_true", -- help="output more debugging info.") -+ help="output more debugging info.") - parser.add_option("--profile", default=False, action="store_true", -- help="output timing/profile info.") -+ help="output timing/profile info.") - parser.add_option("-x", "--excludes", default=[], action="append", -- help="files to exclude") -+ help="files to exclude") - parser.add_option("--basedir", default=os.getcwd(), -- help="basedir for path to directories") -+ help="basedir for path to directories") - parser.add_option("-u", "--baseurl", default=None, -- help="baseurl to append on all files") -+ help="baseurl to append on all files") - parser.add_option("-g", "--groupfile", default=None, -- help="path to groupfile to include in metadata") -- parser.add_option("-s", "--checksum", default="sha256", dest='sumtype', -- help="Deprecated, ignore") -+ help="path to groupfile to include in metadata") -+ parser.add_option("-s", "--checksum", default=_def, dest='sumtype', -+ help="specify the checksum type to use (default: %s)" % _def) - parser.add_option("-p", "--pretty", default=False, action="store_true", -- help="make sure all xml generated is formatted") -+ help="make sure all xml generated is formatted") - parser.add_option("-c", "--cachedir", default=None, -- help="set path to cache dir") -+ help="set path to cache dir") - parser.add_option("-C", "--checkts", default=False, action="store_true", -- help="check timestamps on files vs the metadata to see if we need to update") -+ help="check timestamps on files vs the metadata to see" \ -+ "if we need to update") - parser.add_option("-d", "--database", default=False, action="store_true", -- help="create sqlite database files") -+ help="create sqlite database files") - # temporarily disabled -- #parser.add_option("--database-only", default=False, action="store_true", dest='database_only', -- # help="Only make the sqlite databases - does not work with --update, yet") -+ #parser.add_option("--database-only", default=False, action="store_true", -+ # dest='database_only', -+ # help="Only make the sqlite databases - does not work with --update, yet") - parser.add_option("--update", default=False, action="store_true", -- help="use the existing repodata to speed up creation of new") -+ help="use the existing repodata to speed up creation of new") - parser.add_option("--update-md-path", default=None, dest='update_md_path', -- help="use the existing repodata for --update from this path") -- parser.add_option("--skip-stat", dest='skip_stat', default=False, action="store_true", -- help="skip the stat() call on a --update, assumes if the file" \ -- "name is the same then the file is still the same" \ -- "(only use this if you're fairly trusting or gullible)" ) -+ help="use the existing repodata for --update from this path") -+ parser.add_option("--skip-stat", dest='skip_stat', default=False, -+ help="skip the stat() call on a --update, assumes if the file" \ -+ "name is the same then the file is still the same" \ -+ "(only use this if you're fairly trusting or gullible)", -+ action="store_true") - parser.add_option("--split", default=False, action="store_true", -- help="generate split media") -+ help="generate split media") - parser.add_option("-i", "--pkglist", default=None, -- help="use only the files listed in this file from the directory specified") -+ help="use only the files listed in this file from the" \ -+ "directory specified") - parser.add_option("-o", "--outputdir", default=None, -- help=" = optional directory to output to") -+ help=" = optional directory to output to") - parser.add_option("-S", "--skip-symlinks", dest="skip_symlinks", -- default=False, action="store_true", -- help="ignore symlinks of packages") -+ default=False, action="store_true", help="ignore symlinks of packages") - parser.add_option("--changelog-limit", dest="changelog_limit", -- default=None, help="only import the last N changelog entries") -+ default=None, help="only import the last N changelog entries") - parser.add_option("--unique-md-filenames", dest="unique_md_filenames", -- default=False, action="store_true", -- help="include the file's checksum in the filename, helps" \ -- "with proxies") -+ help="include the file's checksum in the filename,helps with proxies", -+ default=False, action="store_true") - parser.add_option("--distro", default=[], action="append", -- help="distro tag and optional cpeid: --distro 'cpeid,textname'") -- parser.add_option("--content", default=[], dest='content_tags', action="append", -- help="tags for the content in the repository") -+ help="distro tag and optional cpeid: --distro" "'cpeid,textname'") -+ parser.add_option("--content", default=[], dest='content_tags', -+ action="append", help="tags for the content in the repository") - parser.add_option("--revision", default=None, -- help="user-specified revision for this repository") -+ help="user-specified revision for this repository") - parser.add_option("--deltas", default=False, action="store_true", -- help="create delta rpms and metadata") -+ help="create delta rpms and metadata") - parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", -- action="append", help="paths to look for older pkgs to delta against") -+ action="append", help="paths to look for older pkgs to delta against") - parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', -- help="the number of older versions to make deltas against") -+ help="the number of older versions to make deltas against") - - - (opts, argsleft) = parser.parse_args(args) -@@ -115,16 +121,25 @@ def parseArgs(args, conf): - else: - directories = argsleft - -+ if opts.sumtype == 'sha1': -+ errorprint(_('Warning: It is more compatible to use sha instead of sha1')) + -+ if opts.sumtype != 'sha' and opts.sumtype not in _avail: -+ errorprint(_('Error: Checksum %s not available (sha, %s)') % -+ (opts.sumtype, ", ".join(sorted(_avail)))) -+ sys.exit(1) -+ - if opts.split and opts.checkts: - errorprint(_('--split and --checkts options are mutually exclusive')) - sys.exit(1) - - -- # let's switch over to using the conf object - put all the opts options into it -+ # let's switch over to using the conf object - put all the opts into it - for opt in parser.option_list: - if opt.dest is None: # this is fairly silly - continue -- if getattr(opts, opt.dest) is None: # if it's not set, take the default from the base class -+ # if it's not set, take the default from the base class -+ if getattr(opts, opt.dest) is None: - continue - setattr(conf, opt.dest, getattr(opts, opt.dest)) - -@@ -136,7 +151,7 @@ def parseArgs(args, conf): - - for spec in opts.distro: - if spec.find(',') == -1: -- conf.distro_tags.append((None,spec)) -+ conf.distro_tags.append((None, spec)) - else: - splitspec = spec.split(',') - conf.distro_tags.append((splitspec[0], splitspec[1])) -@@ -159,31 +174,38 @@ def parseArgs(args, conf): - return conf - - class MDCallBack(object): -+ """cli callback object for createrepo""" - def errorlog(self, thing): -+ """error log output""" - print >> sys.stderr, thing + def _getLength(self, in_data): + length = 0 + for val in in_data: +@@ -62,27 +61,12 @@ class DeltaRPMPackage: + length += ord(val) + return length - def log(self, thing): -+ """log output""" - print thing - - def progress(self, item, current, total): -+ """progress bar""" - beg = "%*d/%d - " % (len(str(total)), current, total) - left = 80 - len(beg) - sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) - sys.stdout.flush() +- def _getOldInfo(self, fo): +- try: +- compobj = gzip.GzipFile("", "rb", 9, fo) +- except: +- raise zlibError("Data not stored in gzip format") +- +- if compobj.read(4)[:3] != "DLT": +- raise Exception("Not a deltarpm") ++ def _getDRPMInfo(self, filename): ++ d = deltarpm.readDeltaRPM(filename) ++ self.oldnevrstring = d['old_nevr'] ++ self.oldnevr = self._stringToNEVR(d['old_nevr']) ++ self.sequence = d['seq'] - def main(args): -+ """createrepo from cli main flow""" - start_st = time.time() - conf = createrepo.MetaDataConfig() -- conf = parseArgs(args, conf) -+ conf = parse_args(args, conf) - if conf.profile: - print ('start time: %0.3f' % (time.time() - start_st)) - - mid_st = time.time() - try: - if conf.split: -- mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) -+ mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, -+ callback=MDCallBack()) - else: -- mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) -+ mdgen = createrepo.MetaDataGenerator(config_obj=conf, -+ callback=MDCallBack()) - if mdgen.checkTimeStamps(): - if mdgen.conf.verbose: - print _('repo is up to date') -@@ -200,14 +222,14 @@ def main(args): - mdgen.doRepoMetadata() - if conf.profile: - print ('rm time: %0.3f' % (time.time() - rm_st)) -- fm_st = time.time() -+ fm_st = time.time() - mdgen.doFinalMove() - if conf.profile: - print ('fm time: %0.3f' % (time.time() - fm_st)) - - -- except MDError, e: -- errorprint(_('%s') % e) -+ except MDError, errormsg: -+ errorprint(_('%s') % errormsg) - sys.exit(1) - - -diff --git a/modifyrepo.py b/modifyrepo.py -index 1efe966..bbf938d 100755 ---- a/modifyrepo.py -+++ b/modifyrepo.py -@@ -24,11 +24,12 @@ - import os - import sys - --from createrepo.utils import checksum_and_rename, GzipFile -+from createrepo.utils import checksum_and_rename, GzipFile, MDError - from yum.misc import checksum - - from xml.dom import minidom - -+ - class RepoMetadata: - - def __init__(self, repo): -@@ -37,11 +38,13 @@ class RepoMetadata: - self.repomdxml = os.path.join(self.repodir, 'repomd.xml') - self.checksum_type = 'sha256' - if not os.path.exists(self.repomdxml): -- raise Exception('%s not found' % self.repomdxml) -+ raise MDError, '%s not found' % self.repomdxml - self.doc = minidom.parse(self.repomdxml) - -- def _insert_element(self, parent, name, attrs={}, text=None): -+ def _insert_element(self, parent, name, attrs=None, text=None): - child = self.doc.createElement(name) -+ if not attrs: -+ attrs = {} - for item in attrs.items(): - child.setAttribute(item[0], item[1]) - if text: -@@ -57,7 +60,7 @@ class RepoMetadata: - """ - md = None - if not metadata: -- raise Exception('metadata cannot be None') -+ raise MDError, 'metadata cannot be None' - if isinstance(metadata, minidom.Document): - md = metadata.toxml() - mdname = 'updateinfo.xml' -@@ -71,9 +74,9 @@ class RepoMetadata: - oldmd.close() - mdname = os.path.basename(metadata) - else: -- raise Exception('%s not found' % metadata) -+ raise MDError, '%s not found' % metadata - else: -- raise Exception('invalid metadata type') -+ raise MDError, 'invalid metadata type' - - ## Compress the metadata and move it into the repodata - if not mdname.endswith('.gz'): -@@ -88,7 +91,7 @@ class RepoMetadata: - open_csum = checksum(self.checksum_type, metadata) - - -- csum, destmd = checksum_and_rename(destmd) -+ csum, destmd = checksum_and_rename(destmd, self.checksum_type) - base_destmd = os.path.basename(destmd) - - -@@ -106,13 +109,15 @@ class RepoMetadata: - self._insert_element(data, 'location', - attrs={ 'href' : 'repodata/' + base_destmd }) - data.appendChild(self.doc.createTextNode("\n ")) -- self._insert_element(data, 'checksum', attrs={ 'type' : self.checksum_type }, -+ self._insert_element(data, 'checksum', -+ attrs={ 'type' : self.checksum_type }, - text=csum) - data.appendChild(self.doc.createTextNode("\n ")) - self._insert_element(data, 'timestamp', - text=str(os.stat(destmd).st_mtime)) - data.appendChild(self.doc.createTextNode("\n ")) -- self._insert_element(data, 'open-checksum', attrs={ 'type' : self.checksum_type }, -+ self._insert_element(data, 'open-checksum', -+ attrs={ 'type' : self.checksum_type }, - text=open_csum) - - data.appendChild(self.doc.createTextNode("\n ")) -@@ -138,12 +143,12 @@ if __name__ == '__main__': - sys.exit() - try: - repomd = RepoMetadata(sys.argv[2]) -- except Exception, e: -+ except MDError, e: - print "Could not access repository: %s" % str(e) - sys.exit(1) - try: - repomd.add(sys.argv[1]) -- except Exception, e: -+ except MDError, e: - print "Could not add metadata from file %s: %s" % (sys.argv[1], str(e)) - sys.exit(1) - +- nevr_length = self._getLength(compobj.read(4)) +- nevr = compobj.read(nevr_length).strip("\x00") +- seq_length = self._getLength(compobj.read(4)) +- seq = compobj.read(seq_length) +- hex_seq = "" +- for char in seq: +- hex_seq += str("%02x" % ord(char)) +- self.oldnevrstring = nevr +- self.oldnevr = self._stringToNEVR(nevr) +- self.sequence = hex_seq +- compobj.close() +- + def _stringToVersion(self, strng): + i = strng.find(':') + if i != -1: diff --git a/createrepo.spec b/createrepo.spec index 106685f..ee3b507 100644 --- a/createrepo.spec +++ b/createrepo.spec @@ -3,12 +3,12 @@ Summary: Creates a common metadata repository Name: createrepo Version: 0.9.8 -Release: 2%{?dist} +Release: 3%{?dist} License: GPLv2 Group: System Environment/Base Source: %{name}-%{version}.tar.gz Patch0: ten-changelog-limit.patch -Patch1: createrepo-drpm.patch +Patch1: createrepo-head.patch URL: http://createrepo.baseurl.org/ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArchitectures: noarch @@ -46,6 +46,9 @@ rm -rf $RPM_BUILD_ROOT %{python_sitelib}/createrepo %changelog +* Tue Dec 22 2009 Seth Vidal - 0.9.8-3 +- patch to latest HEAD from upstream + * Thu Sep 3 2009 Seth Vidal - 0.9.8-2 - add drpm patch from https://bugzilla.redhat.com/show_bug.cgi?id=518658