From 50066e95537646e8e40d9b7479e71886d1e38f93 Mon Sep 17 00:00:00 2001 From: Seth Vidal Date: May 05 2009 14:26:13 +0000 Subject: update to latest head patch to see if I've fixed all the random issues for the -s sha1/sha users --- diff --git a/createrepo-head.patch b/createrepo-head.patch index 905a7ec..ac223cd 100644 --- a/createrepo-head.patch +++ b/createrepo-head.patch @@ -1,7 +1,39 @@ +diff --git a/Makefile b/Makefile +index facbc5b..c821ae4 100644 +--- a/Makefile ++++ b/Makefile +@@ -95,6 +95,12 @@ distclean: clean + $(MAKE) -C $$subdir distclean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ + done + ++pylint: ++ @pylint --rcfile=test/createrepo-pylintrc *.py createrepo ++ ++pylint-short: ++ @pylint -r n --rcfile=test/createrepo-pylintrc *.py createrepo ++ + mostlyclean: + $(MAKE) clean + diff --git a/createrepo/__init__.py b/createrepo/__init__.py -index 2a880a8..00c14da 100644 +index 2a880a8..26cdff7 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py +@@ -11,12 +11,12 @@ + # You should have received a copy of the GNU General Public License + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +-# Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org ++# Copyright 2009 Red Hat, Inc - ++# written by seth vidal skvidal at fedoraproject.org + + import os + import sys + import libxml2 +-import string + import fnmatch + import time + import yumbased @@ -24,6 +24,7 @@ import shutil from bz2 import BZ2File from urlgrabber import grabber @@ -10,7 +42,24 @@ index 2a880a8..00c14da 100644 from yum import misc, Errors, to_unicode from yum.sqlutils import executeSQL -@@ -71,6 +72,7 @@ class MetaDataConfig(object): +@@ -43,7 +44,8 @@ try: + except ImportError: + pass + +-from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, checksum_and_rename ++from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \ ++ checksum_and_rename + import deltarpms + + __version__ = '0.9.7' +@@ -66,11 +68,13 @@ class MetaDataConfig(object): + self.split = False + self.update = False + self.deltas = False # do the deltarpm thing +- self.deltadir = None # where to put the .drpms - defaults to 'drpms' inside 'repodata' ++ # where to put the .drpms - defaults to 'drpms' inside 'repodata' ++ self.deltadir = None + self.delta_relative = 'drpms/' self.oldpackage_paths = [] # where to look for the old packages - self.deltafile = 'prestodelta.xml.gz' self.num_deltas = 1 # number of older versions to delta (max) @@ -18,7 +67,264 @@ index 2a880a8..00c14da 100644 self.update_md_path = None self.skip_stat = False self.database = False -@@ -584,6 +586,13 @@ class MetaDataGenerator: +@@ -153,10 +157,12 @@ class MetaDataGenerator: + self.conf.basedir = os.path.realpath(self.conf.basedir) + self.conf.relative_dir = self.conf.directory + +- self.package_dir = os.path.join(self.conf.basedir, self.conf.relative_dir) ++ self.package_dir = os.path.join(self.conf.basedir, ++ self.conf.relative_dir) + + if not self.conf.outputdir: +- self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir) ++ self.conf.outputdir = os.path.join(self.conf.basedir, ++ self.conf.relative_dir) + + def _test_setup_dirs(self): + # start the sanity/stupidity checks +@@ -187,7 +193,8 @@ class MetaDataGenerator: + raise MDError, _('Cannot create/verify %s') % temp_final + + if self.conf.deltas: +- temp_delta = os.path.join(self.conf.outputdir, self.conf.delta_relative) ++ temp_delta = os.path.join(self.conf.outputdir, ++ self.conf.delta_relative) + if not checkAndMakeDir(temp_delta): + raise MDError, _('Cannot create/verify %s') % temp_delta + self.conf.deltadir = temp_delta +@@ -202,7 +209,8 @@ class MetaDataGenerator: + direcs.append('deltadir') + + for direc in direcs: +- filepath = os.path.join(self.conf.outputdir, getattr(self.conf, direc)) ++ filepath = os.path.join(self.conf.outputdir, getattr(self.conf, ++ direc)) + if os.path.exists(filepath): + if not os.access(filepath, os.W_OK): + raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath +@@ -227,7 +235,7 @@ class MetaDataGenerator: + if self.conf.cachedir: + a = self.conf.cachedir + if not os.path.isabs(a): +- a = os.path.join(self.conf.outputdir ,a) ++ a = os.path.join(self.conf.outputdir, a) + if not checkAndMakeDir(a): + raise MDError, _('Error: cannot open/write to cache dir %s' % a) + +@@ -264,7 +272,7 @@ class MetaDataGenerator: + elif fn[-extlen:].lower() == '%s' % (ext): + relativepath = dirname.replace(startdir, "", 1) + relativepath = relativepath.lstrip("/") +- filelist.append(os.path.join(relativepath,fn)) ++ filelist.append(os.path.join(relativepath, fn)) + + filelist = [] + startdir = directory + '/' +@@ -276,8 +284,8 @@ class MetaDataGenerator: + errorprint(thing) + + def checkTimeStamps(self): +- """check the timestamp of our target dir. If it is not newer than the repodata +- return False, else True""" ++ """check the timestamp of our target dir. If it is not newer than ++ the repodata return False, else True""" + if self.conf.checkts: + dn = os.path.join(self.conf.basedir, self.conf.directory) + files = self.getFileList(dn, '.rpm') +@@ -295,14 +303,14 @@ class MetaDataGenerator: + + def trimRpms(self, files): + badrpms = [] +- for file in files: ++ for rpm_file in files: + for glob in self.conf.excludes: +- if fnmatch.fnmatch(file, glob): +- if file not in badrpms: +- badrpms.append(file) +- for file in badrpms: +- if file in files: +- files.remove(file) ++ if fnmatch.fnmatch(rpm_file, glob): ++ if rpm_file not in badrpms: ++ badrpms.append(rpm_file) ++ for rpm_file in badrpms: ++ if rpm_file in files: ++ files.remove(rpm_file) + return files + + def _setup_old_metadata_lookup(self): +@@ -369,34 +377,40 @@ class MetaDataGenerator: + + def _setupPrimary(self): + # setup the primary metadata file +- primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.primaryfile) ++ primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, ++ self.conf.primaryfile) + fo = _gzipOpen(primaryfilepath, 'w') + fo.write('\n') +- fo.write('' % ++ fo.write('' % + self.pkgcount) + return fo + + def _setupFilelists(self): + # setup the filelist file +- filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.filelistsfile) ++ filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, ++ self.conf.filelistsfile) + fo = _gzipOpen(filelistpath, 'w') + fo.write('\n') +- fo.write('' % +- self.pkgcount) ++ fo.write('' % self.pkgcount) + return fo + + def _setupOther(self): + # setup the other file +- otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.otherfile) ++ otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, ++ self.conf.otherfile) + fo = _gzipOpen(otherfilepath, 'w') + fo.write('\n') +- fo.write('' % ++ fo.write('' % + self.pkgcount) + return fo + + def _setupDelta(self): + # setup the other file +- deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, self.conf.deltafile) ++ deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, ++ self.conf.deltafile) + fo = _gzipOpen(deltafilepath, 'w') + fo.write('\n') + fo.write('\n') +@@ -405,7 +419,6 @@ class MetaDataGenerator: + + def read_in_package(self, rpmfile, pkgpath=None, reldir=None): + """rpmfile == relative path to file from self.packge_dir""" +- remote_package = False + baseurl = self.conf.baseurl + + if not pkgpath: +@@ -415,7 +428,6 @@ class MetaDataGenerator: + raise MDError, "Blank filename passed in, skipping" + + if rpmfile.find("://") != -1: +- remote_package = True + + if not hasattr(self, 'tempdir'): + self.tempdir = tempfile.mkdtemp() +@@ -425,18 +437,20 @@ class MetaDataGenerator: + reldir = self.tempdir + dest = os.path.join(self.tempdir, pkgname) + if not self.conf.quiet: +- self.callback.log('\nDownloading %s' % rpmfile) ++ self.callback.log('\nDownloading %s' % rpmfile) + try: + rpmfile = self.grabber.urlgrab(rpmfile, dest) + except grabber.URLGrabError, e: +- raise MDError, "Unable to retrieve remote package %s: %s" %(rpmfile, e) ++ raise MDError, "Unable to retrieve remote package %s: %s" % ( ++ rpmfile, e) + + + else: + rpmfile = '%s/%s' % (pkgpath, rpmfile) + + try: +- po = yumbased.CreateRepoPackage(self.ts, rpmfile) ++ po = yumbased.CreateRepoPackage(self.ts, rpmfile, ++ sumtype=self.conf.sumtype) + except Errors.MiscError, e: + raise MDError, "Unable to open package: %s" % e + # external info we need +@@ -449,7 +463,8 @@ class MetaDataGenerator: + self.rpmlib_reqs[r] = 1 + + if po.checksum in (None, ""): +- raise MDError, "No Package ID found for package %s, not going to add it" % po ++ raise MDError, "No Package ID found for package %s, not going to" \ ++ " add it" % po + + return po + +@@ -459,9 +474,9 @@ class MetaDataGenerator: + pkglist = self.conf.pkglist + + if not pkgpath: +- directory=self.conf.directory ++ directory = self.conf.directory + else: +- directory=pkgpath ++ directory = pkgpath + + for pkg in pkglist: + self.current_pkg += 1 +@@ -492,13 +507,15 @@ class MetaDataGenerator: + if not isinstance(pkg, YumAvailablePackage): + + try: +- po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir) ++ po = self.read_in_package(pkg, pkgpath=pkgpath, ++ reldir=reldir) + except MDError, e: + # need to say something here + self.callback.errorlog("\nError %s: %s\n" % (pkg, e)) + continue + # we can use deltas: +- self._do_delta_rpm_package(po) ++ if self.conf.deltas: ++ self._do_delta_rpm_package(po) + + else: + po = pkg +@@ -513,12 +530,13 @@ class MetaDataGenerator: + clog_limit=self.conf.changelog_limit)) + else: + if self.conf.verbose: +- self.callback.log(_("Using data from old metadata for %s") % pkg) ++ self.callback.log(_("Using data from old metadata for %s") ++ % pkg) + (primarynode, filenode, othernode) = nodes + +- for node, outfile in ((primarynode,self.primaryfile), +- (filenode,self.flfile), +- (othernode,self.otherfile)): ++ for node, outfile in ((primarynode, self.primaryfile), ++ (filenode, self.flfile), ++ (othernode, self.otherfile)): + if node is None: + break + output = node.serialize('UTF-8', self.conf.pretty) +@@ -526,17 +544,19 @@ class MetaDataGenerator: + outfile.write(output) + else: + if self.conf.verbose: +- self.callback.log(_("empty serialize on write to %s in %s") % (outfile, pkg)) ++ self.callback.log(_("empty serialize on write to" \ ++ "%s in %s") % (outfile, pkg)) + outfile.write('\n') + + self.oldData.freeNodes(pkg) + #FIXME - if we're in update and we have deltas enabled +- # check the presto data for this pkg and write its info back out +- # to our deltafile ++ # check the presto data for this pkg and write its info back out ++ # to our deltafile + + if not self.conf.quiet: + if self.conf.verbose: +- self.callback.log('%d/%d - %s' % (self.current_pkg, self.pkgcount, pkg)) ++ self.callback.log('%d/%d - %s' % (self.current_pkg, ++ self.pkgcount, pkg)) + else: + self.callback.progress(pkg, self.current_pkg, self.pkgcount) + +@@ -584,11 +604,18 @@ class MetaDataGenerator: """makes the drpms, if possible, for this package object. returns the presto/delta xml metadata as a string """ @@ -31,8 +337,26 @@ index 2a880a8..00c14da 100644 + return # generate a list of all the potential 'old rpms' - opd = self._get_old_package_dict() # yes I could make this a property but -@@ -616,7 +625,8 @@ class MetaDataGenerator: +- opd = self._get_old_package_dict() # yes I could make this a property but +- +- # for each of our old_package_paths - make a drpm from the newest of that pkg ++ opd = self._get_old_package_dict() ++ # for each of our old_package_paths - ++ # make a drpm from the newest of that pkg + # get list of potential candidates which are likely to match + for d in self.conf.oldpackage_paths: + pot_cand = [] +@@ -599,7 +626,8 @@ class MetaDataGenerator: + candidates = [] + for fn in pot_cand: + try: +- thispo = yumbased.CreateRepoPackage(self.ts, fn) ++ thispo = yumbased.CreateRepoPackage(self.ts, fn, ++ sumtype=self.conf.sumtype) + except Errors.MiscError, e: + continue + if (thispo.name, thispo.arch) != (pkg.name, pkg.arch): +@@ -616,7 +644,8 @@ class MetaDataGenerator: for delta_p in candidates[0:self.conf.num_deltas]: #make drpm of pkg and delta_p drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir) @@ -42,33 +366,49 @@ index 2a880a8..00c14da 100644 delta_p, pkg, drpmfn)) def _get_old_package_dict(self): -@@ -627,6 +637,11 @@ class MetaDataGenerator: +@@ -627,6 +656,12 @@ class MetaDataGenerator: opl = [] for d in self.conf.oldpackage_paths: for f in self.getFileList(d, 'rpm'): + fp = d + '/' + f -+ if int(os.stat(fp)[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: ++ fpstat = os.stat(fp) ++ if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: + self.callback.log("Skipping %s package " \ -+ "that is > max_delta_rpm_size" % pkg) ++ "that is > max_delta_rpm_size" % f) + continue if not self._old_package_dict.has_key(d): self._old_package_dict[d] = [] self._old_package_dict[d].append(d + '/' + f) -@@ -652,10 +667,10 @@ class MetaDataGenerator: +@@ -644,18 +679,21 @@ class MetaDataGenerator: + targets = {} + result = u'' + for drpm_fn in self.getFileList(self.conf.deltadir, 'drpm'): +- drpm_rel_fn = os.path.normpath(self.conf.delta_relative + '/' + drpm_fn) # this is annoying +- drpm_po = yumbased.CreateRepoPackage(self.ts, self.conf.deltadir + '/' + drpm_fn) ++ drpm_rel_fn = os.path.normpath(self.conf.delta_relative + ++ '/' + drpm_fn) # this is annoying ++ drpm_po = yumbased.CreateRepoPackage(self.ts, ++ self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype) + +- drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, drpm_rel_fn) ++ drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, ++ drpm_rel_fn) + if not targets.has_key(drpm_po.pkgtup): targets[drpm_po.pkgtup] = u'' targets[drpm_po.pkgtup] += drpm.xml_dump_metadata() - for (n,e,v,r,a) in targets.keys(): - result += """ \n""" % ( - n,e,v,r,a) +- n,e,v,r,a) - for src in targets[(n,e,v,r,a)]: -+ for src in targets[(n,a,e,v,r)]: ++ for (n, a, e, v, r) in targets.keys(): ++ result += """ \n""" % ( ++ n, e, v, r, a) ++ for src in targets[(n, a, e, v, r)]: result += src result += """ \n""" -@@ -695,7 +710,7 @@ class MetaDataGenerator: +@@ -695,7 +733,7 @@ class MetaDataGenerator: if self.conf.unique_md_filenames: @@ -77,8 +417,37 @@ index 2a880a8..00c14da 100644 sfile = os.path.basename(outfn) else: if compress: -@@ -760,7 +775,7 @@ class MetaDataGenerator: - repoid='garbageid' +@@ -721,12 +759,13 @@ class MetaDataGenerator: + timestamp = data.newChild(None, 'timestamp', str(timest)) + + # add the random stuff +- for (k,v) in attribs.items(): ++ for (k, v) in attribs.items(): + data.newChild(None, k, str(v)) + + + def doRepoMetadata(self): +- """wrapper to generate the repomd.xml file that stores the info on the other files""" ++ """wrapper to generate the repomd.xml file that stores the info ++ on the other files""" + repodoc = libxml2.newDoc("1.0") + reporoot = repodoc.newChild(None, "repomd", None) + repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) +@@ -740,7 +779,7 @@ class MetaDataGenerator: + tags = reporoot.newChild(None, 'tags', None) + for item in self.conf.content_tags: + c_tags = tags.newChild(None, 'content', item) +- for (cpeid,item) in self.conf.distro_tags: ++ for (cpeid, item) in self.conf.distro_tags: + d_tags = tags.newChild(None, 'distro', item) + if cpeid: + d_tags.newProp('cpeid', cpeid) +@@ -757,10 +796,10 @@ class MetaDataGenerator: + (self.conf.filelistsfile, 'filelists'), + (self.conf.primaryfile, 'primary')] + db_workfiles = [] +- repoid='garbageid' ++ repoid = 'garbageid' if self.conf.deltas: - workfiles.append((self.conf.deltafile, 'deltainfo')) @@ -86,7 +455,28 @@ index 2a880a8..00c14da 100644 if self.conf.database: if not self.conf.quiet: self.callback.log('Generating sqlite DBs') try: -@@ -793,54 +808,54 @@ class MetaDataGenerator: +@@ -769,8 +808,8 @@ class MetaDataGenerator: + dbversion = '9' + rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None) + +- for (file, ftype) in workfiles: +- complete_path = os.path.join(repopath, file) ++ for (rpm_file, ftype) in workfiles: ++ complete_path = os.path.join(repopath, rpm_file) + + zfo = _gzipOpen(complete_path) + uncsum = misc.checksum(sumtype, zfo) +@@ -783,7 +822,8 @@ class MetaDataGenerator: + + if self.conf.database: + if self.conf.verbose: +- self.callback.log("Starting %s db creation: %s" % (ftype, time.ctime())) ++ self.callback.log("Starting %s db creation: %s" % (ftype, ++ time.ctime())) + + if ftype == 'primary': + rp.getPrimary(complete_path, csum) +@@ -793,54 +833,63 @@ class MetaDataGenerator: elif ftype == 'other': rp.getOtherdata(complete_path, csum) @@ -132,13 +522,16 @@ index 2a880a8..00c14da 100644 + # compress the files + bzipFile(resultpath, result_compressed) + # csum the compressed file -+ db_compressed_sums[ftype] = misc.checksum(sumtype, result_compressed) ++ db_compressed_sums[ftype] = misc.checksum(sumtype, ++ result_compressed) + # remove the uncompressed file + os.unlink(resultpath) + + if self.conf.unique_md_filenames: -+ csum_compressed_name = '%s-%s.bz2' % (db_compressed_sums[ftype], good_name) -+ csum_result_compressed = os.path.join(repopath, csum_compressed_name) ++ csum_compressed_name = '%s-%s.bz2' % ( ++ db_compressed_sums[ftype], good_name) ++ csum_result_compressed = os.path.join(repopath, ++ csum_compressed_name) + os.rename(result_compressed, csum_result_compressed) + result_compressed = csum_result_compressed + compressed_name = csum_compressed_name @@ -173,23 +566,227 @@ index 2a880a8..00c14da 100644 + if self.conf.baseurl is not None: + location.newProp('xml:base', self.conf.baseurl) + -+ location.newProp('href', os.path.join(self.conf.finaldir, compressed_name)) -+ checksum = data.newChild(None, 'checksum', db_compressed_sums[ftype]) ++ location.newProp('href', os.path.join(self.conf.finaldir, ++ compressed_name)) ++ checksum = data.newChild(None, 'checksum', ++ db_compressed_sums[ftype]) + checksum.newProp('type', sumtype) -+ db_tstamp = data.newChild(None, 'timestamp', str(db_timestamp)) -+ unchecksum = data.newChild(None, 'open-checksum', db_csums[ftype]) ++ db_tstamp = data.newChild(None, 'timestamp', ++ str(db_timestamp)) ++ unchecksum = data.newChild(None, 'open-checksum', ++ db_csums[ftype]) + unchecksum.newProp('type', sumtype) -+ database_version = data.newChild(None, 'database_version', dbversion) ++ database_version = data.newChild(None, 'database_version', ++ dbversion) + if self.conf.verbose: -+ self.callback.log("Ending %s db creation: %s" % (ftype, time.ctime())) ++ self.callback.log("Ending %s db creation: %s" % (ftype, ++ time.ctime())) +@@ -857,19 +906,20 @@ class MetaDataGenerator: + location.newProp('xml:base', self.conf.baseurl) + if self.conf.unique_md_filenames: + res_file = '%s-%s.xml.gz' % (csum, ftype) +- orig_file = os.path.join(repopath, file) ++ orig_file = os.path.join(repopath, rpm_file) + dest_file = os.path.join(repopath, res_file) + os.rename(orig_file, dest_file) + + else: +- res_file = file ++ res_file = rpm_file + +- file = res_file ++ rpm_file = res_file + +- location.newProp('href', os.path.join(self.conf.finaldir, file)) ++ location.newProp('href', os.path.join(self.conf.finaldir, rpm_file)) + + +- if not self.conf.quiet and self.conf.database: self.callback.log('Sqlite DBs complete') ++ if not self.conf.quiet and self.conf.database: ++ self.callback.log('Sqlite DBs complete') + + for (fn, ftype) in db_workfiles: + attribs = {'database_version':dbversion} +@@ -883,7 +933,8 @@ class MetaDataGenerator: + + if self.conf.groupfile is not None: + self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot) +- self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, compress=False) ++ self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot, ++ compress=False) + + if self.conf.additional_metadata: + for md_type, mdfile in self.conf.additional_metadata.items(): +@@ -900,7 +951,8 @@ class MetaDataGenerator: + try: + repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) + except: +- self.callback.errorlog(_('Error saving temp file for repomd.xml: %s') % repofilepath) ++ self.callback.errorlog( ++ _('Error saving temp file for repomd.xml: %s') % repofilepath) + raise MDError, 'Could not save temp file: %s' % repofilepath + + del repodoc +@@ -918,8 +970,8 @@ class MetaDataGenerator: + try: + os.rename(output_final_dir, output_old_dir) + except: +- raise MDError, _('Error moving final %s to old dir %s' % (output_final_dir, +- output_old_dir)) ++ raise MDError, _('Error moving final %s to old dir %s' % ( ++ output_final_dir, output_old_dir)) + + output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) + +@@ -930,7 +982,8 @@ class MetaDataGenerator: + os.rename(output_old_dir, output_final_dir) + raise MDError, _('Error moving final metadata into place') + +- for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']: ++ for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', ++ 'groupfile']: + if getattr(self.conf, f): + fn = os.path.basename(getattr(self.conf, f)) + else: +@@ -941,7 +994,8 @@ class MetaDataGenerator: + try: + os.remove(oldfile) + except OSError, e: +- raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e) ++ raise MDError, _( ++ 'Could not remove old metadata file: %s: %s') % (oldfile, e) + + # Move everything else back from olddir (eg. repoview files) + for f in os.listdir(output_old_dir): +@@ -952,7 +1006,8 @@ class MetaDataGenerator: + 'other.xml.gz','filelists.xml.gz'): + os.remove(oldfile) # kill off the old ones + continue +- if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', 'primary.sqlite.bz2'): ++ if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', ++ 'primary.sqlite.bz2'): + os.remove(oldfile) + continue + +@@ -964,7 +1019,8 @@ class MetaDataGenerator: + else: + os.remove(oldfile) + except OSError, e: +- raise MDError, _('Could not remove old metadata file: %s: %s') % (oldfile, e) ++ raise MDError, _( ++ 'Could not remove old metadata file: %s: %s') % (oldfile, e) + else: + try: + os.rename(oldfile, finalfile) +@@ -976,7 +1032,8 @@ class MetaDataGenerator: + try: + os.rmdir(output_old_dir) + except OSError, e: +- self.errorlog(_('Could not remove old metadata dir: %s') % self.conf.olddir) ++ self.errorlog(_('Could not remove old metadata dir: %s') ++ % self.conf.olddir) + self.errorlog(_('Error was %s') % e) + self.errorlog(_('Please clean up this directory manually.')) + +@@ -986,7 +1043,8 @@ class MetaDataGenerator: + try: + self.md_sqlite = MetaDataSqlite(destdir) + except sqlite.OperationalError, e: +- raise MDError, _('Cannot create sqlite databases: %s.\nMaybe you need to clean up a .repodata dir?') % e ++ raise MDError, _('Cannot create sqlite databases: %s.\n'\ ++ 'Maybe you need to clean up a .repodata dir?') % e + + + +@@ -1014,11 +1072,11 @@ class SplitMetaDataGenerator(MetaDataGenerator): + for fn in names: + if os.path.isdir(fn): + continue +- elif string.lower(fn[-extlen:]) == '%s' % (ext): ++ elif fn[-extlen:].lower() == '%s' % (ext): + reldir = os.path.basename(dirname) + if reldir == os.path.basename(directory): + reldir = "" +- arg.append(os.path.join(reldir,fn)) ++ arg.append(os.path.join(reldir, fn)) + + rpmlist = [] + os.path.walk(directory, extension_visitor, rpmlist) +diff --git a/createrepo/deltarpms.py b/createrepo/deltarpms.py +index f201ab7..26a6715 100644 +--- a/createrepo/deltarpms.py ++++ b/createrepo/deltarpms.py +@@ -19,8 +19,7 @@ import os.path + import commands + from yum import misc + import gzip +-import yumbased +-from utils import _, errorprint, MDError ++from utils import MDError + + class DeltaRPMPackage: + """each drpm is one object, you pass it a drpm file +diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py +index 7ed79e2..285e54d 100644 +--- a/createrepo/readMetadata.py ++++ b/createrepo/readMetadata.py +@@ -210,10 +210,10 @@ if __name__ == "__main__": + + idx = MetadataIndex(cwd, opts) + for fn in idx.basenodes.keys(): +- a,b,c, = idx.getNodes(fn) +- a.serialize() +- b.serialize() +- c.serialize() +- idx.freeNodes(fn) ++ a,b,c, = idx.getNodes(fn) ++ a.serialize() ++ b.serialize() ++ c.serialize() ++ idx.freeNodes(fn) + + diff --git a/createrepo/utils.py b/createrepo/utils.py -index 3f6414b..894594c 100644 +index 3f6414b..fb23964 100644 --- a/createrepo/utils.py +++ b/createrepo/utils.py -@@ -100,10 +100,10 @@ def checkAndMakeDir(dir): +@@ -76,34 +76,34 @@ def returnFD(filename): + raise MDError, "Error opening file" + return fdno + +-def checkAndMakeDir(dir): ++def checkAndMakeDir(directory): + """ +- check out the dir and make it, if possible, return 1 if done, else return 0 ++ check out the directory and make it, if possible, return 1 if done, else return 0 + """ +- if os.path.exists(dir): +- if not os.path.isdir(dir): +- #errorprint(_('%s is not a dir') % dir) ++ if os.path.exists(directory): ++ if not os.path.isdir(directory): ++ #errorprint(_('%s is not a dir') % directory) + result = False + else: +- if not os.access(dir, os.W_OK): +- #errorprint(_('%s is not writable') % dir) ++ if not os.access(directory, os.W_OK): ++ #errorprint(_('%s is not writable') % directory) + result = False + else: + result = True + else: + try: +- os.mkdir(dir) ++ os.mkdir(directory) + except OSError, e: +- #errorprint(_('Error creating dir %s: %s') % (dir, e)) ++ #errorprint(_('Error creating dir %s: %s') % (directory, e)) + result = False + else: result = True return result @@ -203,10 +800,36 @@ index 3f6414b..894594c 100644 fndir = os.path.dirname(fn_path) csum_fn = csum + '-' + fn diff --git a/createrepo/yumbased.py b/createrepo/yumbased.py -index b820fd8..32eed04 100644 +index b820fd8..f04f4fb 100644 --- a/createrepo/yumbased.py +++ b/createrepo/yumbased.py -@@ -60,7 +60,7 @@ class CreateRepoPackage(YumLocalPackage): +@@ -18,21 +18,19 @@ + import os + import rpm + import types +-import re + + from yum.packages import YumLocalPackage + from yum.Errors import * + from yum import misc +-from yum.sqlutils import executeSQL +-from rpmUtils.transaction import initReadOnlyTransaction +-from rpmUtils.miscutils import flagToString, stringToVersion + import utils + import tempfile + + class CreateRepoPackage(YumLocalPackage): +- def __init__(self, ts, package): ++ def __init__(self, ts, package, sumtype=None): + YumLocalPackage.__init__(self, ts, package) +- ++ if sumtype: ++ self.checksum_type = sumtype ++ + def _do_checksum(self): + """return a checksum for a package: + - check if the checksum cache is enabled +@@ -60,7 +58,7 @@ class CreateRepoPackage(YumLocalPackage): if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType: t.append("".join(self.hdr[rpm.RPMTAG_HDRID])) @@ -244,23 +867,274 @@ index e7cce4f..ba2f327 100644 .SH "EXAMPLES" Here is an example of a repository with a groups file. Note that the diff --git a/genpkgmetadata.py b/genpkgmetadata.py -index 041bc02..c987532 100755 +index 041bc02..e25097d 100755 --- a/genpkgmetadata.py +++ b/genpkgmetadata.py -@@ -52,7 +52,7 @@ def parseArgs(args, conf): +@@ -16,7 +16,8 @@ + # along with this program; if not, write to the Free Software + # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + # Copyright 2004 Duke University +-# Portions Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org ++# Portions Copyright 2009 Red Hat, Inc - ++# written by seth vidal skvidal at fedoraproject.org + + import os + import sys +@@ -29,7 +30,7 @@ from createrepo import MDError + from createrepo.utils import errorprint, _ + + +-def parseArgs(args, conf): ++def parse_args(args, conf): + """ + Parse the command line args. return a config object. + Sanity check all the things being passed in. +@@ -38,67 +39,69 @@ def parseArgs(args, conf): + parser = OptionParser(version = "createrepo %s" % createrepo.__version__) + # query options + parser.add_option("-q", "--quiet", default=False, action="store_true", +- help="output nothing except for serious errors") ++ help="output nothing except for serious errors") + parser.add_option("-v", "--verbose", default=False, action="store_true", +- help="output more debugging info.") ++ help="output more debugging info.") + parser.add_option("--profile", default=False, action="store_true", +- help="output timing/profile info.") ++ help="output timing/profile info.") + parser.add_option("-x", "--excludes", default=[], action="append", +- help="files to exclude") ++ help="files to exclude") + parser.add_option("--basedir", default=os.getcwd(), +- help="basedir for path to directories") ++ help="basedir for path to directories") + parser.add_option("-u", "--baseurl", default=None, +- help="baseurl to append on all files") ++ help="baseurl to append on all files") parser.add_option("-g", "--groupfile", default=None, - help="path to groupfile to include in metadata") +- help="path to groupfile to include in metadata") ++ help="path to groupfile to include in metadata") parser.add_option("-s", "--checksum", default="sha256", dest='sumtype', - help="Deprecated, ignore") -+ help="specify the checksum type to use") ++ help="specify the checksum type to use") parser.add_option("-p", "--pretty", default=False, action="store_true", - help="make sure all xml generated is formatted") +- help="make sure all xml generated is formatted") ++ help="make sure all xml generated is formatted") parser.add_option("-c", "--cachedir", default=None, +- help="set path to cache dir") ++ help="set path to cache dir") + parser.add_option("-C", "--checkts", default=False, action="store_true", +- help="check timestamps on files vs the metadata to see if we need to update") ++ help="check timestamps on files vs the metadata to see" \ ++ "if we need to update") + parser.add_option("-d", "--database", default=False, action="store_true", +- help="create sqlite database files") ++ help="create sqlite database files") + # temporarily disabled +- #parser.add_option("--database-only", default=False, action="store_true", dest='database_only', +- # help="Only make the sqlite databases - does not work with --update, yet") ++ #parser.add_option("--database-only", default=False, action="store_true", ++ # dest='database_only', ++ # help="Only make the sqlite databases - does not work with --update, yet") + parser.add_option("--update", default=False, action="store_true", +- help="use the existing repodata to speed up creation of new") ++ help="use the existing repodata to speed up creation of new") + parser.add_option("--update-md-path", default=None, dest='update_md_path', +- help="use the existing repodata for --update from this path") +- parser.add_option("--skip-stat", dest='skip_stat', default=False, action="store_true", +- help="skip the stat() call on a --update, assumes if the file" \ +- "name is the same then the file is still the same" \ +- "(only use this if you're fairly trusting or gullible)" ) ++ help="use the existing repodata for --update from this path") ++ parser.add_option("--skip-stat", dest='skip_stat', default=False, ++ help="skip the stat() call on a --update, assumes if the file" \ ++ "name is the same then the file is still the same" \ ++ "(only use this if you're fairly trusting or gullible)", ++ action="store_true") + parser.add_option("--split", default=False, action="store_true", +- help="generate split media") ++ help="generate split media") + parser.add_option("-i", "--pkglist", default=None, +- help="use only the files listed in this file from the directory specified") ++ help="use only the files listed in this file from the" \ ++ "directory specified") + parser.add_option("-o", "--outputdir", default=None, +- help=" = optional directory to output to") ++ help=" = optional directory to output to") + parser.add_option("-S", "--skip-symlinks", dest="skip_symlinks", +- default=False, action="store_true", +- help="ignore symlinks of packages") ++ default=False, action="store_true", help="ignore symlinks of packages") + parser.add_option("--changelog-limit", dest="changelog_limit", +- default=None, help="only import the last N changelog entries") ++ default=None, help="only import the last N changelog entries") + parser.add_option("--unique-md-filenames", dest="unique_md_filenames", +- default=False, action="store_true", +- help="include the file's checksum in the filename, helps" \ +- "with proxies") ++ help="include the file's checksum in the filename,helps with proxies", ++ default=False, action="store_true") + parser.add_option("--distro", default=[], action="append", +- help="distro tag and optional cpeid: --distro 'cpeid,textname'") +- parser.add_option("--content", default=[], dest='content_tags', action="append", +- help="tags for the content in the repository") ++ help="distro tag and optional cpeid: --distro" "'cpeid,textname'") ++ parser.add_option("--content", default=[], dest='content_tags', ++ action="append", help="tags for the content in the repository") + parser.add_option("--revision", default=None, +- help="user-specified revision for this repository") ++ help="user-specified revision for this repository") + parser.add_option("--deltas", default=False, action="store_true", +- help="create delta rpms and metadata") ++ help="create delta rpms and metadata") + parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", +- action="append", help="paths to look for older pkgs to delta against") ++ action="append", help="paths to look for older pkgs to delta against") + parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', +- help="the number of older versions to make deltas against") ++ help="the number of older versions to make deltas against") + + + (opts, argsleft) = parser.parse_args(args) +@@ -120,11 +123,12 @@ def parseArgs(args, conf): + sys.exit(1) + + +- # let's switch over to using the conf object - put all the opts options into it ++ # let's switch over to using the conf object - put all the opts into it + for opt in parser.option_list: + if opt.dest is None: # this is fairly silly + continue +- if getattr(opts, opt.dest) is None: # if it's not set, take the default from the base class ++ # if it's not set, take the default from the base class ++ if getattr(opts, opt.dest) is None: + continue + setattr(conf, opt.dest, getattr(opts, opt.dest)) + +@@ -136,7 +140,7 @@ def parseArgs(args, conf): + + for spec in opts.distro: + if spec.find(',') == -1: +- conf.distro_tags.append((None,spec)) ++ conf.distro_tags.append((None, spec)) + else: + splitspec = spec.split(',') + conf.distro_tags.append((splitspec[0], splitspec[1])) +@@ -159,31 +163,38 @@ def parseArgs(args, conf): + return conf + + class MDCallBack(object): ++ """cli callback object for createrepo""" + def errorlog(self, thing): ++ """error log output""" + print >> sys.stderr, thing + + def log(self, thing): ++ """log output""" + print thing + + def progress(self, item, current, total): ++ """progress bar""" + beg = "%*d/%d - " % (len(str(total)), current, total) + left = 80 - len(beg) + sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) + sys.stdout.flush() + + def main(args): ++ """createrepo from cli main flow""" + start_st = time.time() + conf = createrepo.MetaDataConfig() +- conf = parseArgs(args, conf) ++ conf = parse_args(args, conf) + if conf.profile: + print ('start time: %0.3f' % (time.time() - start_st)) + + mid_st = time.time() + try: + if conf.split: +- mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) ++ mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, ++ callback=MDCallBack()) + else: +- mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) ++ mdgen = createrepo.MetaDataGenerator(config_obj=conf, ++ callback=MDCallBack()) + if mdgen.checkTimeStamps(): + if mdgen.conf.verbose: + print _('repo is up to date') +@@ -200,14 +211,14 @@ def main(args): + mdgen.doRepoMetadata() + if conf.profile: + print ('rm time: %0.3f' % (time.time() - rm_st)) +- fm_st = time.time() ++ fm_st = time.time() + mdgen.doFinalMove() + if conf.profile: + print ('fm time: %0.3f' % (time.time() - fm_st)) + + +- except MDError, e: +- errorprint(_('%s') % e) ++ except MDError, errormsg: ++ errorprint(_('%s') % errormsg) + sys.exit(1) + + diff --git a/modifyrepo.py b/modifyrepo.py -index 1efe966..341f3ab 100755 +index 1efe966..bbf938d 100755 --- a/modifyrepo.py +++ b/modifyrepo.py -@@ -88,7 +88,7 @@ class RepoMetadata: +@@ -24,11 +24,12 @@ + import os + import sys + +-from createrepo.utils import checksum_and_rename, GzipFile ++from createrepo.utils import checksum_and_rename, GzipFile, MDError + from yum.misc import checksum + + from xml.dom import minidom + ++ + class RepoMetadata: + + def __init__(self, repo): +@@ -37,11 +38,13 @@ class RepoMetadata: + self.repomdxml = os.path.join(self.repodir, 'repomd.xml') + self.checksum_type = 'sha256' + if not os.path.exists(self.repomdxml): +- raise Exception('%s not found' % self.repomdxml) ++ raise MDError, '%s not found' % self.repomdxml + self.doc = minidom.parse(self.repomdxml) + +- def _insert_element(self, parent, name, attrs={}, text=None): ++ def _insert_element(self, parent, name, attrs=None, text=None): + child = self.doc.createElement(name) ++ if not attrs: ++ attrs = {} + for item in attrs.items(): + child.setAttribute(item[0], item[1]) + if text: +@@ -57,7 +60,7 @@ class RepoMetadata: + """ + md = None + if not metadata: +- raise Exception('metadata cannot be None') ++ raise MDError, 'metadata cannot be None' + if isinstance(metadata, minidom.Document): + md = metadata.toxml() + mdname = 'updateinfo.xml' +@@ -71,9 +74,9 @@ class RepoMetadata: + oldmd.close() + mdname = os.path.basename(metadata) + else: +- raise Exception('%s not found' % metadata) ++ raise MDError, '%s not found' % metadata + else: +- raise Exception('invalid metadata type') ++ raise MDError, 'invalid metadata type' + + ## Compress the metadata and move it into the repodata + if not mdname.endswith('.gz'): +@@ -88,7 +91,7 @@ class RepoMetadata: open_csum = checksum(self.checksum_type, metadata) @@ -269,3 +1143,36 @@ index 1efe966..341f3ab 100755 base_destmd = os.path.basename(destmd) +@@ -106,13 +109,15 @@ class RepoMetadata: + self._insert_element(data, 'location', + attrs={ 'href' : 'repodata/' + base_destmd }) + data.appendChild(self.doc.createTextNode("\n ")) +- self._insert_element(data, 'checksum', attrs={ 'type' : self.checksum_type }, ++ self._insert_element(data, 'checksum', ++ attrs={ 'type' : self.checksum_type }, + text=csum) + data.appendChild(self.doc.createTextNode("\n ")) + self._insert_element(data, 'timestamp', + text=str(os.stat(destmd).st_mtime)) + data.appendChild(self.doc.createTextNode("\n ")) +- self._insert_element(data, 'open-checksum', attrs={ 'type' : self.checksum_type }, ++ self._insert_element(data, 'open-checksum', ++ attrs={ 'type' : self.checksum_type }, + text=open_csum) + + data.appendChild(self.doc.createTextNode("\n ")) +@@ -138,12 +143,12 @@ if __name__ == '__main__': + sys.exit() + try: + repomd = RepoMetadata(sys.argv[2]) +- except Exception, e: ++ except MDError, e: + print "Could not access repository: %s" % str(e) + sys.exit(1) + try: + repomd.add(sys.argv[1]) +- except Exception, e: ++ except MDError, e: + print "Could not add metadata from file %s: %s" % (sys.argv[1], str(e)) + sys.exit(1) + diff --git a/createrepo.spec b/createrepo.spec index b65bdf6..3e479ba 100644 --- a/createrepo.spec +++ b/createrepo.spec @@ -3,13 +3,12 @@ Summary: Creates a common metadata repository Name: createrepo Version: 0.9.7 -Release: 6%{?dist} +Release: 7%{?dist} License: GPLv2 Group: System Environment/Base Source: %{name}-%{version}.tar.gz Patch0: ten-changelog-limit.patch Patch1: createrepo-head.patch -Patch2: rawhide-fix.patch URL: http://createrepo.baseurl.org/ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArchitectures: noarch @@ -25,7 +24,6 @@ packages. %setup -q %patch0 -p0 %patch1 -p1 -%patch2 -p1 %build @@ -48,6 +46,9 @@ rm -rf $RPM_BUILD_ROOT %{python_sitelib}/createrepo %changelog +* Tue May 5 2009 Seth Vidal - 0.9.7-7 +- reup to head + * Sat Apr 18 2009 James Antill - 0.9.7-6 - add patch to fix size limit