From 731c7f8f16db88037a0754c500f9ef8c7da673ed Mon Sep 17 00:00:00 2001 From: Seth Vidal Date: Oct 24 2011 17:49:54 +0000 Subject: latest upstream --- diff --git a/createrepo-head.patch b/createrepo-head.patch index 14e40e6..e5a8ebc 100644 --- a/createrepo-head.patch +++ b/createrepo-head.patch @@ -1,8 +1,27 @@ diff --git a/createrepo.bash b/createrepo.bash -index 54ac8b2..4222fa0 100644 +index 54ac8b2..f0695ba 100644 --- a/createrepo.bash +++ b/createrepo.bash -@@ -30,6 +30,10 @@ _cr_createrepo() +@@ -1,11 +1,17 @@ + # bash completion for createrepo and friends + ++_cr_compress_type() ++{ ++ COMPREPLY=( $( compgen -W "$( ${1:-createrepo} --compress-type=FOO / 2>&1 \ ++ | sed -ne 's/,/ /g' -ne 's/.*[Cc]ompression.*://p' )" -- "$2" ) ) ++} ++ + _cr_createrepo() + { + COMPREPLY=() + + case $3 in +- --version|-h|--help|-u|--baseurl|--distro|--content|--repo|--workers|\ ++ --version|-h|--help|-u|--baseurl|--distro|--content|--repo|\ + --revision|-x|--excludes|--changelog-limit|--max-delta-rpm-size) + return 0 + ;; +@@ -30,10 +36,24 @@ _cr_createrepo() COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) ) return 0 ;; @@ -13,22 +32,102 @@ index 54ac8b2..4222fa0 100644 --num-deltas) COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) ) return 0 -@@ -42,8 +46,8 @@ _cr_createrepo() + ;; ++ --workers) ++ local min=2 max=$( getconf _NPROCESSORS_ONLN 2>/dev/null ) ++ [[ -z $max || $max -lt $min ]] && max=$min ++ COMPREPLY=( $( compgen -W "{1..$max}" -- "$2" ) ) ++ return 0 ++ ;; ++ --compress-type) ++ _cr_compress_type "$1" "$2" ++ return 0 ++ ;; + esac + + if [[ $2 == -* ]] ; then +@@ -42,9 +62,9 @@ _cr_createrepo() --cachedir --checkts --no-database --update --update-md-path --skip-stat --split --pkglist --includepkg --outputdir --skip-symlinks --changelog-limit --unique-md-filenames - --simple-md-filenames --distro --content --repo --revision --deltas - --oldpackagedirs --num-deltas --read-pkgs-list +- --max-delta-rpm-size --workers' -- "$2" ) ) + --simple-md-filenames --retain-old-md --distro --content --repo + --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list - --max-delta-rpm-size --workers' -- "$2" ) ) ++ --max-delta-rpm-size --workers --xz --compress-type' -- "$2" ) ) else COMPREPLY=( $( compgen -d -- "$2" ) ) + fi +@@ -63,10 +83,14 @@ _cr_mergerepo() + COMPREPLY=( $( compgen -d -- "$2" ) ) + return 0 + ;; ++ --compress-type) ++ _cr_compress_type "" "$2" ++ return 0 ++ ;; + esac + + COMPREPLY=( $( compgen -W '--version --help --repo --archlist --no-database +- --outputdir --nogroups --noupdateinfo' -- "$2" ) ) ++ --outputdir --nogroups --noupdateinfo --compress-type' -- "$2" ) ) + } && + complete -F _cr_mergerepo -o filenames mergerepo mergerepo.py + +@@ -78,17 +102,22 @@ _cr_modifyrepo() + --version|-h|--help|--mdtype) + return 0 + ;; ++ --compress-type) ++ _cr_compress_type "" "$2" ++ return 0 ++ ;; + esac + + if [[ $2 == -* ]] ; then +- COMPREPLY=( $( compgen -W '--version --help --mdtype' -- "$2" ) ) ++ COMPREPLY=( $( compgen -W '--version --help --mdtype --remove ++ --compress --compress-type' -- "$2" ) ) + return 0 + fi + + local i argnum=1 + for (( i=1; i < ${#COMP_WORDS[@]}-1; i++ )) ; do + if [[ ${COMP_WORDS[i]} != -* && +- ${COMP_WORDS[i-1]} != @(=|--mdtype) ]]; then ++ ${COMP_WORDS[i-1]} != @(=|--@(md|compress-)type) ]]; then + argnum=$(( argnum+1 )) + fi + done +diff --git a/createrepo.spec b/createrepo.spec +index 1e491cd..eea7092 100644 +--- a/createrepo.spec ++++ b/createrepo.spec +@@ -11,7 +11,7 @@ URL: http://createrepo.baseurl.org/ + BuildRoot: %{_tmppath}/%{name}-%{version}root + BuildArchitectures: noarch + Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python +-Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm ++Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm, pyliblzma + + %description + This utility will generate a common metadata repository from a directory of +@@ -43,6 +43,9 @@ make DESTDIR=$RPM_BUILD_ROOT sysconfdir=%{_sysconfdir} install + %{python_sitelib}/createrepo + + %changelog ++* Fri Sep 9 2011 Seth Vidal ++- add lzma dep ++ + * Wed Jan 26 2011 Seth Vidal + - bump to 0.9.9 + - add worker.py diff --git a/createrepo/__init__.py b/createrepo/__init__.py -index 8f2538e..30f7422 100644 +index 8f2538e..61694f9 100644 --- a/createrepo/__init__.py +++ b/createrepo/__init__.py -@@ -27,11 +27,11 @@ import stat +@@ -27,14 +27,14 @@ import stat import fcntl import subprocess @@ -42,16 +141,125 @@ index 8f2538e..30f7422 100644 +from yum.packages import YumAvailablePackage import rpmUtils.transaction - from utils import _, errorprint, MDError -@@ -110,6 +110,7 @@ class MetaDataConfig(object): +-from utils import _, errorprint, MDError ++from utils import _, errorprint, MDError, lzma, _available_compression + import readMetadata + try: + import sqlite3 as sqlite +@@ -46,7 +46,7 @@ try: + except ImportError: + pass + +-from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \ ++from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \ + checksum_and_rename, split_list_into_equal_chunks + import deltarpms + +@@ -74,7 +74,7 @@ class MetaDataConfig(object): + self.deltadir = None + self.delta_relative = 'drpms/' + self.oldpackage_paths = [] # where to look for the old packages - +- self.deltafile = 'prestodelta.xml.gz' ++ self.deltafile = 'prestodelta.xml' + self.num_deltas = 1 # number of older versions to delta (max) + self.max_delta_rpm_size = 100000000 + self.update_md_path = None +@@ -86,9 +86,9 @@ class MetaDataConfig(object): + self.skip_symlinks = False + self.pkglist = [] + self.database_only = False +- self.primaryfile = 'primary.xml.gz' +- self.filelistsfile = 'filelists.xml.gz' +- self.otherfile = 'other.xml.gz' ++ self.primaryfile = 'primary.xml' ++ self.filelistsfile = 'filelists.xml' ++ self.otherfile = 'other.xml' + self.repomdfile = 'repomd.xml' + self.tempdir = '.repodata' + self.finaldir = 'repodata' +@@ -108,8 +108,10 @@ class MetaDataConfig(object): + self.collapse_glibc_requires = True + self.workers = 1 # number of workers to fork off to grab metadata from the pkgs self.worker_cmd = '/usr/share/createrepo/worker.py' - +- #self.worker_cmd = './worker.py' # helpful when testing + self.retain_old_md = 0 ++ self.compress_type = 'gz' ++ class SimpleMDCallBack(object): def errorlog(self, thing): -@@ -530,39 +531,19 @@ class MetaDataGenerator: +@@ -145,6 +147,14 @@ class MetaDataGenerator: + if not self.conf.directory and not self.conf.directories: + raise MDError, "No directory given on which to run." + ++ if not self.conf.compress_type: ++ self.conf.compress_type = 'gz' ++ ++ if self.conf.compress_type not in utils._available_compression: ++ raise MDError, "Compression %s not available: Please choose from: %s" \ ++ % (self.conf.compress_type, ', '.join(utils._available_compression)) ++ ++ + if not self.conf.directories: # just makes things easier later + self.conf.directories = [self.conf.directory] + if not self.conf.directory: # ensure we have both in the config object +@@ -410,9 +420,11 @@ class MetaDataGenerator: + + def _setupPrimary(self): + # setup the primary metadata file ++ # FIXME - make this be conf.compress_type once y-m-p is fixed ++ fpz = self.conf.primaryfile + '.' + 'gz' + primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, +- self.conf.primaryfile) +- fo = _gzipOpen(primaryfilepath, 'w') ++ fpz) ++ fo = compressOpen(primaryfilepath, 'w', 'gz') + fo.write('\n') + fo.write('' % +@@ -421,9 +433,11 @@ class MetaDataGenerator: + + def _setupFilelists(self): + # setup the filelist file ++ # FIXME - make this be conf.compress_type once y-m-p is fixed ++ fpz = self.conf.filelistsfile + '.' + 'gz' + filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, +- self.conf.filelistsfile) +- fo = _gzipOpen(filelistpath, 'w') ++ fpz) ++ fo = compressOpen(filelistpath, 'w', 'gz') + fo.write('\n') + fo.write('' % self.pkgcount) +@@ -431,9 +445,11 @@ class MetaDataGenerator: + + def _setupOther(self): + # setup the other file ++ # FIXME - make this be conf.compress_type once y-m-p is fixed ++ fpz = self.conf.otherfile + '.' + 'gz' + otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, +- self.conf.otherfile) +- fo = _gzipOpen(otherfilepath, 'w') ++ fpz) ++ fo = compressOpen(otherfilepath, 'w', 'gz') + fo.write('\n') + fo.write('' % +@@ -442,9 +458,10 @@ class MetaDataGenerator: + + def _setupDelta(self): + # setup the other file ++ fpz = self.conf.deltafile + '.' + self.conf.compress_type + deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, +- self.conf.deltafile) +- fo = _gzipOpen(deltafilepath, 'w') ++ fpz) ++ fo = compressOpen(deltafilepath, 'w', self.conf.compress_type) + fo.write('\n') + fo.write('\n') + return fo +@@ -530,39 +547,19 @@ class MetaDataGenerator: old_pkg = pkg if pkg.find("://") != -1: old_pkg = os.path.basename(pkg) @@ -101,7 +309,7 @@ index 8f2538e..30f7422 100644 #FIXME - if we're in update and we have deltas enabled # check the presto data for this pkg and write its info back out # to our deltafile -@@ -584,12 +565,12 @@ class MetaDataGenerator: +@@ -584,12 +581,12 @@ class MetaDataGenerator: po = None if isinstance(pkg, YumAvailablePackage): po = pkg @@ -116,16 +324,18 @@ index 8f2538e..30f7422 100644 self.read_pkgs.append(pkg) if po: -@@ -609,7 +590,7 @@ class MetaDataGenerator: +@@ -608,8 +605,8 @@ class MetaDataGenerator: + # waitfor the workers to finish and as each one comes in # open the files they created and write them out to our metadata # add up the total pkg counts and return that value - worker_tmp_path = tempfile.mkdtemp() +- worker_tmp_path = tempfile.mkdtemp() - worker_chunks = utils.split_list_into_equal_chunks(pkgfiles, self.conf.workers) ++ self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later + worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers) worker_cmd_dict = {} worker_jobs = {} base_worker_cmdline = [self.conf.worker_cmd, -@@ -617,7 +598,8 @@ class MetaDataGenerator: +@@ -617,7 +614,8 @@ class MetaDataGenerator: '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires, '--pkgoptions=_cachedir=%s' % self.conf.cachedir, '--pkgoptions=_baseurl=%s' % self.conf.baseurl, @@ -135,7 +345,30 @@ index 8f2538e..30f7422 100644 if self.conf.quiet: base_worker_cmdline.append('--quiet') -@@ -660,7 +642,12 @@ class MetaDataGenerator: +@@ -626,15 +624,20 @@ class MetaDataGenerator: + base_worker_cmdline.append('--verbose') + + for worker_num in range(self.conf.workers): ++ pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num ++ f = open(pkl, 'w') ++ f.write('\n'.join(worker_chunks[worker_num])) ++ f.close() ++ + # make the worker directory + workercmdline = [] + workercmdline.extend(base_worker_cmdline) +- thisdir = worker_tmp_path + '/' + str(worker_num) ++ thisdir = self._worker_tmp_path + '/' + str(worker_num) + if checkAndMakeDir(thisdir): + workercmdline.append('--tmpmdpath=%s' % thisdir) + else: + raise MDError, "Unable to create worker path: %s" % thisdir +- workercmdline.extend(worker_chunks[worker_num]) ++ workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num)) + worker_cmd_dict[worker_num] = workercmdline + + +@@ -660,7 +663,12 @@ class MetaDataGenerator: if line: self.callback.errorlog('Worker %s: %s' % (num, line.rstrip())) @@ -149,7 +382,16 @@ index 8f2538e..30f7422 100644 if not self.conf.quiet: self.callback.log("Workers Finished") # finished with workers -@@ -784,7 +771,6 @@ class MetaDataGenerator: +@@ -671,7 +679,7 @@ class MetaDataGenerator: + for (fn, fo) in (('primary.xml', self.primaryfile), + ('filelists.xml', self.flfile), + ('other.xml', self.otherfile)): +- fnpath = worker_tmp_path + '/' + str(num) + '/' + fn ++ fnpath = self._worker_tmp_path + '/' + str(num) + '/' + fn + if os.path.exists(fnpath): + fo.write(open(fnpath, 'r').read()) + +@@ -784,7 +792,6 @@ class MetaDataGenerator: return self._old_package_dict self._old_package_dict = {} @@ -157,7 +399,38 @@ index 8f2538e..30f7422 100644 for d in self.conf.oldpackage_paths: for f in self.getFileList(d, '.rpm'): fp = d + '/' + f -@@ -874,7 +860,6 @@ class MetaDataGenerator: +@@ -833,7 +840,7 @@ class MetaDataGenerator: + return ' '.join(results) + + def _createRepoDataObject(self, mdfile, mdtype, compress=True, +- compress_type='gzip', attribs={}): ++ compress_type=None, attribs={}): + """return random metadata as RepoData object to be added to RepoMD + mdfile = complete path to file + mdtype = the metadata type to use +@@ -843,15 +850,13 @@ class MetaDataGenerator: + sfile = os.path.basename(mdfile) + fo = open(mdfile, 'r') + outdir = os.path.join(self.conf.outputdir, self.conf.tempdir) ++ if not compress_type: ++ compress_type = self.conf.compress_type + if compress: +- if compress_type == 'gzip': +- sfile = '%s.gz' % sfile +- outfn = os.path.join(outdir, sfile) +- output = GzipFile(filename = outfn, mode='wb') +- elif compress_type == 'bzip2': +- sfile = '%s.bz2' % sfile +- outfn = os.path.join(outdir, sfile) +- output = BZ2File(filename = outfn, mode='wb') ++ sfile = '%s.%s' % (sfile, compress_type) ++ outfn = os.path.join(outdir, sfile) ++ output = compressOpen(outfn, mode='wb', compress_type=compress_type) ++ + else: + outfn = os.path.join(outdir, sfile) + output = open(outfn, 'w') +@@ -874,7 +879,6 @@ class MetaDataGenerator: thisdata = RepoData() thisdata.type = mdtype @@ -165,7 +438,49 @@ index 8f2538e..30f7422 100644 thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile)) thisdata.checksum = (self.conf.sumtype, csum) if compress: -@@ -1046,7 +1031,7 @@ class MetaDataGenerator: +@@ -925,9 +929,13 @@ class MetaDataGenerator: + rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None) + + for (rpm_file, ftype) in workfiles: ++ # when we fix y-m-p and non-gzipped xml files - then we can make this just add ++ # self.conf.compress_type ++ if ftype in ('other', 'filelists', 'primary'): ++ rpm_file = rpm_file + '.' + 'gz' + complete_path = os.path.join(repopath, rpm_file) + +- zfo = _gzipOpen(complete_path) ++ zfo = compressOpen(complete_path) + # This is misc.checksum() done locally so we can get the size too. + data = misc.Checksums([sumtype]) + while data.read(zfo, 2**16): +@@ -968,12 +976,13 @@ class MetaDataGenerator: + + # rename from silly name to not silly name + os.rename(tmp_result_path, resultpath) +- compressed_name = '%s.bz2' % good_name ++ ext = self.conf.compress_type ++ compressed_name = '%s.%s' % (good_name, ext) + result_compressed = os.path.join(repopath, compressed_name) + db_csums[ftype] = misc.checksum(sumtype, resultpath) + + # compress the files +- bzipFile(resultpath, result_compressed) ++ compressFile(resultpath, result_compressed, self.conf.compress_type) + # csum the compressed file + db_compressed_sums[ftype] = misc.checksum(sumtype, + result_compressed) +@@ -983,8 +992,8 @@ class MetaDataGenerator: + os.unlink(resultpath) + + if self.conf.unique_md_filenames: +- csum_compressed_name = '%s-%s.bz2' % ( +- db_compressed_sums[ftype], good_name) ++ csum_compressed_name = '%s-%s.%s' % ( ++ db_compressed_sums[ftype], good_name, ext) + csum_result_compressed = os.path.join(repopath, + csum_compressed_name) + os.rename(result_compressed, csum_result_compressed) +@@ -1046,7 +1055,7 @@ class MetaDataGenerator: if self.conf.additional_metadata: @@ -174,7 +489,7 @@ index 8f2538e..30f7422 100644 mdcontent = self._createRepoDataObject(md_file, md_type) repomd.repoData[mdcontent.type] = mdcontent -@@ -1110,23 +1095,42 @@ class MetaDataGenerator: +@@ -1110,23 +1119,43 @@ class MetaDataGenerator: raise MDError, _( 'Could not remove old metadata file: %s: %s') % (oldfile, e) @@ -199,11 +514,15 @@ index 8f2538e..30f7422 100644 - 'other.xml.gz','filelists.xml.gz'): - os.remove(oldfile) # kill off the old ones - continue +- if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', +- 'primary.sqlite.bz2'): +- os.remove(oldfile) + -+ for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr), -+ ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl), -+ ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)): -+ if f.endswith(end): ++ for (end,lst) in (('-primary.sqlite', old_pr_db), ('-primary.xml', old_pr), ++ ('-filelists.sqlite', old_fl_db), ('-filelists.xml', old_fl), ++ ('-other.sqlite', old_ot_db), ('-other.xml', old_ot)): ++ fn = '.'.join(f.split('.')[:-1]) ++ if fn.endswith(end): + lst.append(oldfile) + break + @@ -217,11 +536,9 @@ index 8f2538e..30f7422 100644 + for f in os.listdir(output_old_dir): + oldfile = os.path.join(output_old_dir, f) + finalfile = os.path.join(output_final_dir, f) -+ - if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2', -- 'primary.sqlite.bz2'): -- os.remove(oldfile) -+ 'primary.sqlite.bz2') or oldfile in old_to_remove: ++ fn = '.'.join(f.split('.')[:-1]) ++ if fn in ('filelists.sqlite', 'other.sqlite.', ++ 'primary.sqlite') or oldfile in old_to_remove: + try: + os.remove(oldfile) + except (OSError, IOError), e: @@ -230,7 +547,7 @@ index 8f2538e..30f7422 100644 continue if os.path.exists(finalfile): -@@ -1147,14 +1151,11 @@ class MetaDataGenerator: +@@ -1147,14 +1176,19 @@ class MetaDataGenerator: msg += _('Error was %s') % e raise MDError, msg @@ -242,14 +559,22 @@ index 8f2538e..30f7422 100644 - self.errorlog(_('Error was %s') % e) - self.errorlog(_('Please clean up this directory manually.')) + self._cleanup_tmp_repodata_dir() ++ self._cleanup_update_tmp_dir() + self._write_out_read_pkgs_list() -+ ++ ++ def _cleanup_update_tmp_dir(self): ++ if not self.conf.update: ++ return ++ ++ shutil.rmtree(self.oldData._repo.basecachedir, ignore_errors=True) ++ shutil.rmtree(self.oldData._repo.base_persistdir, ignore_errors=True) ++ + def _write_out_read_pkgs_list(self): # write out the read_pkgs_list file with self.read_pkgs if self.conf.read_pkgs_list: try: -@@ -1167,6 +1168,20 @@ class MetaDataGenerator: +@@ -1167,6 +1201,23 @@ class MetaDataGenerator: % self.conf.read_pkgs_list) self.errorlog(_('Error was %s') % e) @@ -266,11 +591,14 @@ index 8f2538e..30f7422 100644 + % dirbase) + self.errorlog(_('Error was %s') % e) + self.errorlog(_('Please clean up this directory manually.')) ++ # our worker tmp path ++ if hasattr(self, '_worker_tmp_path') and os.path.exists(self._worker_tmp_path): ++ shutil.rmtree(self._worker_tmp_path, ignore_errors=True) + def setup_sqlite_dbs(self, initdb=True): """sets up the sqlite dbs w/table schemas and db_infos""" destdir = os.path.join(self.conf.outputdir, self.conf.tempdir) -@@ -1240,7 +1255,6 @@ class SplitMetaDataGenerator(MetaDataGenerator): +@@ -1240,7 +1291,6 @@ class SplitMetaDataGenerator(MetaDataGenerator): self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano) try: self.openMetadataDocs() @@ -334,7 +662,7 @@ index b3b2ea1..1ac43bb 100644 if self.updateinfo: ui_fn = mytempdir + '/updateinfo.xml' diff --git a/createrepo/readMetadata.py b/createrepo/readMetadata.py -index 27d3690..88e5d95 100644 +index 27d3690..54863cb 100644 --- a/createrepo/readMetadata.py +++ b/createrepo/readMetadata.py @@ -16,11 +16,25 @@ @@ -365,7 +693,7 @@ index 27d3690..88e5d95 100644 class MetadataIndex(object): -@@ -30,178 +44,73 @@ class MetadataIndex(object): +@@ -30,178 +44,72 @@ class MetadataIndex(object): opts = {} self.opts = opts self.outputdir = outputdir @@ -390,6 +718,7 @@ index 27d3690..88e5d95 100644 + self._repo = yum.yumRepo.YumRepository('garbageid') + self._repo.baseurl = 'file://' + realpath + self._repo.basecachedir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo") ++ self._repo.base_persistdir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo-p") + self._repo.metadata_expire = 1 + self._repo.gpgcheck = 0 + self._repo.repo_gpgcheck = 0 @@ -538,12 +867,12 @@ index 27d3690..88e5d95 100644 + continue + + self.pkg_tups_by_path[relpath] = thispo.pkgtup ++ - def getNodes(self, relpath): - """Return base, filelist, and other nodes for file, if they exist - Returns a tuple of nodes, or None if not found -+ + def getNodes(self, relpath): + """return a package object based on relative path of pkg """ @@ -575,12 +904,7 @@ index 27d3690..88e5d95 100644 - pkgid = self.pkg_ids.get(relpath,None) - if pkgid is None: - print _("No pkgid found for: %s") % relpath -+ if relpath in self.pkg_tups_by_path: -+ pkgtup = self.pkg_tups_by_path[relpath] -+ return self._repo.sack.searchPkgTuple(pkgtup)[0] -+ else: -+ print _("No pkg found for: %s") % relpath - return None +- return None - del self.pkg_ids[relpath] - dups = self.pkgrefs.get(pkgid) - dups.remove(relpath) @@ -594,12 +918,16 @@ index 27d3690..88e5d95 100644 - node.unlinkNode() - node.freeNode() - del nodes[pkgid] ++ if relpath in self.pkg_tups_by_path: ++ pkgtup = self.pkg_tups_by_path[relpath] ++ return self._repo.sack.searchPkgTuple(pkgtup)[0] ++ return None + if __name__ == "__main__": cwd = os.getcwd() -@@ -209,9 +118,9 @@ if __name__ == "__main__": +@@ -209,9 +117,9 @@ if __name__ == "__main__": 'pkgdir': cwd} idx = MetadataIndex(cwd, opts) @@ -615,8 +943,91 @@ index 27d3690..88e5d95 100644 + print po.xml_dump_filelists_metadata() + print po.xml_dump_other_metadata() + +diff --git a/createrepo/utils.py b/createrepo/utils.py +index 995c3b9..c816640 100644 +--- a/createrepo/utils.py ++++ b/createrepo/utils.py +@@ -23,6 +23,12 @@ import bz2 + import gzip + from gzip import write32u, FNAME + from yum import misc ++_available_compression = ['gz', 'bz2'] ++try: ++ import lzma ++ _available_compression.append('xz') ++except ImportError: ++ lzma = None + + def errorprint(stuff): + print >> sys.stderr, stuff +@@ -69,6 +75,65 @@ def bzipFile(source, dest): + s_fn.close() + + ++def xzFile(source, dest): ++ if not 'xz' in _available_compression: ++ raise MDError, "Cannot use xz for compression, library/module is not available" ++ ++ s_fn = open(source, 'rb') ++ destination = lzma.LZMAFile(dest, 'w') ++ ++ while True: ++ data = s_fn.read(1024000) ++ ++ if not data: break ++ destination.write(data) ++ ++ destination.close() ++ s_fn.close() ++ ++def gzFile(source, dest): ++ ++ s_fn = open(source, 'rb') ++ destination = GzipFile(dest, 'w') ++ ++ while True: ++ data = s_fn.read(1024000) ++ ++ if not data: break ++ destination.write(data) ++ ++ destination.close() ++ s_fn.close() ++ ++ ++ ++def compressFile(source, dest, compress_type): ++ """Compress an existing file using any compression type from source to dest""" ++ ++ if compress_type == 'xz': ++ xzFile(source, dest) ++ elif compress_type == 'bz2': ++ bzipFile(source, dest) ++ elif compress_type == 'gz': ++ gzFile(source, dest) ++ else: ++ raise MDError, "Unknown compression type %s" % compress_type ++ ++def compressOpen(fn, mode='rb', compress_type=None): ++ ++ if not compress_type: ++ # we are readonly and we don't give a compress_type - then guess based on the file extension ++ compress_type = fn.split('.')[-1] ++ ++ if compress_type == 'xz': ++ return lzma.LZMAFile(fn, mode) ++ elif compress_type == 'bz2': ++ return bz2.BZ2File(fn, mode) ++ elif compress_type == 'gz': ++ return _gzipOpen(fn, mode) ++ else: ++ raise MDError, "Unknown compression type %s" % compress_type ++ + def returnFD(filename): + try: + fdno = os.open(filename, os.O_RDONLY) diff --git a/docs/createrepo.8 b/docs/createrepo.8 -index e3c4c3b..96b5bf8 100644 +index e3c4c3b..4734392 100644 --- a/docs/createrepo.8 +++ b/docs/createrepo.8 @@ -53,7 +53,8 @@ gullible). @@ -629,8 +1040,25 @@ index e3c4c3b..96b5bf8 100644 .br .IP "\fB\--split\fP" Run in split media mode. Rather than pass a single directory, take a set of +@@ -104,7 +105,15 @@ Tells createrepo to generate deltarpms and the delta metadata + paths to look for older pkgs to delta against. Can be specified multiple times + .IP "\fB\--num-deltas\fP int" + the number of older versions to make deltas against. Defaults to 1 +- ++.IP "\fB\--read-pkgs-list\fP READ_PKGS_LIST ++output the paths to the pkgs actually read useful with --update ++.IP "\fB\--max-delta-rpm-size\fP MAX_DELTA_RPM_SIZE ++max size of an rpm that to run deltarpm against (in bytes) ++.IP "\fB\--workers\fP WORKERS ++number of workers to spawn to read rpms ++.IP "\fB\--xz\fP ++use xz for repodata compression ++.IP + + .SH "EXAMPLES" + Here is an example of a repository with a groups file. Note that the diff --git a/genpkgmetadata.py b/genpkgmetadata.py -index 8c98191..512420b 100755 +index 8c98191..af0ecb4 100755 --- a/genpkgmetadata.py +++ b/genpkgmetadata.py @@ -100,6 +100,8 @@ def parse_args(args, conf): @@ -642,7 +1070,36 @@ index 8c98191..512420b 100755 parser.add_option("--distro", default=[], action="append", help="distro tag and optional cpeid: --distro" "'cpeid,textname'") parser.add_option("--content", default=[], dest='content_tags', -@@ -240,6 +242,7 @@ def main(args): +@@ -119,10 +121,15 @@ def parse_args(args, conf): + parser.add_option("--max-delta-rpm-size", default=100000000, + dest='max_delta_rpm_size', type='int', + help="max size of an rpm that to run deltarpm against (in bytes)") +- + parser.add_option("--workers", default=1, + dest='workers', type='int', + help="number of workers to spawn to read rpms") ++ parser.add_option("--xz", default=False, ++ action="store_true", ++ help="use xz for repodata compression") ++ parser.add_option("--compress-type", default=None, dest="compress_type", ++ help="which compression type to use") ++ + + (opts, argsleft) = parser.parse_args(args) + if len(argsleft) > 1 and not opts.split: +@@ -155,6 +162,11 @@ def parse_args(args, conf): + + if opts.nodatabase: + opts.database = False ++ ++ # xz is just a shorthand for compress_type ++ if opts.xz and not opts.compress_type: ++ opts.compress_type='xz' ++ + + # let's switch over to using the conf object - put all the opts into it + for opt in parser.option_list: +@@ -240,6 +252,7 @@ def main(args): if mdgen.checkTimeStamps(): if mdgen.conf.verbose: print _('repo is up to date') @@ -651,7 +1108,7 @@ index 8c98191..512420b 100755 if conf.profile: diff --git a/mergerepo.py b/mergerepo.py -index 05e5f5e..069a70b 100755 +index 05e5f5e..80cb1a8 100755 --- a/mergerepo.py +++ b/mergerepo.py @@ -18,6 +18,7 @@ @@ -662,7 +1119,17 @@ index 05e5f5e..069a70b 100755 from optparse import OptionParser #TODO: -@@ -77,9 +78,12 @@ def main(args): +@@ -47,6 +48,9 @@ def parse_args(args): + help="Do not merge group(comps) metadata") + parser.add_option("", "--noupdateinfo", default=False, action="store_true", + help="Do not merge updateinfo metadata") ++ parser.add_option("--compress-type", default=None, dest="compress_type", ++ help="which compression type to use") ++ + (opts, argsleft) = parser.parse_args(args) + + if len(opts.repos) < 2: +@@ -77,9 +81,14 @@ def main(args): rmbase.groups = False if opts.noupdateinfo: rmbase.updateinfo = False @@ -670,6 +1137,8 @@ index 05e5f5e..069a70b 100755 - rmbase.merge_repos() - rmbase.write_metadata() - ++ if opts.compress_type: ++ rmbase.mdconf.compress_type = opts.compress_type + try: + rmbase.merge_repos() + rmbase.write_metadata() @@ -680,7 +1149,7 @@ index 05e5f5e..069a70b 100755 if __name__ == "__main__": main(sys.argv[1:]) diff --git a/modifyrepo.py b/modifyrepo.py -index 17094a4..c3370e8 100755 +index 17094a4..153ad4d 100755 --- a/modifyrepo.py +++ b/modifyrepo.py @@ -1,11 +1,15 @@ @@ -700,7 +1169,7 @@ index 17094a4..c3370e8 100755 # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -@@ -20,6 +24,7 @@ +@@ -20,11 +24,12 @@ # (C) Copyright 2006 Red Hat, Inc. # Luke Macken # modified by Seth Vidal 2008 @@ -708,7 +1177,22 @@ index 17094a4..c3370e8 100755 import os import sys -@@ -49,6 +54,35 @@ class RepoMetadata: + from createrepo import __version__ +-from createrepo.utils import checksum_and_rename, GzipFile, MDError ++from createrepo.utils import checksum_and_rename, compressOpen, MDError + from yum.misc import checksum + + from yum.repoMDObject import RepoMD, RepoMDError, RepoData +@@ -39,6 +44,8 @@ class RepoMetadata: + self.repodir = os.path.abspath(repo) + self.repomdxml = os.path.join(self.repodir, 'repomd.xml') + self.checksum_type = 'sha256' ++ self.compress = False ++ self.compress_type='xz' + + if not os.path.exists(self.repomdxml): + raise MDError, '%s not found' % self.repomdxml +@@ -49,6 +56,35 @@ class RepoMetadata: except RepoMDError, e: raise MDError, 'Could not parse %s' % self.repomdxml @@ -744,19 +1228,44 @@ index 17094a4..c3370e8 100755 def add(self, metadata, mdtype=None): """ Insert arbitrary metadata into this repository. -@@ -78,9 +112,8 @@ class RepoMetadata: +@@ -63,8 +99,8 @@ class RepoMetadata: + mdname = 'updateinfo.xml' + elif isinstance(metadata, str): + if os.path.exists(metadata): +- if metadata.endswith('.gz'): +- oldmd = GzipFile(filename=metadata, mode='rb') ++ if metadata.split('.')[-1] in ('gz', 'bz2', 'xz'): ++ oldmd = compressOpen(metadata, mode='rb') + else: + oldmd = file(metadata, 'r') + md = oldmd.read() +@@ -75,14 +111,19 @@ class RepoMetadata: + else: + raise MDError, 'invalid metadata type' + ++ do_compress = False ## Compress the metadata and move it into the repodata - if not mdname.endswith('.gz'): - mdname += '.gz' +- if not mdname.endswith('.gz'): +- mdname += '.gz' - if not mdtype: - mdtype = mdname.split('.')[0] - ++ if self.compress or not mdname.split('.')[-1] in ('gz', 'bz2', 'xz'): ++ do_compress = True ++ mdname += '.' + self.compress_type + mdtype = self._get_mdtype(mdname, mdtype) + destmd = os.path.join(self.repodir, mdname) - newmd = GzipFile(filename=destmd, mode='wb') +- newmd = GzipFile(filename=destmd, mode='wb') ++ if do_compress: ++ newmd = compressOpen(destmd, mode='wb', compress_type=self.compress_type) ++ else: ++ newmd = open(destmd, 'wb') ++ newmd.write(md) -@@ -91,11 +124,8 @@ class RepoMetadata: + newmd.close() + print "Wrote:", destmd +@@ -91,11 +132,8 @@ class RepoMetadata: csum, destmd = checksum_and_rename(destmd, self.checksum_type) base_destmd = os.path.basename(destmd) @@ -770,7 +1279,7 @@ index 17094a4..c3370e8 100755 new_rd = RepoData() new_rd.type = mdtype -@@ -105,18 +135,28 @@ class RepoMetadata: +@@ -105,18 +143,28 @@ class RepoMetadata: new_rd.size = str(os.stat(destmd).st_size) new_rd.timestamp = str(os.stat(destmd).st_mtime) self.repoobj.repoData[new_rd.type] = new_rd @@ -811,22 +1320,30 @@ index 17094a4..c3370e8 100755 def main(args): -@@ -124,7 +164,9 @@ def main(args): +@@ -124,7 +172,13 @@ def main(args): # query options parser.add_option("--mdtype", dest='mdtype', help="specific datatype of the metadata, will be derived from the filename if not specified") - parser.usage = "modifyrepo [options] " + parser.add_option("--remove", action="store_true", + help="remove specified file from repodata") ++ parser.add_option("--compress", action="store_true", default=False, ++ help="compress the new repodata before adding it to the repo") ++ parser.add_option("--compress-type", dest='compress_type', default='xz', ++ help="compression format to use") + parser.usage = "modifyrepo [options] [--remove] " (opts, argsleft) = parser.parse_args(args) if len(argsleft) != 2: -@@ -137,6 +179,17 @@ def main(args): +@@ -137,11 +191,27 @@ def main(args): except MDError, e: print "Could not access repository: %s" % str(e) return 1 + ++ ++ repomd.compress = opts.compress ++ repomd.compress_type = opts.compress_type ++ + # remove + if opts.remove: + try: @@ -840,11 +1357,49 @@ index 17094a4..c3370e8 100755 try: repomd.add(metadata, mdtype=opts.mdtype) except MDError, e: + print "Could not add metadata from file %s: %s" % (metadata, str(e)) + return 1 ++ + + if __name__ == '__main__': + ret = main(sys.argv[1:]) diff --git a/worker.py b/worker.py -index eb35ef7..ab78d90 100755 +index eb35ef7..23c87a3 100755 --- a/worker.py +++ b/worker.py -@@ -80,11 +80,14 @@ def main(args): +@@ -5,6 +5,7 @@ import yum + import createrepo + import os + import rpmUtils ++import re + from optparse import OptionParser + + +@@ -23,6 +24,8 @@ def main(args): + parser = OptionParser() + parser.add_option('--tmpmdpath', default=None, + help="path where the outputs should be dumped for this worker") ++ parser.add_option('--pkglist', default=None, ++ help="file to read the pkglist from in lieu of all of them on the cli") + parser.add_option("--pkgoptions", default=[], action='append', + help="pkgoptions in the format of key=value") + parser.add_option("--quiet", default=False, action='store_true', +@@ -68,7 +71,13 @@ def main(args): + fl = open(opts.tmpmdpath + '/filelists.xml' , 'w') + other = open(opts.tmpmdpath + '/other.xml' , 'w') + +- ++ if opts.pkglist: ++ for line in open(opts.pkglist,'r').readlines(): ++ line = line.strip() ++ if re.match('^\s*\#.*', line) or re.match('^\s*$', line): ++ continue ++ pkgs.append(line) ++ + for pkgfile in pkgs: + pkgpath = reldir + '/' + pkgfile + if not os.path.exists(pkgpath): +@@ -80,11 +89,14 @@ def main(args): print "reading %s" % (pkgfile) pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath, diff --git a/createrepo.spec b/createrepo.spec index e27cfe8..cbd1a61 100644 --- a/createrepo.spec +++ b/createrepo.spec @@ -3,7 +3,7 @@ Summary: Creates a common metadata repository Name: createrepo Version: 0.9.9 -Release: 6%{?dist} +Release: 7%{?dist} License: GPLv2 Group: System Environment/Base Source: %{name}-%{version}.tar.gz @@ -47,6 +47,10 @@ rm -rf $RPM_BUILD_ROOT %{python_sitelib}/createrepo %changelog +* Mon Oct 24 2011 Seth Vidal - 0.9.9-7 +- latest upstream +- --compress-type among other deals. + * Fri Jul 29 2011 Seth Vidal - 0.9.9-6 - latest upstream - fixes bugs: 713747, 581632, 581628