diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d347de --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/virt-bootstrap-*.tar.gz +/x86_64 +/.build-*.log +*.rpm diff --git a/0001-docker-source-Avoid-skopeo-copy-in-cache.patch b/0001-docker-source-Avoid-skopeo-copy-in-cache.patch new file mode 100644 index 0000000..8a707e8 --- /dev/null +++ b/0001-docker-source-Avoid-skopeo-copy-in-cache.patch @@ -0,0 +1,88 @@ +From 2153dcdb87571f8ed4ad90e187543e4dd6c89cb2 Mon Sep 17 00:00:00 2001 +From: Radostin Stoyanov +Date: Thu, 7 Dec 2017 20:42:28 +0000 +Subject: [PATCH] docker-source: Avoid skopeo copy in cache + +The `skopeo copy` command has changed it's behaviour to keep only a files for +single container image per directory. To get around this and keep cache of +downloaded images is used temporary destination directory for 'skopeo copy' +and image files are then moved in the cache folder. + +(cherry picked from commit 76a945f9f2b88407bac8f7a8cc0d1464db183f21) +--- + src/virtBootstrap/sources/docker_source.py | 16 +++++++++++++--- + src/virtBootstrap/utils.py | 14 ++++++++++++++ + 2 files changed, 27 insertions(+), 3 deletions(-) + +diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py +index ef72d64..32252b2 100644 +--- a/src/virtBootstrap/sources/docker_source.py ++++ b/src/virtBootstrap/sources/docker_source.py +@@ -133,10 +133,16 @@ class DockerSource(object): + """ + Download image layers using "skopeo copy". + """ ++ ++ if self.no_cache: ++ dest_dir = self.images_dir ++ else: ++ dest_dir = utils.get_image_dir(no_cache=True) ++ + # Note: we don't want to expose --src-cert-dir to users as + # they should place the certificates in the system + # folders for broader enablement +- skopeo_copy = ["skopeo", "copy", self.url, "dir:" + self.images_dir] ++ skopeo_copy = ["skopeo", "copy", self.url, "dir:" + dest_dir] + + if self.insecure: + skopeo_copy.append('--src-tls-verify=false') +@@ -146,8 +152,12 @@ class DockerSource(object): + self.progress("Downloading container image", value=0, logger=logger) + # Run "skopeo copy" command + self.read_skopeo_progress(skopeo_copy) +- # Remove the manifest file as it is not needed +- os.remove(os.path.join(self.images_dir, "manifest.json")) ++ ++ if not self.no_cache: ++ os.remove(os.path.join(dest_dir, "manifest.json")) ++ os.remove(os.path.join(dest_dir, "version")) ++ utils.copytree(dest_dir, self.images_dir) ++ shutil.rmtree(dest_dir) + + def parse_output(self, proc): + """ +diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py +index 84d1629..8dbf948 100644 +--- a/src/virtBootstrap/utils.py ++++ b/src/virtBootstrap/utils.py +@@ -32,6 +32,7 @@ import subprocess + import sys + import tempfile + import logging ++import shutil + + import guestfs + import passlib.hosts +@@ -350,6 +351,19 @@ def get_mime_type(path): + return output.read().decode('utf-8').split()[1] + + ++def copytree(src, dst, symlinks=False, ignore=None): ++ """ ++ Copy an entire directory of files into an existing directory. ++ """ ++ for item in os.listdir(src): ++ src_item = os.path.join(src, item) ++ dst_item = os.path.join(dst, item) ++ if os.path.isdir(src_item): ++ shutil.copytree(src_item, dst_item, symlinks, ignore) ++ else: ++ shutil.copy2(src_item, dst_item) ++ ++ + def get_image_dir(no_cache=False): + """ + Get the directory where image layers are stored. +-- +2.17.0 + diff --git a/0002-docker-source-Get-list-of-layers-without-raw.patch b/0002-docker-source-Get-list-of-layers-without-raw.patch new file mode 100644 index 0000000..eb61e1f --- /dev/null +++ b/0002-docker-source-Get-list-of-layers-without-raw.patch @@ -0,0 +1,182 @@ +From 90933bf858be3c2aeb6be83d8177f0af3955c9c6 Mon Sep 17 00:00:00 2001 +From: Radostin Stoyanov +Date: Fri, 15 Dec 2017 23:05:33 +0000 +Subject: [PATCH] docker-source: Get list of layers without `--raw` + +When `skopeo inspect --raw docker://feodra` is used the returned +manifest content contains a list with manifests for specific +platforms [1] rather than a list with layers. + +By using `skopeo inpect docker://fedora` the correct manifest +content is retrieved and a list with layers is provided. In addition, +skopeo handles the difference between schemaVersion 1 and 2. + +[1] https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list-field-descriptions + +(cherry picked from commit a223b15938237665fe135cd710f4301ea23bf85d) +--- + src/virtBootstrap/sources/docker_source.py | 29 ++++------ + tests/docker_source.py | 66 ++++------------------ + 2 files changed, 22 insertions(+), 73 deletions(-) + +diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py +index 32252b2..5969bf0 100644 +--- a/src/virtBootstrap/sources/docker_source.py ++++ b/src/virtBootstrap/sources/docker_source.py +@@ -73,7 +73,7 @@ class DockerSource(object): + self.no_cache = kwargs.get('no_cache', False) + self.progress = kwargs['progress'].update_progress + self.images_dir = utils.get_image_dir(self.no_cache) +- self.manifest = None ++ self.image_details = None + self.layers = [] + self.checksums = [] + +@@ -87,30 +87,25 @@ class DockerSource(object): + Retrive manifest from registry and get layers' digest, + sum_type, size and file_path in a list. + """ +- self.manifest = utils.get_image_details(self.url, raw=True, ++ image_details = utils.get_image_details(self.url, raw=False, + insecure=self.insecure, + username=self.username, + password=self.password) + +- if self.manifest['schemaVersion'] == 1: +- layers_list = self.manifest['fsLayers'][::-1] +- digest_field = 'blobSum' +- elif self.manifest['schemaVersion'] == 2: +- layers_list = self.manifest['layers'] +- digest_field = 'digest' +- else: +- raise ValueError('Unsupported manifest schema.') ++ if not 'Layers' in image_details or not image_details['Layers']: ++ raise ValueError('No image layers.') + +- for layer in layers_list: +- # Store checksums of layers +- layer_digest = layer[digest_field] ++ # Layers are in order: ++ # - root layer first, and then successive layered layers ++ # Ref: https://github.com/containers/image/blob/master/image/oci.go ++ for layer_digest in image_details['Layers']: + sum_type, layer_sum = layer_digest.split(':') +- self.checksums.append([sum_type, layer_sum]) ++ self.checksums.append([sum_type, layer_sum]) # Store checksums + +- # Store file path and size of each layer ++ # Layers are tar files with hashsum used as name + file_path = os.path.join(self.images_dir, layer_sum + '.tar') +- layer_size = layer.get('size', None) +- self.layers.append([file_path, layer_size]) ++ # Store 'file path' and set placeholder for 'size' ++ self.layers.append([file_path, None]) + + def gen_valid_uri(self, uri): + """ +diff --git a/tests/docker_source.py b/tests/docker_source.py +index 9090988..c8f4e08 100644 +--- a/tests/docker_source.py ++++ b/tests/docker_source.py +@@ -90,11 +90,8 @@ class CreateLayers(object): + """ + return { + "schemaVersion": 2, +- "layers": [ +- { +- "digest": +- "sha256:" + os.path.basename(layer).split('.')[0] +- } ++ "Layers": [ ++ "sha256:" + os.path.basename(layer).split('.')[0] + for layer in self.layers + ] + } +@@ -340,7 +337,7 @@ class TestDockerSource(unittest.TestCase): + 'progress': mock.Mock() + } + +- manifest = {'schemaVersion': 2, 'layers': []} ++ manifest = {'schemaVersion': 2, 'Layers': ['sha256:a7050fc1']} + (src_instance, + m_uri, m_utils) = self._mock_retrieve_layers_info(manifest, + src_kwargs) +@@ -349,7 +346,7 @@ class TestDockerSource(unittest.TestCase): + 'insecure': src_instance.insecure, + 'username': src_instance.username, + 'password': src_instance.password, +- 'raw': True ++ 'raw': False + } + m_utils['get_image_details'].assert_called_once_with(m_uri(), **kwargs) + +@@ -365,11 +362,11 @@ class TestDockerSource(unittest.TestCase): + } + + manifest = { +- 'schemaVersion': 1, +- 'fsLayers': [ +- {'blobSum': 'sha256:75c416ea'}, +- {'blobSum': 'sha256:c6ff40b6'}, +- {'blobSum': 'sha256:a7050fc1'} ++ 'schemaVersion': 2, ++ 'Layers': [ ++ 'sha256:a7050fc1', ++ 'sha256:c6ff40b6', ++ 'sha256:75c416ea' + ] + } + +@@ -382,47 +379,4 @@ class TestDockerSource(unittest.TestCase): + with mock.patch('os.path.getsize') as m_getsize: + m_getsize.return_value = None + src_instance = self._mock_retrieve_layers_info(manifest, kwargs)[0] +- self.assertEqual(src_instance.layers, expected_result) +- +- def test_retrieve_layers_info_schema_version_2(self): +- """ +- Ensures that retrieve_layers_info() extracts the layers' information +- from manifest with schema version 2 a list with format: +- ["digest", "sum_type", "file_path", "size"]. +- """ +- kwargs = { +- 'uri': '', +- 'progress': mock.Mock() +- } +- +- manifest = { +- 'schemaVersion': 2, +- "layers": [ +- {"size": 47103294, "digest": "sha256:75c416ea"}, +- {"size": 814, "digest": "sha256:c6ff40b6"}, +- {"size": 513, "digest": "sha256:a7050fc1"} +- ] +- } +- +- expected_result = [ +- ['/images_path/75c416ea.tar', 47103294], +- ['/images_path/c6ff40b6.tar', 814], +- ['/images_path/a7050fc1.tar', 513] +- ] +- +- src_instance = self._mock_retrieve_layers_info(manifest, kwargs)[0] +- self.assertEqual(src_instance.layers, expected_result) +- +- def test_retrieve_layers_info_raise_error_on_invalid_schema_version(self): +- """ +- Ensures that retrieve_layers_info() calls get_image_details() +- with all passed arguments. +- """ +- kwargs = { +- 'uri': '', +- 'progress': mock.Mock() +- } +- +- manifest = {'schemaVersion': 3} +- with self.assertRaises(ValueError): +- self._mock_retrieve_layers_info(manifest, kwargs) ++ self.assertEqual(src_instance.layers, expected_result) +\ No newline at end of file +-- +2.17.0 + diff --git a/0003-docker-source-Support-blobs-without-.tar-ext.patch b/0003-docker-source-Support-blobs-without-.tar-ext.patch new file mode 100644 index 0000000..8fc86f5 --- /dev/null +++ b/0003-docker-source-Support-blobs-without-.tar-ext.patch @@ -0,0 +1,89 @@ +From 3be034a2669e779b13d16184394fea0373ef34c2 Mon Sep 17 00:00:00 2001 +From: Radostin Stoyanov +Date: Tue, 24 Apr 2018 09:03:34 +0100 +Subject: [PATCH] docker-source: Support blobs without .tar ext + +Since skopeo v0.1.29 [1] blobs are saved without the .tar extension. +This commit changes the docker-source module to handle both cases (with +or without .tar extension) + + [1] commit: projectatomic/skopeo@43acc74 + Fix skopeo tests with changes to dir transport + + The dir transport has been changed to save the blobs without the .tar extension + Fixes the skopeo tests failing due to this change + +Signed-off-by: Radostin Stoyanov +(cherry picked from commit f389756a192b6b6ad0951856d8681cc2077ccf6b) +--- + src/virtBootstrap/sources/docker_source.py | 21 +++++++++++++++++++-- + tests/docker_source.py | 6 +++--- + 2 files changed, 22 insertions(+), 5 deletions(-) + +diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py +index 5969bf0..96a558a 100644 +--- a/src/virtBootstrap/sources/docker_source.py ++++ b/src/virtBootstrap/sources/docker_source.py +@@ -103,7 +103,7 @@ class DockerSource(object): + self.checksums.append([sum_type, layer_sum]) # Store checksums + + # Layers are tar files with hashsum used as name +- file_path = os.path.join(self.images_dir, layer_sum + '.tar') ++ file_path = os.path.join(self.images_dir, layer_sum) + # Store 'file path' and set placeholder for 'size' + self.layers.append([file_path, None]) + +@@ -154,6 +154,17 @@ class DockerSource(object): + utils.copytree(dest_dir, self.images_dir) + shutil.rmtree(dest_dir) + ++ # Old versions of skopeo use '.tar' extension to blobs. ++ # Make sure we use the correct file name. ++ for i in range(len(self.layers)): ++ path = self.layers[i][0] ++ if not os.path.exists(path): ++ if os.path.exists(path + '.tar'): ++ self.layers[i][0] += '.tar' ++ else: ++ raise ValueError('Blob %s does not exist.' % path) ++ ++ + def parse_output(self, proc): + """ + Read stdout from skopeo's process asynchconosly. +@@ -254,8 +265,14 @@ class DockerSource(object): + sum_type, sum_expected = checksum + + logger.debug("Checking layer: %s", path) +- if not (os.path.exists(path) ++ if (os.path.exists(path) + and utils.checksum(path, sum_type, sum_expected)): ++ continue ++ if (not path.endswith('.tar') ++ and os.path.exists(path + '.tar') ++ and utils.checksum(path + '.tar', sum_type, sum_expected)): ++ self.layers[index][0] += '.tar' ++ else: + return False + return True + +diff --git a/tests/docker_source.py b/tests/docker_source.py +index c8f4e08..1b13948 100644 +--- a/tests/docker_source.py ++++ b/tests/docker_source.py +@@ -371,9 +371,9 @@ class TestDockerSource(unittest.TestCase): + } + + expected_result = [ +- ['/images_path/a7050fc1.tar', None], +- ['/images_path/c6ff40b6.tar', None], +- ['/images_path/75c416ea.tar', None] ++ ['/images_path/a7050fc1', None], ++ ['/images_path/c6ff40b6', None], ++ ['/images_path/75c416ea', None] + ] + + with mock.patch('os.path.getsize') as m_getsize: +-- +2.17.0 + diff --git a/0004-safe_untar-Check-for-permissions-to-set-attribs.patch b/0004-safe_untar-Check-for-permissions-to-set-attribs.patch new file mode 100644 index 0000000..eb4f8a6 --- /dev/null +++ b/0004-safe_untar-Check-for-permissions-to-set-attribs.patch @@ -0,0 +1,44 @@ +From 7c79077ffa34ebe61b7c720670dd3c637d5033f8 Mon Sep 17 00:00:00 2001 +From: Radostin Stoyanov +Date: Tue, 24 Apr 2018 09:18:26 +0100 +Subject: [PATCH] safe_untar: Check for permissions to set attribs + +Make sure we have permissions to restore file extended attributes. + +[1] ... all processes have read access to extended security attributes, +and write access is limited to processes that have the CAP_SYS_ADMIN +capability. + +[2] The file owner and processes capable of CAP_FOWNER are granted the +right to modify ACLs of a file. This is analogous to the permissions +required for accessing the file mode. (On current Linux systems, root +is the only user with the CAP_FOWNER capability.) + +[1] https://linux.die.net/man/5/attr +[2] https://linux.die.net/man/1/setfacl + +Signed-off-by: Radostin Stoyanov +(cherry picked from commit bc2744cea7800f7b84e1731e367e75b0f2d60d48) +--- + src/virtBootstrap/utils.py | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py +index 8dbf948..bf3c611 100644 +--- a/src/virtBootstrap/utils.py ++++ b/src/virtBootstrap/utils.py +@@ -275,6 +275,11 @@ def safe_untar(src, dest): + # qemu:/// driver. This flag must not be used outside virt-sandbox. + params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*', + '--overwrite', '--absolute-names'] ++ ++ # Preserve file attributes following the specification in ++ # https://github.com/opencontainers/image-spec/blob/master/layer.md ++ if os.geteuid() == 0: ++ params.extend(['--acls', '--xattrs', '--selinux']) + execute(virt_sandbox + params) + + +-- +2.17.0 + diff --git a/README.md b/README.md deleted file mode 100644 index 7145763..0000000 --- a/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# virt-bootstrap - -The virt-bootstrap package \ No newline at end of file diff --git a/sources b/sources new file mode 100644 index 0000000..dd7f33f --- /dev/null +++ b/sources @@ -0,0 +1 @@ +SHA512 (virt-bootstrap-1.0.0.tar.gz) = 76aa801f4c448d3e7df6c282e7765e5b55f4fa4d00f8a4edc6d83b285ae87eb06f01ebad6471e1d4c8318e42176c24956bd178813e288ee5598ec2d568dba4c6 diff --git a/virt-bootstrap.spec b/virt-bootstrap.spec new file mode 100644 index 0000000..a24cfa4 --- /dev/null +++ b/virt-bootstrap.spec @@ -0,0 +1,78 @@ +Name: virt-bootstrap +Version: 1.0.0 +Release: 2%{?dist} +Summary: System container rootfs creation tool + +License: GPLv3+ +URL: https://github.com/virt-manager/virt-bootstrap +Source0: http://virt-manager.org/download/sources/virt-bootstrap/%{name}-%{version}.tar.gz + +# Upstream patches +Patch0001: 0001-docker-source-Avoid-skopeo-copy-in-cache.patch +Patch0002: 0002-docker-source-Get-list-of-layers-without-raw.patch +Patch0003: 0003-docker-source-Support-blobs-without-.tar-ext.patch +Patch0004: 0004-safe_untar-Check-for-permissions-to-set-attribs.patch + +BuildRequires: /usr/bin/pod2man +BuildRequires: /usr/bin/git +BuildRequires: python3-devel +BuildRequires: python3-libguestfs +BuildRequires: python3-passlib +BuildRequires: python3-setuptools +BuildRequires: fdupes + +Requires: python3-libguestfs +Requires: python3-passlib +Requires: skopeo +Requires: libvirt-sandbox + +BuildArch: noarch + +%description +Provides a way to create the root file system to use for +libvirt containers. + +%prep +%autosetup -S git + +%build +%py3_build + +%install +%py3_install +%fdupes %{buildroot}%{_prefix} + +# Replace '#!/usr/bin/env python3' with '#!/usr/bin/python3' +# The format is ideal for upstream, but not a distro. See: +# https://fedoraproject.org/wiki/Features/SystemPythonExecutablesUseSystemPython +for f in $(find %{buildroot} -type f -executable -print); do + sed -i '1 s/^#!\/usr\/bin\/env python3/#!%{__python3}/' $f || : +done + +# Delete '#!/usr/bin/env python' +# The format is ideal for upstream, but not a distro. See: +# https://fedoraproject.org/wiki/Features/SystemPythonExecutablesUseSystemPython +for f in $(find %{buildroot} -type f \! -executable -print); do + sed -i '/^#!\/usr\/bin\/env python/d' $f || : +done + +%files +%license LICENSE +%doc README.md ChangeLog AUTHORS +%{_bindir}/virt-bootstrap +%{python3_sitelib}/virtBootstrap +%{python3_sitelib}/virt_bootstrap-*.egg-info +%{_mandir}/man1/virt-bootstrap* + +%changelog +* Thu May 17 2018 Fabiano Fidêncio - 1.0.0-2 +- Set "BuildArch: noarch" as this is an arch independent package +- Drop "Buildroot" tag as it's obsolete +- Drop "%%defattr" tag as it's obsolete +- Add "BuildRequires: /usr/bin/git" (due to %%autosetup -S git) +- Add a note to make clear that the patches are backported from upstream +- Replace '#!/usr/bin/env python3' with '#!/usr/bin/python3' +- Delete '#!/usr/bin/env python' from non executable files + +* Wed May 16 2018 Fabiano Fidêncio - 1.0.0-1 +- Initial release