From f3f4d4e80c58e86a85cdcb8ae4fef842459221b7 Mon Sep 17 00:00:00 2001 From: Benjamin A. Beasley Date: Aug 11 2022 13:11:02 +0000 Subject: Backport upstream PR#1709 (fix RHBZ#2113133, fix RHBZ#2090763, close RHBZ#2094317) --- diff --git a/1709.patch b/1709.patch new file mode 100644 index 0000000..6482add --- /dev/null +++ b/1709.patch @@ -0,0 +1,532 @@ +From d323a8700a4bfc7d653df14887cee1bfe87d3e54 Mon Sep 17 00:00:00 2001 +From: Tristan van Berkom +Date: Mon, 1 Aug 2022 21:59:33 +0900 +Subject: [PATCH 1/4] requirements: Update requirements removing the + ruamel.yaml restriction + +--- + requirements/cov-requirements.txt | 6 +++--- + requirements/dev-requirements.txt | 23 ++++++++++++----------- + requirements/plugin-requirements.txt | 2 +- + requirements/requirements.in | 2 +- + requirements/requirements.txt | 18 +++++++++--------- + 5 files changed, 26 insertions(+), 25 deletions(-) + +diff --git a/requirements/cov-requirements.txt b/requirements/cov-requirements.txt +index e52fbdfe4..c092d2344 100644 +--- a/requirements/cov-requirements.txt ++++ b/requirements/cov-requirements.txt +@@ -1,11 +1,11 @@ + coverage==4.5.4 + pytest-cov==2.10.1 + ## The following requirements were added by pip freeze: +-attrs==21.4.0 ++attrs==22.1.0 + iniconfig==1.1.1 + packaging==21.3 + pluggy==1.0.0 + py==1.11.0 +-pyparsing==3.0.7 +-pytest==7.0.1 ++pyparsing==3.0.9 ++pytest==7.1.2 + tomli==2.0.1 +diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt +index 109a5d2c3..84d78261c 100644 +--- a/requirements/dev-requirements.txt ++++ b/requirements/dev-requirements.txt +@@ -1,26 +1,27 @@ + pep8==1.7.1 +-pylint==2.12.2 +-pytest==7.0.1 +-pytest-datafiles==2.0 ++pylint==2.14.5 ++pytest==7.1.2 ++pytest-datafiles==2.0.1 + pytest-env==0.6.2 + pytest-xdist==2.5.0 + pytest-timeout==2.1.0 + pyftpdlib==1.5.6 + ## The following requirements were added by pip freeze: +-astroid==2.9.3 +-attrs==21.4.0 ++astroid==2.11.7 ++attrs==22.1.0 ++dill==0.3.5.1 + execnet==1.9.0 + iniconfig==1.1.1 + isort==5.10.1 + lazy-object-proxy==1.7.1 +-mccabe==0.6.1 ++mccabe==0.7.0 + packaging==21.3 +-platformdirs==2.5.0 ++platformdirs==2.5.2 + pluggy==1.0.0 + py==1.11.0 +-pyparsing==3.0.7 ++pyparsing==3.0.9 + pytest-forked==1.4.0 +-toml==0.10.2 + tomli==2.0.1 +-typing_extensions==4.1.1 +-wrapt==1.13.3 ++tomlkit==0.11.1 ++typing-extensions==4.3.0 ++wrapt==1.14.1 +diff --git a/requirements/plugin-requirements.txt b/requirements/plugin-requirements.txt +index 0ab7e2b08..33f59b1f2 100644 +--- a/requirements/plugin-requirements.txt ++++ b/requirements/plugin-requirements.txt +@@ -1,2 +1,2 @@ +-arpy==2.2.0 ++arpy==2.3.0 + ## The following requirements were added by pip freeze: +diff --git a/requirements/requirements.in b/requirements/requirements.in +index d810ba990..eda65c0c6 100644 +--- a/requirements/requirements.in ++++ b/requirements/requirements.in +@@ -4,6 +4,6 @@ jinja2 >= 2.10 + pluginbase + protobuf >= 3.19 + psutil +-ruamel.yaml < 0.17 ++ruamel.yaml + setuptools + ujson +diff --git a/requirements/requirements.txt b/requirements/requirements.txt +index 030c57b09..9044ca1ba 100644 +--- a/requirements/requirements.txt ++++ b/requirements/requirements.txt +@@ -1,13 +1,13 @@ +-click==8.0.3 +-grpcio==1.43.0 +-Jinja2==3.0.3 ++click==8.1.3 ++grpcio==1.48.0 ++Jinja2==3.1.2 + pluginbase==1.0.1 +-protobuf==3.19.4 +-psutil==5.9.0 +-ruamel.yaml==0.16.13 +-setuptools==59.6.0 +-ujson==5.1.0 ++protobuf==4.21.4 ++psutil==5.9.1 ++ruamel.yaml==0.17.21 ++setuptools==44.1.1 ++ujson==5.4.0 + ## The following requirements were added by pip freeze: +-MarkupSafe==2.0.1 ++MarkupSafe==2.1.1 + ruamel.yaml.clib==0.2.6 + six==1.16.0 + +From 1f80db8a80ebb9f2fdde1b85a9a8cf4db63d3e9f Mon Sep 17 00:00:00 2001 +From: Tristan van Berkom +Date: Mon, 1 Aug 2022 23:04:56 +0900 +Subject: [PATCH 2/4] .pylintrc and other sources: Fix linting errors from + upgrading pylint + +--- + .pylintrc | 14 +++----------- + buildstream/element.py | 4 ++-- + buildstream/source.py | 4 ++-- + buildstream/utils.py | 6 +++--- + 4 files changed, 10 insertions(+), 18 deletions(-) + +diff --git a/.pylintrc b/.pylintrc +index 594187fd7..e48d35632 100644 +--- a/.pylintrc ++++ b/.pylintrc +@@ -3,7 +3,7 @@ + # A comma-separated list of package or module names from where C extensions may + # be loaded. Extensions are loading into the active Python interpreter and may + # run arbitrary code +-extension-pkg-whitelist= ++extension-pkg-whitelist=ujson + + # Add files or directories to the blacklist. They should be base names, not + # paths. +@@ -68,14 +68,13 @@ confidence= + # be nice, but too much work atm. + # + +-disable=##################################### ++disable=, ++ ##################################### + # Messages that are of no use to us # + ##################################### +- , + consider-using-f-string, + fixme, + missing-docstring, +- no-self-use, + no-else-return, + protected-access, + too-few-public-methods, +@@ -422,13 +421,6 @@ max-line-length=119 + # Maximum number of lines in a module + max-module-lines=1000 + +-# List of optional constructs for which whitespace checking is disabled. `dict- +-# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +-# `trailing-comma` allows a space between comma and closing bracket: (a, ). +-# `empty-line` allows space-only lines. +-no-space-check=trailing-comma, +- dict-separator +- + # Allow the body of a class to be on the same line as the declaration if body + # contains single statement. + single-line-class-stmt=no +diff --git a/buildstream/element.py b/buildstream/element.py +index 1d4208ce9..5b51d4321 100644 +--- a/buildstream/element.py ++++ b/buildstream/element.py +@@ -1551,9 +1551,9 @@ def cleanup_rootdir(): + self.__staged_sources_directory.lstrip(os.sep)) + try: + utils.copy_files(workspace.get_absolute_path(), sandbox_path) +- except UtilError as e: ++ except UtilError as err: + self.warn("Failed to preserve workspace state for failed build sysroot: {}" +- .format(e)) ++ .format(err)) + + raise + +diff --git a/buildstream/source.py b/buildstream/source.py +index bfa1ee5c0..cd717f1f2 100644 +--- a/buildstream/source.py ++++ b/buildstream/source.py +@@ -1023,7 +1023,7 @@ def __do_fetch(self, **kwargs): + success = True + break + if not success: +- raise last_error ++ raise last_error # pylint: disable=used-before-assignment + else: + alias = self._get_alias() + if self.__first_pass: +@@ -1070,7 +1070,7 @@ def __do_track(self, **kwargs): + last_error = e + continue + return ref +- raise last_error ++ raise last_error # pylint: disable=used-before-assignment + + # Ensures a fully constructed path and returns it + def __ensure_directory(self, directory): +diff --git a/buildstream/utils.py b/buildstream/utils.py +index f6fc2e70e..57e1b8dc0 100644 +--- a/buildstream/utils.py ++++ b/buildstream/utils.py +@@ -338,12 +338,12 @@ def safe_remove(path): + + try: + os.rmdir(path) +- except OSError as e: +- if e.errno == errno.ENOTEMPTY: ++ except OSError as err: ++ if err.errno == errno.ENOTEMPTY: + return False + else: + raise UtilError("Failed to remove '{}': {}" +- .format(path, e)) from e ++ .format(path, err)) from err + + return True + + +From c727ec9c073883a05405788559375b546ad0b771 Mon Sep 17 00:00:00 2001 +From: Tristan van Berkom +Date: Mon, 1 Aug 2022 23:04:08 +0900 +Subject: [PATCH 3/4] _yaml.py, _frontend/widget.py: Fix deprecation warnings + from new ruamel.yaml + +Fixes #1623 +--- + buildstream/_frontend/widget.py | 17 ++----- + buildstream/_yaml.py | 82 +++++++++++++++++++++++++++------ + tests/testutils/runcli.py | 8 ++-- + 3 files changed, 76 insertions(+), 31 deletions(-) + +diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py +index 6721beaa1..c39919922 100644 +--- a/buildstream/_frontend/widget.py ++++ b/buildstream/_frontend/widget.py +@@ -24,7 +24,6 @@ + import re + import textwrap + import click +-from ruamel import yaml + + from . import Profile + from .. import Element, Consistency +@@ -381,30 +380,22 @@ def show_pipeline(self, dependencies, format_): + # Element configuration + if "%{config" in format_: + config = _yaml.node_sanitize(element._Element__config) +- line = p.fmt_subst( +- line, 'config', +- yaml.round_trip_dump(config, default_flow_style=False, allow_unicode=True)) ++ line = p.fmt_subst(line, 'config', _yaml.dump_string(config)) + + # Variables + if "%{vars" in format_: + variables = dict(element._Element__variables) +- line = p.fmt_subst( +- line, 'vars', +- yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True)) ++ line = p.fmt_subst(line, 'vars', _yaml.dump_string(variables)) + + # Environment + if "%{env" in format_: + environment = _yaml.node_sanitize(element._Element__environment) +- line = p.fmt_subst( +- line, 'env', +- yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) ++ line = p.fmt_subst(line, 'env', _yaml.dump_string(environment)) + + # Public + if "%{public" in format_: + environment = _yaml.node_sanitize(element._Element__public) +- line = p.fmt_subst( +- line, 'public', +- yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) ++ line = p.fmt_subst(line, 'public', _yaml.dump_string(environment)) + + # Workspaced + if "%{workspaced" in format_: +diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py +index 1c3e95d34..beb3d3f20 100644 +--- a/buildstream/_yaml.py ++++ b/buildstream/_yaml.py +@@ -20,6 +20,7 @@ + import sys + import collections + import string ++from io import StringIO + from copy import deepcopy + from contextlib import ExitStack + from pathlib import Path +@@ -29,9 +30,30 @@ + from ruamel.yaml.constructor import RoundTripConstructor + from ._exceptions import LoadError, LoadErrorReason + ++ ++# SanitizedDict is an OrderedDict that is dumped as unordered mapping. ++# This provides deterministic output for unordered mappings. ++# ++class SanitizedDict(collections.OrderedDict): ++ pass ++ ++ + # This overrides the ruamel constructor to treat everything as a string + RoundTripConstructor.add_constructor('tag:yaml.org,2002:int', RoundTripConstructor.construct_yaml_str) + RoundTripConstructor.add_constructor('tag:yaml.org,2002:float', RoundTripConstructor.construct_yaml_str) ++RoundTripConstructor.add_constructor('tag:yaml.org,2002:null', RoundTripConstructor.construct_yaml_str) ++ ++ ++# Represent simple types as strings ++def represent_as_str(self, value): ++ return self.represent_str(str(value)) ++ ++ ++RoundTripRepresenter.add_representer(SanitizedDict, SafeRepresenter.represent_dict) ++RoundTripRepresenter.add_representer(type(None), represent_as_str) ++RoundTripRepresenter.add_representer(int, represent_as_str) ++RoundTripRepresenter.add_representer(float, represent_as_str) ++ + + # We store information in the loaded yaml on a DictProvenance + # stored in all dictionaries under this key +@@ -207,13 +229,37 @@ def load(filename, shortname=None, copy_tree=False, *, project=None): + "{} is a directory. bst command expects a .bst file." + .format(filename)) from e + ++# A function to get the roundtrip yaml handle ++# ++# Args: ++# write (bool): Whether we intend to write ++# ++def prepare_roundtrip_yaml(write=False): ++ yml = yaml.YAML() ++ yml.preserve_quotes=True ++ ++ # For each of YAML 1.1 and 1.2, force everything to be a plain string ++ ++ for version in [(1, 1), (1, 2), None]: ++ yml.resolver.add_version_implicit_resolver( ++ version, ++ 'tag:yaml.org,2002:str', ++ yaml.util.RegExp(r'.*'), ++ None) ++ ++ # When writing, we want to represent boolean as strings ++ if write: ++ yml.representer.add_representer(bool, represent_as_str) ++ ++ return yml + + # Like load(), but doesnt require the data to be in a file + # + def load_data(data, file=None, copy_tree=False): + ++ yml = prepare_roundtrip_yaml() + try: +- contents = yaml.load(data, yaml.loader.RoundTripLoader, preserve_quotes=True) ++ contents = yml.load(data) + except (yaml.scanner.ScannerError, yaml.composer.ComposerError, yaml.parser.ParserError) as e: + raise LoadError(LoadErrorReason.INVALID_YAML, + "Malformed YAML:\n\n{}\n\n{}\n".format(e.problem, e.problem_mark)) from e +@@ -230,6 +276,27 @@ def load_data(data, file=None, copy_tree=False): + return node_decorated_copy(file, contents, copy_tree=copy_tree) + + ++# Dumps a previously loaded YAML node to a file handle ++# ++def dump_file_handle(node, fh): ++ yml = prepare_roundtrip_yaml(write=True) ++ yml.dump(node, fh) ++ ++ ++# Dumps a previously loaded YAML node to a file ++# ++# Args: ++# node (dict): A node previously loaded with _yaml.load() above ++# ++# Returns: ++# (str): The generated string ++# ++def dump_string(node): ++ with StringIO() as f: ++ dump_file_handle(node, f) ++ return f.getvalue() ++ ++ + # Dumps a previously loaded YAML node to a file + # + # Args: +@@ -243,7 +310,7 @@ def dump(node, filename=None): + f = stack.enter_context(utils.save_file_atomic(filename, 'w')) + else: + f = sys.stdout +- yaml.round_trip_dump(node, f) ++ dump_file_handle(node, f) + + + # node_decorated_copy() +@@ -905,17 +972,6 @@ def composite(target, source): + e.actual_type.__name__)) from e + + +-# SanitizedDict is an OrderedDict that is dumped as unordered mapping. +-# This provides deterministic output for unordered mappings. +-# +-class SanitizedDict(collections.OrderedDict): +- pass +- +- +-RoundTripRepresenter.add_representer(SanitizedDict, +- SafeRepresenter.represent_dict) +- +- + # node_sanitize() + # + # Returnes an alphabetically ordered recursive copy +diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py +index 144dc7fda..29a2ec1e9 100644 +--- a/tests/testutils/runcli.py ++++ b/tests/testutils/runcli.py +@@ -7,7 +7,6 @@ + import traceback + import subprocess + from contextlib import contextmanager, ExitStack +-from ruamel import yaml + import pytest + + # XXX Using pytest private internals here +@@ -398,7 +397,8 @@ def get_element_config(self, project, element_name): + ]) + + result.assert_success() +- return yaml.safe_load(result.output) ++ yml = _yaml.prepare_roundtrip_yaml() ++ return yml.load(result.output) + + # Fetch the elements that would be in the pipeline with the given + # arguments. +@@ -463,10 +463,8 @@ def run(self, *args, project_config=None, **kwargs): + # dictionaries need to be loaded via _yaml.load_data() first + # + with tempfile.TemporaryDirectory(dir=project_directory) as scratchdir: +- + temp_project = os.path.join(scratchdir, 'project.conf') +- with open(temp_project, 'w') as f: +- yaml.safe_dump(project_config, f) ++ _yaml.dump(project_config, temp_project) + + project_config = _yaml.load(temp_project) + + +From 63676c10efb9db6f6c53ac7fae8c215afa7d7aac Mon Sep 17 00:00:00 2001 +From: Tristan van Berkom +Date: Wed, 3 Aug 2022 22:58:33 +0900 +Subject: [PATCH 4/4] Fix side effects of loading YAML booleans as strings + +In the previous YAML behavior, we would serialize booleans as strings but +load booleans as boolean values, but after porting to the new ruamel API +it appears difficult to achieve this behavior in advance of a new ruamel +release including the fix https://sourceforge.net/p/ruamel-yaml/tickets/341/ + +This patch ensures buildstream at least internally uses the _yaml APIs +to access booleans loaded from the yaml so that tests pass. + +Plugins should in theory be using Plugin.node_get_member() and such APIs +to access booleans already, but if they do not, they will break with this +branch. +--- + buildstream/element.py | 4 ++-- + tests/frontend/project/sources/fetch_source.py | 6 ++++-- + 2 files changed, 6 insertions(+), 4 deletions(-) + +diff --git a/buildstream/element.py b/buildstream/element.py +index 5b51d4321..a83fe232a 100644 +--- a/buildstream/element.py ++++ b/buildstream/element.py +@@ -2498,7 +2498,7 @@ def __get_artifact_metadata_workspaced(self, key=None): + # Parse the expensive yaml now and cache the result + meta_file = os.path.join(artifact_base, 'meta', 'workspaced.yaml') + meta = _yaml.load(meta_file) +- workspaced = meta['workspaced'] ++ workspaced = _yaml.node_get(meta, bool, 'workspaced') + + # Cache it under both strong and weak keys + strong_key, weak_key = self.__get_artifact_metadata_keys(key) +@@ -2528,7 +2528,7 @@ def __get_artifact_metadata_workspaced_dependencies(self, key=None): + # Parse the expensive yaml now and cache the result + meta_file = os.path.join(artifact_base, 'meta', 'workspaced-dependencies.yaml') + meta = _yaml.load(meta_file) +- workspaced = meta['workspaced-dependencies'] ++ workspaced = _yaml.node_get(meta, list, 'workspaced-dependencies') + + # Cache it under both strong and weak keys + strong_key, weak_key = self.__get_artifact_metadata_keys(key) +diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py +index 10e89960c..e590b2343 100644 +--- a/tests/frontend/project/sources/fetch_source.py ++++ b/tests/frontend/project/sources/fetch_source.py +@@ -42,8 +42,10 @@ def configure(self, node): + self.original_urls = self.node_get_member(node, list, 'urls') + self.output_file = self.node_get_member(node, str, 'output-text') + self.fetch_succeeds = {} +- if 'fetch-succeeds' in node: +- self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])} ++ ++ fetch_succeeds_node = self.node_get_member(node, dict, 'fetch-succeeds', {}) ++ for key, _ in self.node_items(fetch_succeeds_node): ++ self.fetch_succeeds[key] = self.node_get_member(fetch_succeeds_node, bool, key) + + # First URL is the primary one for this test + # diff --git a/buildstream.spec b/buildstream.spec index 593098f..e7b36e1 100644 --- a/buildstream.spec +++ b/buildstream.spec @@ -11,6 +11,20 @@ Version: 1.6.6 Release: %autorelease Source0: https://github.com/apache/buildstream/archive/%{version}/buildstream-%{version}.tar.gz + +# BUG: https://github.com/apache/buildstream/issues/1623 +# [bst-1] Depend on new version of ruamel.yaml +# PR: https://github.com/apache/buildstream/pull/1709 +# [bst-1] Update version of ruamel.yaml +# +# This fixes the deprecation warnings for using ruamel.yaml in bst-1 +# +# Based on [@abderrahim](https://github.com/abderrahim)'s work in +# [#1546](https://github.com/apache/buildstream/pull/1546) +# +# Fixes [#1623](https://github.com/apache/buildstream/issues/1623) +Patch: https://github.com/apache/buildstream/pull/1709.patch + BuildRequires: bubblewrap >= 0.1.2 BuildRequires: make BuildRequires: python3-devel >= 3.5 @@ -30,7 +44,7 @@ BuildRequires: python3-jinja2 >= 2.10 BuildRequires: python3-pluginbase BuildRequires: python3-protobuf >= 3.19 BuildRequires: python3-psutil -BuildRequires: python3-ruamel-yaml < 0.17 +BuildRequires: python3-ruamel-yaml BuildRequires: python3-setuptools BuildRequires: python3-ujson @@ -72,7 +86,7 @@ Requires: python3-jinja2 >= 2.10 Requires: python3-pluginbase Requires: python3-protobuf >= 3.6 Requires: python3-psutil -Requires: python3-ruamel-yaml < 0.17 +Requires: python3-ruamel-yaml Requires: python3-setuptools Requires: python3-ujson Requires: tar