From bb9f46e788db1ee2046929a959060395b5e3f18d Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Apr 01 2021 22:15:16 +0000 Subject: Bump to latest git, re-sync spec with upstream --- diff --git a/.gitignore b/.gitignore index c454684..78d7fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,5 @@ /assetcache-e9b44740c89522368b2561bf94a34d86d11eb62d.tar.xz /openQA-91baf79349a1310c8145e4b24169bca6881d55b7.tar.gz /assetcache-91baf79349a1310c8145e4b24169bca6881d55b7.tar.xz +/openQA-0e542b64451a0cd95095b78d36dac5fc5419f5b0.tar.gz +/assetcache-0e542b64451a0cd95095b78d36dac5fc5419f5b0.tar.xz diff --git a/0001-Fix-create_admin-by-not-expecting-ARGV-after-parsing.patch b/0001-Fix-create_admin-by-not-expecting-ARGV-after-parsing.patch deleted file mode 100644 index 7c6b8e7..0000000 --- a/0001-Fix-create_admin-by-not-expecting-ARGV-after-parsing.patch +++ /dev/null @@ -1,54 +0,0 @@ -From 66b6cfec66330d9536ffaa0d9abc7570815d2d0d Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Thu, 5 Nov 2020 16:36:54 -0800 -Subject: [PATCH] Fix create_admin by not expecting ARGV after parsing it - -a2d64957 inadvertently broke this script. It kept a check that -bails out unless ARGV is longer than 1, which was I think meant -to at least make sure the user passed more than one argument, but -moved option parsing to *before* that check. GetOptions consumes -the items it parses, so if you run the script as intended, then -after option parsing is done, only one thing is left in ARGV (the -user ID) and the check fails. You could force the script to run -by putting some extra argument after the user ID, but otherwise -it would always bail out. - -This fixes it by moving the ARGV length check back to before we -parse the options. It seems like an odd check to me - why do we -want to check just that *at least one* of the allegedly-optional -arguments is passed? - but I figured for now minimal change is -best. - -Signed-off-by: Adam Williamson ---- - script/create_admin | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/script/create_admin b/script/create_admin -index 005a26f55..2213de040 100755 ---- a/script/create_admin -+++ b/script/create_admin -@@ -47,6 +47,9 @@ sub usage { - exit $_[0]; - } - -+# need to count this *before* calling GetOptions -+my $numargs = scalar @ARGV; -+ - my $result = GetOptions( - "email=s" => \$email, - "nickname=s" => \$nickname, -@@ -55,8 +58,9 @@ my $result = GetOptions( - "secret=s" => \$secret, - "help" => \$help, - ); -+ - usage 0 if $help; --usage 1 unless $result && $user && scalar @ARGV > 1; -+usage 1 unless $result && $user && $numargs > 1; - - if (($key || $secret) - && !($key =~ /^[[:xdigit:]]{16}$/ && $secret =~ /^[[:xdigit:]]{16}$/)) --- -2.29.2 - diff --git a/0001-Fix-several-more-API-routes-for-Mojolicious-9.11-rou.patch b/0001-Fix-several-more-API-routes-for-Mojolicious-9.11-rou.patch new file mode 100644 index 0000000..8a9f0a3 --- /dev/null +++ b/0001-Fix-several-more-API-routes-for-Mojolicious-9.11-rou.patch @@ -0,0 +1,101 @@ +From 57feb7a4cf12a670bb01547073ff60880ad0b1ae Mon Sep 17 00:00:00 2001 +From: Adam Williamson +Date: Tue, 30 Mar 2021 16:54:53 -0700 +Subject: [PATCH] Fix several more API routes for Mojolicious 9.11 routing + change + +Route format detection is disabled by default from Mojolicious +9.11. This breaks accessing, for instance, /overview.json and +expecting it to be handled the same as accessing /overview with +a JSON content-type header. We have several routes that are +implemented to allow for this. #1820 adjusted several routes +where the tests actually use format detection to make it work +again; this adjusts several remaining routes that also would +previously have worked this way, but where no test happened to +use it. Without this change, those routes are accessible with +format detection with Mojo <9.11, but not with Mojo >=9.11. +With this change, they work with both, at the minor 'cost' that +URLs generated for them with `url_for` will now have a .html +extension where they did not before. + +Signed-off-by: Adam Williamson +--- + lib/OpenQA/WebAPI.pm | 6 ++++-- + t/22-dashboard.t | 12 ++++++------ + 2 files changed, 10 insertions(+), 8 deletions(-) + +diff --git a/lib/OpenQA/WebAPI.pm b/lib/OpenQA/WebAPI.pm +index b5bb86f71..4026898db 100644 +--- a/lib/OpenQA/WebAPI.pm ++++ b/lib/OpenQA/WebAPI.pm +@@ -131,7 +131,8 @@ sub startup ($self) { + $r->get('/search')->name('search')->to(template => 'search/search'); + + $r->get('/tests')->name('tests')->to('test#list'); +- $r->get('/tests/overview')->name('tests_overview')->to('test#overview'); ++ $r->get('/tests/overview' => [format => ['json', 'html']])->name('tests_overview') ++ ->to('test#overview', format => 'html'); + $r->get('/tests/latest')->name('latest')->to('test#latest'); + + $r->get('/tests/export')->name('tests_export')->to('test#export'); +@@ -195,7 +196,8 @@ sub startup ($self) { + + $r->get('/group_overview/' => [format => ['json', 'html']])->name('group_overview') + ->to('main#job_group_overview', format => 'html'); +- $r->get('/parent_group_overview/')->name('parent_group_overview')->to('main#parent_group_overview'); ++ $r->get('/parent_group_overview/' => [format => ['json', 'html']])->name('parent_group_overview') ++ ->to('main#parent_group_overview', format => 'html'); + + # Favicon + $r->get('/favicon.ico' => sub ($c) { $c->render_static('favicon.ico') }); +diff --git a/t/22-dashboard.t b/t/22-dashboard.t +index 505dfa580..7f989d853 100644 +--- a/t/22-dashboard.t ++++ b/t/22-dashboard.t +@@ -139,8 +139,8 @@ sub check_test_parent { + is_deeply( + \@urls, + [ +- '/tests/overview?distri=opensuse&version=13.1&build=0091&groupid=1001', +- '/tests/overview?distri=opensuse&version=13.1&build=0091&groupid=1002' ++ '/tests/overview.html?distri=opensuse&version=13.1&build=0091&groupid=1001', ++ '/tests/overview.html?distri=opensuse&version=13.1&build=0091&groupid=1002' + ], + 'link URLs' + ); +@@ -339,12 +339,12 @@ $t->get_ok('/dashboard_build_results?limit_builds=20&show_tags=0')->status_is(20 + is(scalar @urls, 12, 'now builds belong to different versions and are split'); + is( + $urls[1]->attr('href'), +- '/tests/overview?distri=suse&version=14.2&build=87.5011&groupid=1001', ++ '/tests/overview.html?distri=suse&version=14.2&build=87.5011&groupid=1001', + 'most recent version/build' + ); + is( + $urls[-1]->attr('href'), +- '/tests/overview?distri=opensuse&version=13.1&build=0091&groupid=1002', ++ '/tests/overview.html?distri=opensuse&version=13.1&build=0091&groupid=1002', + 'oldest version/build still shown' + ); + +@@ -365,7 +365,7 @@ subtest 'build which has jobs with different DISTRIs links to overview with all + my $first_url = $urls[1]->attr('href'); + is( + $first_url, +- '/tests/overview?distri=opensuse&distri=suse&version=14.2&build=87.5011&groupid=1001', ++ '/tests/overview.html?distri=opensuse&distri=suse&version=14.2&build=87.5011&groupid=1001', + 'both distris present in overview link' + ); + $job_with_different_distri->delete; +@@ -453,7 +453,7 @@ subtest 'job parent groups with multiple version and builds' => sub { + my $first_entire_build_url = $entire_build_url_list[0]->attr('href'); + is( + $first_entire_build_url, +- '/tests/overview?distri=suse&version=14.2&build=87.5011&groupid=1001&groupid=1002', ++ '/tests/overview.html?distri=suse&version=14.2&build=87.5011&groupid=1001&groupid=1002', + 'entire build url contains all the child group ids' + ); + +-- +2.31.1 + diff --git a/0001-Handle-placeholders-after-parsing-_URL-settings.patch b/0001-Handle-placeholders-after-parsing-_URL-settings.patch deleted file mode 100644 index 2c23f6d..0000000 --- a/0001-Handle-placeholders-after-parsing-_URL-settings.patch +++ /dev/null @@ -1,207 +0,0 @@ -From fee60fdcc32ceadfba92736b518dd223de3f18b6 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Mon, 16 Nov 2020 15:09:28 -0800 -Subject: [PATCH 1/4] Handle placeholders after parsing _URL settings - -b3bc8ebf caused some problems by moving the parsing of `_URL` -settings to after the call to `JobSettings::generate_settings`, -which is where handling of placeholders (%FOO%), usually from the -templates, is done. This broke the case where e.g. a template -uses `%ISO%` in a value, and the job is posted with `ISO_URL` but -no explicit `ISO` set. In that case the derived `ISO` setting -should be used in the placeholder expansion, but this was broken. - -This fixes the problems by splitting up the "parse _URL settings" -and "create download list" functions of `create_downloads_list`, -and moving the parsing part into `generate_settings`, just before -placeholder expansion happens. This is after we pull in settings -from the templates, so the bug b3bc8ebf was trying to fix (POO -62159) should still be fixed. Download list creation should still -happen the same as currently, so POO #70687 should still be fixed. - -Signed-off-by: Adam Williamson ---- - lib/OpenQA/JobSettings.pm | 39 ++++++++++++++++++++++++ - lib/OpenQA/Utils.pm | 63 +++++++++++++++++---------------------- - t/api/02-iso-download.t | 13 ++++++++ - 3 files changed, 79 insertions(+), 36 deletions(-) - -diff --git a/lib/OpenQA/JobSettings.pm b/lib/OpenQA/JobSettings.pm -index 0d6c11811..b0e364c8c 100644 ---- a/lib/OpenQA/JobSettings.pm -+++ b/lib/OpenQA/JobSettings.pm -@@ -17,6 +17,12 @@ package OpenQA::JobSettings; - use strict; - use warnings; - -+use File::Basename; -+use Mojo::URL; -+use Mojo::Util 'url_unescape'; -+use OpenQA::Log 'log_debug'; -+use OpenQA::Utils 'get_url_short'; -+ - sub generate_settings { - my ($params) = @_; - my $settings = $params->{settings}; -@@ -53,6 +59,7 @@ sub generate_settings { - $settings->{JOB_DESCRIPTION} = $job_template->description if length $job_template->description; - } - -+ parse_url_settings($settings); - handle_plus_in_settings($settings); - return expand_placeholders($settings); - } -@@ -104,4 +111,36 @@ sub handle_plus_in_settings { - } - } - -+# Given a hashref of settings, parse any whose names end in _URL -+# to the short name, then if there is not already a setting with -+# the short name, set it to the filename from the URL (with the -+# compression extension removed in the case of _DECOMPRESS_URL). -+# This has to happen *before* generate_jobs -+sub parse_url_settings { -+ my ($settings) = @_; -+ for my $setting (keys %$settings) { -+ my ($short, $do_extract) = get_url_short($setting); -+ next unless ($short); -+ next if ($settings->{$short}); -+ # As this comes in from an API call, URL will be URI-encoded -+ # This obviously creates a vuln if untrusted users can POST -+ $settings->{$setting} = url_unescape($settings->{$setting}); -+ my $url = $settings->{$setting}; -+ my $filename; -+ $filename = Mojo::URL->new($url)->path->parts->[-1]; -+ if ($do_extract) { -+ # if user wants to extract downloaded file, final filename -+ # will have last extension removed -+ $filename = fileparse($filename, qr/\.[^.]*/); -+ } -+ $settings->{$short} = $filename; -+ if (!$settings->{$short}) { -+ log_debug("Unable to get filename from $url. Ignoring $setting"); -+ delete $settings->{$short} unless $settings->{$short}; -+ next; -+ } -+ } -+ return undef; -+} -+ - 1; -diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm -index 4db65eaeb..6dc3d0880 100644 ---- a/lib/OpenQA/Utils.pm -+++ b/lib/OpenQA/Utils.pm -@@ -21,7 +21,6 @@ use Carp; - use Cwd 'abs_path'; - use IPC::Run(); - use Mojo::URL; --use Mojo::Util 'url_unescape'; - use Regexp::Common 'URI'; - use Try::Tiny; - use Mojo::File 'path'; -@@ -62,6 +61,7 @@ our @EXPORT = qw( - asset_type_from_setting - check_download_url - check_download_passlist -+ get_url_short - create_downloads_list - human_readable_size - locate_asset -@@ -499,45 +499,36 @@ sub check_download_passlist { - return (); - } - -+sub get_url_short { -+ # Given a setting name, if it ends with _URL or _DECOMPRESS_URL -+ # return the name with that string stripped, and a flag indicating -+ # whether decompression will be needed. If it doesn't, returns -+ # empty string and 0. -+ my ($arg) = @_; -+ return ('', 0) unless ($arg =~ /_URL$/); -+ my $short; -+ my $do_extract = 0; -+ if ($arg =~ /_DECOMPRESS_URL$/) { -+ $short = substr($arg, 0, -15); -+ $do_extract = 1; -+ } -+ else { -+ $short = substr($arg, 0, -4); -+ } -+ return ($short, $do_extract); -+} -+ - sub create_downloads_list { - my ($args) = @_; - my %downloads = (); - for my $arg (keys %$args) { -- next unless ($arg =~ /_URL$/); -- # As this comes in from an API call, URL will be URI-encoded -- # This obviously creates a vuln if untrusted users can POST -- $args->{$arg} = url_unescape($args->{$arg}); -- my $url = $args->{$arg}; -- my $do_extract = 0; -- my $short; -- my $filename; -- # if $args{FOO_URL} or $args{FOO_DECOMPRESS_URL} is set but $args{FOO} -- # is not, we will set $args{FOO} (the filename of the downloaded asset) -- # to the URL filename. This has to happen *before* -- # generate_jobs so the jobs have FOO set -- if ($arg =~ /_DECOMPRESS_URL$/) { -- $do_extract = 1; -- $short = substr($arg, 0, -15); # remove whole _DECOMPRESS_URL substring -- } -- else { -- $short = substr($arg, 0, -4); # remove _URL substring -- } -- if (!$args->{$short}) { -- $filename = Mojo::URL->new($url)->path->parts->[-1]; -- if ($do_extract) { -- # if user wants to extract downloaded file, final filename -- # will have last extension removed -- $filename = fileparse($filename, qr/\.[^.]*/); -- } -- $args->{$short} = $filename; -- if (!$args->{$short}) { -- log_debug("Unable to get filename from $url. Ignoring $arg"); -- delete $args->{$short} unless $args->{$short}; -- next; -- } -- } -- else { -- $filename = $args->{$short}; -+ my $url = $args->{$arg}; -+ my ($short, $do_extract) = get_url_short($arg); -+ next unless ($short); -+ my $filename = $args->{$short}; -+ unless ($filename) { -+ log_debug("No target filename set for $url. Ignoring $arg"); -+ next; - } - # We're only going to allow downloading of asset types. We also - # need this to determine the download location later -diff --git a/t/api/02-iso-download.t b/t/api/02-iso-download.t -index fcb3744f0..53b67ebf5 100644 ---- a/t/api/02-iso-download.t -+++ b/t/api/02-iso-download.t -@@ -298,4 +298,17 @@ subtest 'download task only blocks the related job when test suites have differe - is scalar(@{$gru_dep_tasks->{$gru_task_ids[0]}}), 3, 'one download task was created and it blocked 3 jobs'; - }; - -+subtest 'placeholder expansions work with _URL-derived settings' => sub { -+ $test_suites->find({name => 'kde'})->settings->create({key => 'FOOBAR', value => '%ISO%'}); -+ my $new_params = {%params, ISO_URL => 'http://localhost/openSUSE-13.1-DVD-i586-Build0091-Media.iso', TEST => 'kde'}; -+ $rsp = schedule_iso($new_params, 200); -+ is $rsp->json->{count}, 1, 'one job was scheduled'; -+ my $expanderjob = get_job($rsp->json->{ids}->[0]); -+ is( -+ $expanderjob->{settings}->{FOOBAR}, -+ 'openSUSE-13.1-DVD-i586-Build0091-Media.iso', -+ '%ISO% in template is expanded by posted ISO_URL' -+ ); -+}; -+ - done_testing(); --- -2.29.2 - diff --git a/0002-Don-t-set-FOO-from-FOO_URL-if-FOO-is-defined-but-fal.patch b/0002-Don-t-set-FOO-from-FOO_URL-if-FOO-is-defined-but-fal.patch deleted file mode 100644 index 8eec53e..0000000 --- a/0002-Don-t-set-FOO-from-FOO_URL-if-FOO-is-defined-but-fal.patch +++ /dev/null @@ -1,71 +0,0 @@ -From 0b2bb60efb0292764efe3c00ae19743b5165c5d8 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Tue, 17 Nov 2020 10:16:51 -0800 -Subject: [PATCH 2/4] Don't set FOO from FOO_URL if FOO is defined but false - -If someone has explicitly set e.g. ISO to "" for some scenario, -that's a clear signal they want the test in that scenario to run -without an ISO attached, even if it's posted with ISO_URL set. -However, when we decide whether to generate FOO from FOO_URL, -the check we use is not whether FOO is already *defined* in the -settings hash, but whether it *evaluates true*. That seems wrong. - -We ran into this is in real life after b3bc8eb changed the order -in which we do _URL parsing and +-overrides. There is a Fedora -test suite with `+ISO` set to `''`, the intent being that the -'ISO' value derived from the posted `ISO_URL` should be -overridden to `''` when running that test (so it runs with no -disc attached, and boots from the network). This worked so long -as we handled the +-overrides after we parsed the _URL settings, -but b3bc8eb changed things so we do the +-overrides before we -parse the _URL settings, and caused that case to trigger this. - -The previous commit to this one changes the ordering back to fix -a similar issue with placeholder expansion, and so also happens -to hide this bug again in that specific real life case, but this -seems like the 'correct' fix. Consider for instance if the test -suite just set `ISO` (not `+ISO`) to `''`; it seems like even -then we ought not to override that intent by parsing `ISO_URL`, -it should not actually be necessary for the test suite to use -the +-override mechanism. - -Signed-off-by: Adam Williamson ---- - lib/OpenQA/JobSettings.pm | 2 +- - t/api/02-iso-download.t | 9 +++++++++ - 2 files changed, 10 insertions(+), 1 deletion(-) - -diff --git a/lib/OpenQA/JobSettings.pm b/lib/OpenQA/JobSettings.pm -index b0e364c8c..0d438c0e4 100644 ---- a/lib/OpenQA/JobSettings.pm -+++ b/lib/OpenQA/JobSettings.pm -@@ -121,7 +121,7 @@ sub parse_url_settings { - for my $setting (keys %$settings) { - my ($short, $do_extract) = get_url_short($setting); - next unless ($short); -- next if ($settings->{$short}); -+ next if defined($settings->{$short}); - # As this comes in from an API call, URL will be URI-encoded - # This obviously creates a vuln if untrusted users can POST - $settings->{$setting} = url_unescape($settings->{$setting}); -diff --git a/t/api/02-iso-download.t b/t/api/02-iso-download.t -index 53b67ebf5..914d30552 100644 ---- a/t/api/02-iso-download.t -+++ b/t/api/02-iso-download.t -@@ -311,4 +311,13 @@ subtest 'placeholder expansions work with _URL-derived settings' => sub { - ); - }; - -+subtest 'test suite sets short asset setting to false value' => sub { -+ $test_suites->find({name => 'kde'})->settings->create({key => 'ISO', value => ''}); -+ my $new_params = {%params, ISO_URL => 'http://localhost/openSUSE-13.1-DVD-i586-Build0091-Media.iso', TEST => 'kde'}; -+ $rsp = schedule_iso($new_params, 200); -+ is $rsp->json->{count}, 1, 'one job was scheduled'; -+ my $overriddenjob = get_job($rsp->json->{ids}->[0]); -+ is($overriddenjob->{settings}->{ISO}, '', 'false-evaluating ISO in template overrides posted ISO_URL'); -+}; -+ - done_testing(); --- -2.29.2 - diff --git a/0003-parse_url_settings-improve-filename-parse-check.patch b/0003-parse_url_settings-improve-filename-parse-check.patch deleted file mode 100644 index a7a4a8d..0000000 --- a/0003-parse_url_settings-improve-filename-parse-check.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 5d04b8db3ce153540a71e65bf6bcfe6d82004d48 Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Wed, 18 Nov 2020 09:58:58 -0800 -Subject: [PATCH 3/4] parse_url_settings: improve filename parse check - -There's no need to set the value then delete it if it turns out -we set it to an empty string; that's a hangover from when we did -not assign `$filename` but set the value directly to the return -of the `Mojo::URL` call. Let's just check if we managed to get a -filename and bail before setting the value if not. This also -reduces potential confusion around truthiness vs. defined checks. - -Signed-off-by: Adam Williamson ---- - lib/OpenQA/JobSettings.pm | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/lib/OpenQA/JobSettings.pm b/lib/OpenQA/JobSettings.pm -index 0d438c0e4..c3c7d97c9 100644 ---- a/lib/OpenQA/JobSettings.pm -+++ b/lib/OpenQA/JobSettings.pm -@@ -133,12 +133,11 @@ sub parse_url_settings { - # will have last extension removed - $filename = fileparse($filename, qr/\.[^.]*/); - } -- $settings->{$short} = $filename; -- if (!$settings->{$short}) { -+ if (!$filename) { - log_debug("Unable to get filename from $url. Ignoring $setting"); -- delete $settings->{$short} unless $settings->{$short}; - next; - } -+ $settings->{$short} = $filename; - } - return undef; - } --- -2.29.2 - diff --git a/0004-parse_url_settings-don-t-parse-value-for-non-asset-t.patch b/0004-parse_url_settings-don-t-parse-value-for-non-asset-t.patch deleted file mode 100644 index e5d024b..0000000 --- a/0004-parse_url_settings-don-t-parse-value-for-non-asset-t.patch +++ /dev/null @@ -1,83 +0,0 @@ -From 0340774523d7b11bd54c59780e1504d12d697f3d Mon Sep 17 00:00:00 2001 -From: Adam Williamson -Date: Wed, 18 Nov 2020 10:02:08 -0800 -Subject: [PATCH 4/4] parse_url_settings: don't parse value for non-asset types - -If a test sets NOT_ASSET_URL: http://some/parseable.url , we don't -actually want to parse that to NOT_ASSET: parseable.url , because -we aren't going to download it later. We had this bug back in 2016 -and fixed it specifically in f54a18f , then regressed it again in -391f95e3 because we needed to pass the parsed file name to the -asset type check. - -When we split up `parse_url_settings` and `create_downloads_list` -the asset type check wound up in `create_downloads_list`. To fix -the bug we can't avoid duplicating it here - we do have to check -it in both places, here to avoid setting $settings->{$short} if -it's not an asset type, and there to avoid doing the download if -somehow both NOT_ASSET_URL and NOT_ASSET wind up being set by -templates or the scheduler. It's not worth splitting the check to -a shared utility function as you still need flow control. I don't -think the log message will ever be duplicated as we should never -manage to reach and fail the check *both* times (only one or the -other). - -Signed-off-by: Adam Williamson ---- - lib/OpenQA/JobSettings.pm | 13 +++++++++---- - t/api/02-iso-download.t | 1 + - 2 files changed, 10 insertions(+), 4 deletions(-) - -diff --git a/lib/OpenQA/JobSettings.pm b/lib/OpenQA/JobSettings.pm -index c3c7d97c9..335c8d1ef 100644 ---- a/lib/OpenQA/JobSettings.pm -+++ b/lib/OpenQA/JobSettings.pm -@@ -21,7 +21,7 @@ use File::Basename; - use Mojo::URL; - use Mojo::Util 'url_unescape'; - use OpenQA::Log 'log_debug'; --use OpenQA::Utils 'get_url_short'; -+use OpenQA::Utils qw(asset_type_from_setting get_url_short); - - sub generate_settings { - my ($params) = @_; -@@ -113,9 +113,9 @@ sub handle_plus_in_settings { - - # Given a hashref of settings, parse any whose names end in _URL - # to the short name, then if there is not already a setting with --# the short name, set it to the filename from the URL (with the --# compression extension removed in the case of _DECOMPRESS_URL). --# This has to happen *before* generate_jobs -+# the short name and the setting is an asset type, set it to the -+# filename from the URL (with the compression extension removed -+# in the case of _DECOMPRESS_URL). - sub parse_url_settings { - my ($settings) = @_; - for my $setting (keys %$settings) { -@@ -137,6 +137,11 @@ sub parse_url_settings { - log_debug("Unable to get filename from $url. Ignoring $setting"); - next; - } -+ # We shouldn't set the short setting for non-asset types -+ unless (asset_type_from_setting($short, $filename)) { -+ log_debug("_URL downloading only allowed for asset types! $short is not an asset type"); -+ next; -+ } - $settings->{$short} = $filename; - } - return undef; -diff --git a/t/api/02-iso-download.t b/t/api/02-iso-download.t -index 914d30552..a718ef154 100644 ---- a/t/api/02-iso-download.t -+++ b/t/api/02-iso-download.t -@@ -158,6 +158,7 @@ check_job_setting($t, $rsp, 'KERNEL', 'callitvmlinuz', - $rsp = schedule_iso({%params, NO_ASSET_URL => 'http://localhost/nonexistent.iso'}); - is($rsp->json->{count}, $expected_job_count, 'a regular ISO post creates the expected number of jobs'); - check_download_asset('non-asset _URL'); -+check_job_setting($t, $rsp, 'NO_ASSET', undef, 'NO_ASSET is not parsed from NO_ASSET_URL'); - - # Using asset _URL but without filename extractable from URL create warning in log file, jobs, but no gru job - $rsp = schedule_iso({%iso, ISO_URL => 'http://localhost'}); --- -2.29.2 - diff --git a/23-fedora-messaging.t b/23-fedora-messaging.t index f6a7298..1725b31 100644 --- a/23-fedora-messaging.t +++ b/23-fedora-messaging.t @@ -63,7 +63,7 @@ $plugin_mock->mock( } ); -OpenQA::Test::Database->new->create(); +OpenQA::Test::Database->new->create(fixtures_glob => '01-jobs.pl 03-users.pl 05-job_modules.pl'); # this test also serves to test plugin loading via config file my $conf = << 'EOF'; diff --git a/openqa.spec b/openqa.spec index 5fa9f0e..3555cd3 100644 --- a/openqa.spec +++ b/openqa.spec @@ -23,9 +23,9 @@ %global github_owner os-autoinst %global github_name openQA %global github_version 4.6 -%global github_commit 91baf79349a1310c8145e4b24169bca6881d55b7 +%global github_commit 0e542b64451a0cd95095b78d36dac5fc5419f5b0 # if set, will be a post-release snapshot build, otherwise a 'normal' build -%global github_date 20201103 +%global github_date 20210401 %global shortcommit %(c=%{github_commit}; echo ${c:0:7}) # can't use linebreaks here! @@ -43,15 +43,16 @@ # The following line is generated from dependencies.yaml (upstream) %define assetpack_requires perl(CSS::Minifier::XS) >= 0.01 perl(JavaScript::Minifier::XS) >= 0.11 perl(Mojolicious::Plugin::AssetPack) >= 1.36 # The following line is generated from dependencies.yaml (upstream) -%define common_requires perl(Archive::Extract) > 0.7 perl(Config::IniFiles) perl(Cpanel::JSON::XS) perl(Cwd) perl(Data::Dump) perl(Data::Dumper) perl(Digest::MD5) perl(Getopt::Long) perl(Minion) >= 10.12 perl(Mojolicious) >= 8.55 perl(Regexp::Common) perl(Storable) perl(Try::Tiny) +%define common_requires perl >= 5.20.0 perl(Archive::Extract) > 0.7 perl(Config::IniFiles) perl(Cpanel::JSON::XS) perl(Cwd) perl(Data::Dump) perl(Data::Dumper) perl(Digest::MD5) perl(Getopt::Long) perl(Minion) >= 10.12 perl(Mojolicious) >= 8.55 perl(Regexp::Common) perl(Storable) perl(Try::Tiny) # runtime requirements for the main package that are not required by other sub-packages # The following line is generated from dependencies.yaml (upstream) -%define main_requires %assetpack_requires git-core perl(BSD::Resource) perl(Carp) perl(Carp::Always) perl(CommonMark) perl(Config::Tiny) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::OptimisticLocking) perl(DBIx::Class::ResultClass::HashRefInflator) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(Date::Format) perl(DateTime) perl(DateTime::Duration) perl(DateTime::Format::Pg) perl(Exporter) perl(Fcntl) perl(File::Basename) perl(File::Copy) perl(File::Copy::Recursive) perl(File::Path) perl(File::Spec) perl(FindBin) perl(Getopt::Long::Descriptive) perl(IO::Handle) perl(IPC::Run) perl(JSON::Validator) perl(LWP::UserAgent) perl(Module::Load::Conditional) perl(Module::Pluggable) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::RabbitMQ::Client) >= 0.2 perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Pod::POM) perl(SQL::Translator) perl(Scalar::Util) perl(Sort::Versions) perl(Text::Diff) perl(Time::HiRes) perl(Time::ParseDate) perl(Time::Piece) perl(Time::Seconds) perl(URI::Escape) perl(YAML::PP) >= 0.026 perl(YAML::XS) perl(aliased) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) +%define main_requires %assetpack_requires git-core perl(BSD::Resource) perl(Carp) perl(Carp::Always) perl(CommonMark) perl(Config::Tiny) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DeploymentHandler) perl(DBIx::Class::DynamicDefault) perl(DBIx::Class::OptimisticLocking) perl(DBIx::Class::ResultClass::HashRefInflator) perl(DBIx::Class::Schema::Config) perl(DBIx::Class::Storage::Statistics) perl(Date::Format) perl(DateTime) perl(DateTime::Duration) perl(DateTime::Format::Pg) perl(Exporter) perl(Fcntl) perl(File::Basename) perl(File::Copy) perl(File::Copy::Recursive) perl(File::Path) perl(File::Spec) perl(Filesys::Df) perl(FindBin) perl(Getopt::Long::Descriptive) perl(IO::Handle) perl(IPC::Run) perl(JSON::Validator) perl(LWP::UserAgent) perl(Module::Load::Conditional) perl(Module::Pluggable) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::RabbitMQ::Client) >= 0.2 perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Pod::POM) perl(SQL::Translator) perl(Scalar::Util) perl(Sort::Versions) perl(Text::Diff) perl(Time::HiRes) perl(Time::ParseDate) perl(Time::Piece) perl(Time::Seconds) perl(URI::Escape) perl(YAML::PP) >= 0.026 perl(YAML::XS) perl(aliased) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) # The following line is generated from dependencies.yaml (upstream) %define client_requires curl git-core jq perl(Getopt::Long::Descriptive) perl(IO::Socket::SSL) >= 2.009 perl(IPC::Run) perl(JSON::Validator) perl(LWP::Protocol::https) perl(LWP::UserAgent) perl(Test::More) perl(YAML::PP) >= 0.020 perl(YAML::XS) # diff from SUSE: case (they have openQA-client, we have openqa-client) +# Diff from SUSE: we have 'sqlite' not 'sqlite3' # The following line is generated from dependencies.yaml (upstream) -%define worker_requires openqa-client optipng os-autoinst < 5 perl(Minion::Backend::SQLite) >= 5.0.1 perl(Mojo::IOLoop::ReadWriteProcess) >= 0.26 perl(Mojo::SQLite) +%define worker_requires openqa-client optipng os-autoinst < 5 perl(File::Map) perl(Minion::Backend::SQLite) >= 5.0.1 perl(Mojo::IOLoop::ReadWriteProcess) >= 0.26 perl(Mojo::SQLite) psmisc sqlite >= 3.24.0 # The following line is generated from dependencies.yaml (upstream) %define build_requires %assetpack_requires rubygem(sass) @@ -70,7 +71,7 @@ %else %define qemu qemu %endif -# diff from SUSE: perl(Devel::Cover::Report::Codecov) dropped because +# diff from SUSE: perl(Devel::Cover::Report::Codecovbash) dropped because # it's not in Fedora (this means you can't run 'make coverage-codecov') # xorg-x11-fonts dropped because that binary package doesn't exist in # Fedora (it exists as a source package generating multiple binary @@ -86,7 +87,7 @@ Name: openqa Version: %{github_version} -Release: 61%{?github_date:.%{github_date}git%{shortcommit}}%{?dist} +Release: 62%{?github_date:.%{github_date}git%{shortcommit}}%{?dist} Summary: OS-level automated testing framework License: GPLv2+ Url: http://os-autoinst.github.io/openQA/ @@ -108,15 +109,9 @@ Source3: FedoraMessaging.pm # tests for the fedora-messaging publishing plugin Source4: 23-fedora-messaging.t -# Fix create_admin script to not *always* fail -# https://github.com/os-autoinst/openQA/pull/3519 -Patch0: 0001-Fix-create_admin-by-not-expecting-ARGV-after-parsing.patch -# Fix placeholder substitution and overrides for _URL-derived settings -# https://github.com/os-autoinst/openQA/pull/3564 -Patch1: 0001-Handle-placeholders-after-parsing-_URL-settings.patch -Patch2: 0002-Don-t-set-FOO-from-FOO_URL-if-FOO-is-defined-but-fal.patch -Patch3: 0003-parse_url_settings-improve-filename-parse-check.patch -Patch4: 0004-parse_url_settings-don-t-parse-value-for-non-asset-t.patch +# Fix request routing for Mojolicious 9.11 security improvements +# https://github.com/os-autoinst/openQA/pull/3816 +Patch1: 0001-Fix-several-more-API-routes-for-Mojolicious-9.11-rou.patch BuildRequires: make BuildRequires: %{python_scripts_requires} @@ -365,6 +360,9 @@ mkdir %{buildroot}%{_localstatedir}/lib/openqa/webui/cache # We don't do AppArmor rm -rf %{buildroot}%{_sysconfdir}/apparmor.d +# these scripts are very SUSE-specific +rm -f %{buildroot}%{_datadir}/openqa/script/openqa-auto-update +rm -f %{buildroot}%{_datadir}/openqa/script/openqa-rollback %check @@ -377,9 +375,9 @@ touch openqa-debug.log autoinst-log.txt chmod a+w openqa-debug.log autoinst-log.txt # we can't use 'unshare' in Fedora package build env sed -i -e 's,unshare -r -n ,,g' t/40-script_openqa-clone-custom-git-refspec.t t/40-openqa-clone-job.t t/32-openqa_client-script.t -# this test expects a 'not connected' error that it gets with unshare -# but with mock we get 'Connection refused', so just wipe it -sed -i -e '/fails without network/d' t/32-openqa_client-script.t +# these tests expect a 'not connected' error that it gets with unshare +# but with mock we get 'Connection refused', so just wipe them +sed -i -e '/fails without network/d' t/32-openqa_client-script.t t/40-openqa-clone-job.t # Skip tests not working currently, or flaky rm \ t/01-test-utilities.t \ @@ -392,6 +390,10 @@ rm \ # within CI systems, e.g. OBS. See t/lib/OpenQA/Test/TimeLimit.pm export CI=1 export OPENQA_TEST_TIMEOUT_SCALE_CI=15 +# Skip container tests that would need additional requirements, e.g. +# docker-compose. Also, these tests are less relevant (or not relevant) for +# packaging +export CONTAINER_TEST=0 # GIT_CEILING_DIRECTORIES here avoids a case where git error handling # can differ when you run the build in mock and cause 16-utils-runcmd # to fail @@ -565,6 +567,9 @@ fi %{_unitdir}/openqa-worker-cacheservice-minion.service %{_unitdir}/openqa-worker-cacheservice.service %{_unitdir}/openqa-worker-no-cleanup@.service +%{_unitdir}/openqa-worker-auto-restart@.service +%{_unitdir}/openqa-reload-worker-auto-restart@.service +%{_unitdir}/openqa-reload-worker-auto-restart@.path %{_unitdir}/openqa-slirpvde.service %{_unitdir}/openqa-vde_switch.service %{_datadir}/openqa/script/openqa-slirpvde @@ -642,6 +647,10 @@ fi %{_datadir}/openqa/lib/OpenQA/WebAPI/Plugin/FedoraUpdateRestart.pm %changelog +* Thu Apr 01 2021 Adam Williamson - 4.6-62.20210401git0e542b6 +- Bump to latest git, re-sync spec with upstream +- Backport PR #3816: compatibility with Mojolicious 9.11+ + * Tue Mar 02 2021 Zbigniew Jędrzejewski-Szmek - 4.6-61.20201103git91baf79 - Rebuilt for updated systemd-rpm-macros See https://pagure.io/fesco/issue/2583. diff --git a/sources b/sources index 7a9565a..66354f6 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (openQA-91baf79349a1310c8145e4b24169bca6881d55b7.tar.gz) = 791c0234913256dd51bb426fca701237d536b2bcd380022a2c52991c99d5b8ecb3384a251407d5dc55dae176dfa08d6cfb8175ef40d1bfed9e4966b5b41e0b1c -SHA512 (assetcache-91baf79349a1310c8145e4b24169bca6881d55b7.tar.xz) = 39a10e3571f784ceb1db07c31b44f5f771cc25abd732ba949bf2595b886bda8876b3719d6adac8e670117c2c7c79ac2ab8b951753f143507bf2a3c32bacb2d97 +SHA512 (openQA-0e542b64451a0cd95095b78d36dac5fc5419f5b0.tar.gz) = b9782bd8ba9aa1746a74717914355df0038514479bf68126b868577c9f8269437a7749c8326ea2f0c2f89f273b53ea798caf84d57697954e837619b9461c47d1 +SHA512 (assetcache-0e542b64451a0cd95095b78d36dac5fc5419f5b0.tar.xz) = 5a999a57bddc10504c20118fd045fd0d96ad3f52a6050b93a9d209783f77e8683b1b48f6d91ac012885b517b24883e62e57518ade69db3bf074f841e3dcdf92a