#3 Add CI tests using the standard test interface
Closed 5 years ago by ignatenkobrain. Opened 5 years ago by bgoncalv.
rpms/ bgoncalv/dnf dnf-tests  into  master

@@ -0,0 +1,63 @@ 

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ #

+ #   Makefile of /CoreOS/dnf/Sanity/basic-sanity

+ #   Description: basic sanity test

+ #   Author: Eva Mrakova <emrakova@redhat.com>

+ #

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ #

+ #   Copyright (c) 2017 Red Hat, Inc.

+ #

+ #   This program is free software: you can redistribute it and/or

+ #   modify it under the terms of the GNU General Public License as

+ #   published by the Free Software Foundation, either version 2 of

+ #   the License, or (at your option) any later version.

+ #

+ #   This program is distributed in the hope that it will be

+ #   useful, but WITHOUT ANY WARRANTY; without even the implied

+ #   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

+ #   PURPOSE.  See the GNU General Public License for more details.

+ #

+ #   You should have received a copy of the GNU General Public License

+ #   along with this program. If not, see http://www.gnu.org/licenses/.

+ #

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ 

+ export TEST=/CoreOS/dnf/Sanity/basic-sanity

+ export TESTVERSION=1.0

+ 

+ BUILT_FILES=

+ 

+ FILES=$(METADATA) runtest.sh Makefile PURPOSE *.spec

+ 

+ .PHONY: all install download clean

+ 

+ run: $(FILES) build

+ 	./runtest.sh

+ 

+ build: $(BUILT_FILES)

+ 	test -x runtest.sh || chmod a+x runtest.sh

+ 

+ clean:

+ 	rm -f *~ $(BUILT_FILES)

+ 

+ 

+ include /usr/share/rhts/lib/rhts-make.include

+ 

+ $(METADATA): Makefile

+ 	@echo "Owner:           Eva Mrakova <emrakova@redhat.com>" > $(METADATA)

+ 	@echo "Name:            $(TEST)" >> $(METADATA)

+ 	@echo "TestVersion:     $(TESTVERSION)" >> $(METADATA)

+ 	@echo "Path:            $(TEST_DIR)" >> $(METADATA)

+ 	@echo "Description:     basic sanity test" >> $(METADATA)

+ 	@echo "Type:            Sanity" >> $(METADATA)

+ 	@echo "TestTime:        5m" >> $(METADATA)

+ 	@echo "RunFor:          dnf" >> $(METADATA)

+ 	@echo "Requires:        dnf createrepo_c httpd rpm-build" >> $(METADATA)

+ 	@echo "Priority:        Normal" >> $(METADATA)

+ 	@echo "License:         GPLv2+" >> $(METADATA)

+ 	@echo "Confidential:    no" >> $(METADATA)

+ 	@echo "Destructive:     no" >> $(METADATA)

+ 	@echo "Releases:        -RHEL4 -RHELClient5 -RHELServer5 -RHEL6 -RHEL7" >> $(METADATA)

+ 

+ 	rhts-lint $(METADATA)

@@ -0,0 +1,3 @@ 

+ PURPOSE of /CoreOS/dnf/Sanity/basic-sanity

+ Description: basic sanity test

+ Author: Eva Mrakova <emrakova@redhat.com>

@@ -0,0 +1,164 @@ 

+ #!/bin/bash

+ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ #

+ #   runtest.sh of /CoreOS/dnf/Sanity/basic-sanity

+ #   Description: basic sanity test

+ #   Author: Eva Mrakova <emrakova@redhat.com>

+ #

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ #

+ #   Copyright (c) 2017 Red Hat, Inc.

+ #

+ #   This program is free software: you can redistribute it and/or

+ #   modify it under the terms of the GNU General Public License as

+ #   published by the Free Software Foundation, either version 2 of

+ #   the License, or (at your option) any later version.

+ #

+ #   This program is distributed in the hope that it will be

+ #   useful, but WITHOUT ANY WARRANTY; without even the implied

+ #   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR

+ #   PURPOSE.  See the GNU General Public License for more details.

+ #

+ #   You should have received a copy of the GNU General Public License

+ #   along with this program. If not, see http://www.gnu.org/licenses/.

+ #

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ 

+ # Include Beaker environment

+ . /usr/bin/rhts-environment.sh || exit 1

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ PACKAGE="dnf"

+ 

+ DNF_CMD="dnf --disablerepo=\* --enablerepo=tstrepo\*"

+ 

+ RPMS="/root/rpmbuild/RPMS/noarch"

+ REPO="/var/www/html/tstrepo"

+ 

+ pkgtorepo() {

+     # builds a .rpm from given .spec, moves it to a testrepo identified by index

+     # $1 ... pkgname, $2 ... tstrepo index

+     rpmbuild -bb ${1}.spec &> /dev/null

+     cp $RPMS/${1}*.rpm ${REPO}${2}

+     rm -f $RPMS/${1}*.rpm

+ }

+ 

+ rlJournalStart

+     rlPhaseStartSetup

+         rlAssertRpm $PACKAGE

+         rlFileBackup --clean /etc/yum.repos.d/ /var/log/httpd/

+         rlRun "TmpDir=\$(mktemp -d)" 0 "Creating tmp directory"

+         rlRun "cp *.spec $TmpDir"

+         rlRun "pushd $TmpDir"

+         for i in {1..2}; do

+             rlRun "mkdir $REPO$i" 0 "Create dir for tstrepo$i"

+             cat > /etc/yum.repos.d/tstrepo${i}.repo <<EOF

+ [tstrepo$i]

+ name=tstrepo$i

+ baseurl=http://localhost/tstrepo$i/

+ enabled=1

+ gpgcheck=0

+ EOF

+         done

+         pkgtorepo testA 1

+         rlRun "createrepo_c ${REPO}1" 0 "Create tstrepo1"

+         rlRun "createrepo_c ${REPO}2" 0 "Create tstrepo2"

+ 

+         # workaround: when /var/log/httpd does not exist, httpd fails to start

+         if [[ ! -d /var/log/httpd ]]; then mkdir -p /var/log/httpd; fi 

+         rlServiceStart httpd

+     rlPhaseEnd

+ 

+     rlPhaseStartTest "Install, upgrade, downgrade and remove a pkg without dependencies"

+         # install a single pkg

+         rlRun "$DNF_CMD -y install testA" 0 "Install testA"

+         rlAssertRpm testA 1

+         # upgrade the pkg

+         rlRun "sed -i 's/Version.*$/Version: 2/' testA.spec"

+         pkgtorepo testA 2

+         rlRun "createrepo_c ${REPO}2" 0 "Update tstrepo2"

+         rlRun "$DNF_CMD clean all" 0 "Clean dnf cache"

+         rlRun "$DNF_CMD -y upgrade testA" 0 "Upgrade testA"

+         rlAssertRpm testA 2

+         # downgrade the pkg

+         rlRun "$DNF_CMD -y downgrade testA" 0 "Downgrade testA"

+         rlAssertRpm testA 1

+         # remove the pkg

+         rlRun "$DNF_CMD -y remove testA" 0 "Remove testA"

+         rlAssertNotRpm testA

+     rlPhaseEnd

+ 

+     rlPhaseStartTest "Install and remove a pkg with dependencies"

+         # prepare data

+         pkgtorepo testB 2

+         pkgtorepo testC 2

+         rlRun "createrepo_c ${REPO}2" 0 "Update tstrepo2"

+         rlRun "$DNF_CMD clean all" 0 "Clean dnf cache"

+         # install a pkg (B depends on C, C depends on A)

+         rlRun "$DNF_CMD -y install testB" 0 "Install testB and its deps"

+         rlAssertRpm testB 1

+         rlAssertRpm testC 1

+         rlAssertRpm testA 2

+         # remove the pkg (C and B should be removed)

+         rlRun "$DNF_CMD -y --setopt clean_requirements_on_remove=0 remove testC" 0 "Remove testC"

+         rlAssertNotRpm testC

+         rlAssertNotRpm testB

+         # testA should not be removed as an unused dep

+         rlAssertRpm testA 2

+     rlPhaseEnd

+ 

+     rlPhaseStartTest "Getting info (list, info, search, provides, whatprovides)"

+         # list

+         rlRun "$DNF_CMD list installed test\* > list1.out" 0 "Run dnf list installed"

+         rlAssertGrep "testA.*noarch" list1.out

+         rlAssertNotGrep "testB.*noarch" list1.out

+         rlAssertNotGrep "testC.*noarch" list1.out

+         rlRun "$DNF_CMD list available test\* > list2.out" 0 "Run dnf list available"

+         rlAssertGrep "testB.*noarch" list2.out

+         rlAssertGrep "testC.*noarch" list2.out

+         rlAssertNotGrep "testA.*noarch" list2.out

+         rlRun "$DNF_CMD -y downgrade testA" 0 "Downgrade testA"

+         rlRun "$DNF_CMD list upgrades > list3.out" 0 "Run dnf list upgrades"

+         rlAssertGrep "testA.*noarch.*tstrepo2" list3.out

+         # info

+         rlRun "$DNF_CMD info test[B-Z] > info.out" 0 "Run dnf info"

+         rlAssertGrep "Name.*testB" info.out

+         rlAssertGrep "Name.*testC" info.out

+         rlAssertNotGrep "Name.*testA" info.out

+         # search

+         rlRun "$DNF_CMD search 'This is a testB' > search.out" 0 "Run dnf search"

+         rlAssertGrep "Matched: This is a testB" search.out

+         rlAssertGrep "testB.*noarch" search.out

+         # provides

+         rlRun "$DNF_CMD provides testC-provides > provides.out" 0 "Run dnf provides"

+         rlAssertGrep "testC.*noarch" provides.out

+         # whatprovides

+         rlRun "$DNF_CMD whatprovides '/usr/bin/testA' > provides2.out" 0 "Run dnf whatprovides"

+         rlAssertGrep "testA-1.*noarch" provides2.out

+         rlAssertGrep "testA-2.*noarch" provides2.out

+     rlPhaseEnd

+ 

+     rlPhaseStartTest "History (list, info, undo, redo)"

+         rlRun "$DNF_CMD history list > hist1.out" 0 "Run dnf history list"

+         rlAssertGrep "Downgrade" hist1.out

+         # last transaction was testA downgrade, let's get its ID

+         rlRun "$DNF_CMD history info > hist2.out" 0 "Run dnf history info"

+         DOWNGRADE_HIST_ID=$(grep 'Transaction ID' hist2.out | cut -d: -f2)

+         rlRun "$DNF_CMD -y history undo last" 0 "Run dnf history undo"

+         rlAssertRpm testA 2

+         rlRun "$DNF_CMD -y history redo $DOWNGRADE_HIST_ID" 0 "Run dnf history redo"

+         rlAssertRpm testA 1

+     rlPhaseEnd

+ 

+     rlPhaseStartCleanup

+         rlRun "$DNF_CMD -y remove testA"

+         rlRun "rm -rf /var/www/html/tstrepo{1,2}" 0 "Delete tstrepo dirs"

+         rlRun "$DNF_CMD clean all" 0 "Clean dnf cache"

+         rlFileRestore

+         rlServiceRestore httpd

+         rlRun "popd"

+         rlRun "rm -r $TmpDir" 0 "Remove tmp directory"

+     rlPhaseEnd

+ rlJournalPrintText

+ rlJournalEnd

@@ -0,0 +1,21 @@ 

+ Summary: testA Package

+ Name: testA

+ Version: 1

+ Release: 0

+ Group: System Environment/Base

+ License: GPL

+ BuildArch: noarch

+ BuildRoot:  %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)

+ 

+ %description

+ 

+ This is a testA test package

+ 

+ %build

+ 

+ %install

+ mkdir -p %{buildroot}/usr/bin/

+ touch %{buildroot}/usr/bin/testA

+ 

+ %files

+ /usr/bin/testA

@@ -0,0 +1,23 @@ 

+ Summary: testB Package

+ Name: testB

+ Version: 1

+ Release: 0

+ Group: System Environment/Base

+ License: GPL

+ BuildArch: noarch

+ BuildRoot:  %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)

+ Provides: testB-provides

+ Requires: testC

+ 

+ %description

+ 

+ This is a testB test package

+ 

+ %build

+ 

+ %install

+ mkdir -p %{buildroot}/usr/bin/

+ touch %{buildroot}/usr/bin/testB

+ 

+ %files

+ /usr/bin/testB

@@ -0,0 +1,23 @@ 

+ Summary: testC Package

+ Name: testC

+ Version: 1

+ Release: 0

+ Group: System Environment/Base

+ License: GPL

+ BuildArch: noarch

+ BuildRoot:  %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)

+ Provides: testC-provides

+ Requires: testA

+ 

+ %description

+ 

+ This is a testC test package

+ 

+ %build

+ 

+ %install

+ mkdir -p %{buildroot}/usr/bin/

+ touch %{buildroot}/usr/bin/testC

+ 

+ %files

+ /usr/bin/testC

file added
+15
@@ -0,0 +1,15 @@ 

+ ---

+ # This first play always runs on the local staging system

+ - hosts: localhost

+   roles:

+   - role: standard-test-beakerlib

+     tags:

+     - classic

+     tests:

+     - basic-sanity

+     required_packages:

+     - dnf          # required for basic-sanity

+     - createrepo_c # required for basic-sanity

+     - httpd        # required for basic-sanity

+     - rpm-build    # required for basic-sanity

+ 

Adds tests according to the CI wiki [0] specifically the standard test interface in the spec [1].

The playbook includes Tier1 level test cases that have been tested in the following contexts and is passing reliably on Fedora Rawhide: Classic.
Test logs are stored in the Artifacts directory.

The following steps are used to execute the tests using the standard test interface:

  • Classic
    sudo ANSIBLE_INVENTORY=$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory) TEST_SUBJECTS="" TEST_ARTIFACTS=$PWD/artifacts ansible-playbook --tags classic tests.yml

[0] https://fedoraproject.org/wiki/CI
[1] https://fedoraproject.org/wiki/Changes/InvokingTests

Test Logs: (If you you would like a pointer to the complete log, I can include that as well)

  • Atomic

    Test does not run on Atomic

  • Docker

    Test does not run on container

  • Classic

    <snip>
    ==> default: TASK [standard-test-beakerlib : Execute beakerlib tests] ***
    ==> default: changed: [localhost] => (item=basic-sanity)
    ==> default:
    ==> default: TASK [standard-test-beakerlib : Make the master test summary log artifact] *
    ==> default: changed: [localhost] => (item=basic-sanity)
    ==> default:
    ==> default: TASK [standard-test-beakerlib : Pull out the logs]
    *****
    ==> default: changed: [localhost]
    ==> default:
    ==> default: TASK [standard-test-beakerlib : Check the results]
    *****
    ==> default: changed: [localhost]
    ==> default:
    ==> default: PLAY RECAP
    *********
    **
    ==> default: localhost : ok=15 changed=12 unreachable=0 failed=0
    ==> default: ++ '[' 0 -ne 0 ']'
    ==> default: ++ cat /root/dnf/artifacts/test.log
    ==> default: PASS basic-sanity

NO.

We don't use beakerlib and not planning to. We are using behave framework to run our tests.

If you want to get upstream tests in -- write a PoC how to run our upstream behave tests.

https://github.com/rpm-software-management/ci-dnf-stack/tree/master/dnf-docker-test/features

Pull-Request has been closed by ignatenkobrain

5 years ago