#1 Adds tests according to the CI
Opened a year ago by ssahani. Modified a year ago
rpms/ ssahani/keepalived tests  into  master

Adds tests according to the CI
Susant Sahani • a year ago  

@@ -0,0 +1,36 @@ 

+ # SPDX-License-Identifier: LGPL-2.1+

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ #

+ #   Makefile of /CoreOS/keepalived

+ #   Description: Test if keepliaved HA working ok

+ #   Author: Susant Sahani<susant@redhat.com>

+ #

+ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+ export TEST=/CoreOS/keepalived

+ export TESTVERSION=1.0

+ BUILT_FILES=

+ FILES=$(METADATA) runtest.sh Makefile PURPOSE

+ .PHONY: all install download clean

+ run: $(FILES) build

+ 	./runtest.sh

+ build: $(BUILT_FILES)

+ 	test -x runtest.sh || chmod a+x runtest.sh

+ clean:

+ 	rm -f *~ $(BUILT_FILES)

+ include /usr/share/rhts/lib/rhts-make.include

+ $(METADATA): Makefile

+ 	@echo "Owner:           Susant Sahani<susant@redhat.com>" > $(METADATA)

+ 	@echo "Name:            $(TEST)" >> $(METADATA)

+ 	@echo "TestVersion:     $(TESTVERSION)" >> $(METADATA)

+ 	@echo "Path:            $(TEST_DIR)" >> $(METADATA)

+ 	@echo "Description:     Test if the ABI hasn't changed" >> $(METADATA)

+ 	@echo "Type:            Sanity" >> $(METADATA)

+ 	@echo "TestTime:        5m" >> $(METADATA)

+ 	@echo "RunFor:          keepalived" >> $(METADATA)

+ 	@echo "Requires:        keepalived systemd nc socat iproute" >> $(METADATA)

+ 	@echo "Priority:        Normal" >> $(METADATA)

+ 	@echo "License:         GPLv2" >> $(METADATA)

+ 	@echo "Confidential:    no" >> $(METADATA)

+ 	@echo "Destructive:     no" >> $(METADATA)

+ 	@echo "Releases:        -Fedora 28" >> $(METADATA)

+ 	rhts-lint $(METADATA)

@@ -0,0 +1,3 @@ 

+ PURPOSE of /CoreOS/keepalived

+ Description: Failover test for keepalived

+ Author: Susant Sahani<susant@redhat.com>

@@ -0,0 +1,33 @@ 

+ vrrp_script chk_pid {

+         script "pidof nc"

+         interval 2

+ }

+ 

+ vrrp_instance VI_1 {

+         # The interface keepalived will manage

+         interface vethtest

+         state MASTER

+         # How often to send out VRRP advertisements

+         advert_int 2

+         # The virtual router id number to assign the routers to

+         virtual_router_id 51

+         # The priority to assign to this device. This controls

+         # who will become the MASTER and BACKUP for a given

+         # VRRP instance (a lower number get's less priority)

+         priority 100

+         authentication {

+                 auth_type PASS

+                 auth_pass SimplePassword

+         }

+         unicast_src_ip 192.168.111.50

+         unicast_peer {

+                 192.168.111.51

+         }

+         track_script {

+                 chk_pid

+         }

+         # The virtual IP addresses to float between nodes.

+         virtual_ipaddress {

+                 192.168.111.52

+         }

+ }

@@ -0,0 +1,14 @@ 

+ [Unit]

+ Description=LVS and VRRP High Availability Monitor Master

+ After=network-online.target syslog.target

+ Wants=network-online.target

+ 

+ [Service]

+ Type=forking

+ PIDFile=/var/run/keepalived-ci/test-keepalived-master.pid

+ KillMode=process

+ ExecStart=/usr/sbin/keepalived -f /var/run/keepalived-ci/keepalived-master.conf -D --log-file=/var/run/keepalived-ci/test-keepalived-master.log -p /var/run/keepalived-ci/test-keepalived-master.pid --flush-log-file

+ ExecReload=/bin/kill -HUP $MAINPID

+ 

+ [Install]

+ WantedBy=multi-user.target

@@ -0,0 +1,35 @@ 

+ net_namespace test-keepalived-ns

+ 

+ vrrp_script chk_pid {

+         script "pidof socat"

+         interval 2

+ }

+ 

+ vrrp_instance VI_1 {

+         # The interface keepalived will manage

+         interface vethpeer

+         state BACKUP

+         # How often to send out VRRP advertisements

+         advert_int 2

+         # The virtual router id number to assign the routers to

+         virtual_router_id 51

+         # The priority to assign to this device. This controls

+         # who will become the MASTER and BACKUP for a given

+         # VRRP instance (a lower number get's less priority)

+         priority 100

+         authentication {

+                 auth_type PASS

+                 auth_pass SimplePassword

+         }

+         unicast_src_ip 192.168.111.51

+         unicast_peer {

+                 192.168.111.50

+         }

+         track_script {

+                 chk_pid

+         }

+         # The virtual IP addresses to float between nodes.

+         virtual_ipaddress {

+                 192.168.111.52

+         }

+ }

@@ -0,0 +1,14 @@ 

+ [Unit]

+ Description=LVS and VRRP High Availability Monitor Slave

+ After=network-online.target syslog.target

+ Wants=network-online.target

+ 

+ [Service]

+ Type=forking

+ PIDFile=/var/run/keepalived-ci/test-keepalived-slave.pid

+ KillMode=process

+ ExecStart=/usr/sbin/keepalived -f /var/run/keepalived-ci/keepalived-slave.conf -D --log-file=/var/run/keepalived-ci/test-keepalived-slave.log -p /var/run/keepalived-ci/test-keepalived-slave.pid --flush-log-file

+ ExecReload=/bin/kill -HUP $MAINPID

+ 

+ [Install]

+ WantedBy=multi-user.target

@@ -0,0 +1,184 @@ 

+ #!/usr/bin/env python3

+ # SPDX-License-Identifier: LGPL-2.1+

+ # ~~~

+ #   Description: Tests for keepalived - Simple and robust facilities for loadbalancing and high-availability to Linux system

+ #

+ #   Author: Susant Sahani <susant@redhat.com>

+ #   Copyright (c) 2018 Red Hat, Inc.

+ # ~~~

+ 

+ import errno

+ import os

+ import sys

+ import time

+ import unittest

+ import subprocess

+ import signal

+ import shutil

+ import psutil

+ import socket

+ from pyroute2 import NetNS

+ from pyroute2 import IPDB

+ 

+ KEEPALIVED_MASTER_CONFIG_FILE='/var/run/keepalived-ci/keepalived-master.conf'

+ KEEPALIVED_MASTER_PID_FILE='/var/run/keepalived-ci/test-keepalived-master.pid'

+ KEEPALIVED_MASTER_LOG_FILE='/var/run/keepalived-ci/test-keepalived-master.log'

+ 

+ KEEPALIVED_SLAVE_CONFIG_FILE='/var/run/keepalived-ci/keepalived-slave.conf'

+ KEEPALIVED_SLAVE_PID_FILE='/var/run/keepalived-ci/test-keepalived-slave.pid'

+ KEEPALIVED_SLAVE_LOG_FILE='/var/run/keepalived-ci/test-keepalived-slave.log'

+ 

+ #                                            |-------namespace-----|

+ # MASTER<-vethtest---------------------------|-----vethpeer->SLAVE |

+ #--------------------------------------------|---------------------|

+ # Floated IP: 192.168.111.52

+ 

+ def setUpModule():

+     """Initialize the environment, and perform sanity checks on it."""

+ 

+     if shutil.which('keepalived') is None:

+         raise OSError(errno.ENOENT, 'keepalived not found')

+ 

+     if shutil.which('nc') is None:

+         raise OSError(errno.ENOENT, 'nc not found')

+ 

+     if shutil.which('socat') is None:

+         raise OSError(errno.ENOENT, 'socat not found')

+ 

+     if subprocess.call(['systemctl', 'is-active', '--quiet',

+                         'keepalived.service']) == 0:

+         raise unittest.SkipTest('keepalived is already active')

+ 

+ def tearDownModule():

+         pass

+ 

+ class GenericUtilities():

+     """Provide a set of utility functions start stop daemons. write config files etc """

+ 

+     def StartKeepAlivedMasterHost(self):

+         """ start keepalived in the host master mode """

+         subprocess.check_output(['systemctl', 'start', 'keepalived-master.service'])

+         output=subprocess.check_output(['systemctl', 'status', 'keepalived-master.service']).rstrip().decode('utf-8')

+         print(output)

+ 

+     def StopKeepAlivedMasterHost(self):

+         """ stop keepalived in the host master mode """

+         subprocess.check_output(['systemctl', 'stop', 'keepalived-master.service'])

+ 

+     def StartKeepAlivedSlaveNameSpace(self):

+         """start keepalived in the namespace slave mode """

+         subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'systemctl', 'start', 'keepalived-slave.service'])

+         output=subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'systemctl', 'status', 'keepalived-slave.service']).rstrip().decode('utf-8')

+         print(output)

+ 

+     def StopKeepAlivedSlaveNameSpace(self):

+         """stop keepalived in the namespace slave mode """

+         subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'systemctl', 'stop', 'keepalived-slave.service'])

+ 

+     def StartNetCatHost(self):

+         """Start netcat Master """

+         subprocess.check_output(['systemctl', 'start', 'nc-keepalived-test.service'])

+ 

+     def StopNetCatHost(self):

+         """Stop netcat Master """

+         subprocess.check_output(['systemctl', 'stop', 'nc-keepalived-test.service'])

+ 

+     def StartSocatNameSpace(self):

+         """Start socat backup """

+         subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'systemctl', 'start', 'socat-keepalived-test.service'])

+ 

+     def StopSocatNameSpace(self):

+         """Stop socat backup """

+         subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'systemctl', 'stop', 'socat-keepalived-test.service'])

+ 

+     def SetupNetwork(self):

+         """Setup veth interface and namespace """

+ 

+         # start the main network settings database:

+         ipdb_main = IPDB()

+         # start the same for a netns:

+         ipdb_test = IPDB(nl=NetNS('test-keepalived-ns'))

+ 

+         # create VETH

+         ipdb_main.create(ifname='vethtest', kind='veth', peer='vethpeer').commit()

+ 

+         # move peer VETH into the netns

+         with ipdb_main.interfaces.vethpeer as veth:

+             veth.net_ns_fd = 'test-keepalived-ns'

+ 

+         with ipdb_main.interfaces.vethtest as veth:

+             veth.add_ip('192.168.111.50/24')

+             veth.up()

+ 

+         with ipdb_test.interfaces.vethpeer as veth:

+             veth.add_ip('192.168.111.51/24')

+             veth.up()

+ 

+     def TearDownNetwork(self):

+         netns = NetNS('test-keepalived-ns')

+         netns.close()

+         netns.remove()

+ 

+ class KeepalivedTests(unittest.TestCase, GenericUtilities):

+ 

+     def setUp(self):

+         self.SetupNetwork()

+ 

+     def tearDown(self):

+         self.StopKeepAlivedMasterHost()

+         self.StopKeepAlivedSlaveNameSpace()

+ 

+         self.TearDownNetwork()

+ 

+         os.remove(KEEPALIVED_MASTER_LOG_FILE)

+         os.remove(KEEPALIVED_SLAVE_LOG_FILE)

+ 

+     def test_keepalived_failover(self):

+         """ Test Floating IP address is asigned to backup incase of failover (High Availability (HA)) """

+ 

+         self.StartNetCatHost()

+         self.StartSocatNameSpace()

+ 

+         time.sleep(5)

+         self.StartKeepAlivedMasterHost()

+         self.StartKeepAlivedSlaveNameSpace()

+         time.sleep(15)

+ 

+         print('\n----------------------------------------------------')

+         print('Floated IP 192.168.111.52 should be at master vethtest')

+         print('----------------------------------------------------')

+ 

+         output=subprocess.check_output(['ip','address', 'show', 'vethtest']).rstrip().decode('utf-8')

+         print(output)

+         self.assertRegex(output, "192.168.111.50")

+         self.assertRegex(output, "192.168.111.52")

+ 

+         print('----------------------------------------------------')

+         output=subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'ip','address', 'show', 'vethpeer']).rstrip().decode('utf-8')

+         print(output)

+         self.assertRegex(output, "192.168.111.51")

+ 

+         # start failover

+         print('---- *****----------- start failover ----------*****-----------')

+         self.StopNetCatHost()

+         time.sleep(15)

+         output=subprocess.check_output(['ip', 'netns', 'exec', 'test-keepalived-ns', 'ip', 'address', 'show', 'vethpeer']).rstrip().decode('utf-8')

+         print('----------------------------------------------------')

+         print('Floated IP 192.168.111.52 should be at slave which is now master vethpeer')

+         print('----------------------------------------------------')

+ 

+         print(output)

+         self.assertRegex(output, "192.168.111.52")

+ 

+         print('---------------Floated ip 192.168.111.52 should not be at vethtest-----------')

+         print('----------------------------------------------------------------------------')

+         output=subprocess.check_output(['ip','address', 'show', 'vethtest']).rstrip().decode('utf-8')

+         print(output)

+ 

+         self.assertNotRegex(output, "192.168.111.52")

+ 

+         self.StopSocatNameSpace()

+ 

+ if __name__ == '__main__':

+     unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout,

+                                                      verbosity=3))

@@ -0,0 +1,10 @@ 

+ [Unit]

+ Description=Socat test daemon

+ After=multi-user.target network.target

+ 

+ [Service]

+ Type=simple

+ GuessMainPID=true

+ ExecStart=/usr/bin/nc -l 5000

+ [Install]

+ WantedBy=multi-user.target

@@ -0,0 +1,53 @@ 

+ #!/bin/bash

+ # SPDX-License-Identifier: LGPL-2.1+

+ # ~~~

+ #   runtest.sh of keepalived

+ #   Description:  keepalived - Simple and robust facilities for loadbalancing and high-availability to Linux system

+ #

+ #   Author: Susant Sahani <susant@redhat.com>

+ #   Copyright (c) 2018 Red Hat, Inc.

+ # ~~~

+ 

+ # Include Beaker environment

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ PACKAGE_KEEPALIVED="keepalived"

+ PACKAGE_NETCAT="nmap-ncat"

+ PACKAGE_SOCAT="socat"

+ 

+ KEEPALIVED_CONF_DIR="/var/run/keepalived-ci"

+ KEEPALIVED_MASTER_CONFIG_FILE='/var/run/keepalived-ci/keepalived-master.conf'

+ KEEPALIVED_SLAVE_CONFIG_FILE='/var/run/keepalived-ci/keepalived-slave.conf'

+ 

+ rlJournalStart

+     rlPhaseStartSetup

+         rlAssertRpm $PACKAGE_KEEPALIVED

+         rlAssertRpm $PACKAGE_NETCAT

+         rlAssertRpm $PACKAGE_SOCAT

+ 

+         rlRun "systemctl stop firewalld" 0,5

+         rlRun "setenforce 0"

+         rlRun "cp keepalived-tests.py /usr/bin/"

+         rlRun "cp keepalived-master.service keepalived-slave.service nc-keepalived-test.service socat-keepalived-test.service /var/run/systemd/system"

+         rlRun "mkdir -p $KEEPALIVED_CONF_DIR"

+         rlRun "cp keepalived-master.conf $KEEPALIVED_MASTER_CONFIG_FILE"

+         rlRun "cp keepalived-slave.conf $KEEPALIVED_SLAVE_CONFIG_FILE"

+         rlRun "systemctl daemon-reload"

+     rlPhaseEnd

+ 

+     rlPhaseStartTest

+         rlLog "Starting keepalived tests ..."

+         rlRun "/usr/bin/python3 /usr/bin/keepalived-tests.py"

+     rlPhaseEnd

+ 

+     rlPhaseStartCleanup

+         rlRun "rm /usr/bin/keepalived-tests.py /var/run/systemd/system/nc-keepalived-test.service /var/run/systemd/system/socat-keepalived-test.service /var/run/systemd/system/keepalived-master.service /var/run/systemd/system/keepalived-slave.service $KEEPALIVED_CONFIG_FILE"

+         rlrun "rm -rf $KEEPALIVED_CONF_DIR"

+         rlRun "setenforce 1"

+         rlRun "systemctl daemon-reload"

+         rlLog "keepalived tests done"

+     rlPhaseEnd

+ rlJournalPrintText

+ rlJournalEnd

+ 

+ rlGetTestState

@@ -0,0 +1,10 @@ 

+ [Unit]

+ Description=socat test daemon

+ After=multi-user.target network.target

+ 

+ [Service]

+ Type=simple

+ GuessMainPID=true

+ ExecStart=/usr/bin/socat -u - tcp-listen:5001,reuseaddr

+ [Install]

+ WantedBy=multi-user.target

file added
+15

@@ -0,0 +1,15 @@ 

+ - hosts: localhost

+   roles:

+     - role: standard-test-beakerlib

+       tags:

+       - classic

+       tests:

+       - failover

+       required_packages:

+       - keepalived

+       - nmap-ncat

+       - socat

+       - systemd

+       - iproute

+       - python3

+       - python3-pyroute2

justification
Adds tests according to the CI wiki specifically the standard test interface in the spec.
The playbook includes Tier1 level test cases that have been tested in the following contexts and
is passing reliably: Classic. Test logs are stored in the artifacts directory.
The following steps are used to execute the tests using the standard test interface:

Test enveronment
Make sure you have installed packages from the spec

ansible-2.4.1.0-2.fc28.noarch
python2-dnf-2.7.5-1.fc28.noarch
libselinux-python-2.7-2.fc28.x86_64
standard-test-roles-2.5-1.fc28.noarch
Run tests for Classic
Snip of the example test run for Classic tests:

::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
::   Test
::   Test
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

:: [ 19:08:46 ] :: [   LOG    ] :: Starting keepalived tests ...
:: [ 19:08:46 ] :: [   LOG    ] :: Starting keepalived tests ...
:: [ 19:08:46 ] :: [  BEGIN   ] :: Running '/usr/bin/python3 /usr/bin/keepalived-tests.py'
test_keepalived_failover (__main__.KeepalivedTests)
Test Floating IP address is asigned to backup incase of failover (High Availability (HA)) ...
----------------------------------------------------
Floated IP 192.168.111.52 should be at master vethtest
----------------------------------------------------
7: vethtest@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 5e:cd:45:3a:06:b2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.50/24 scope global vethtest
       valid_lft forever preferred_lft forever
    inet 192.168.111.52/32 scope global vethtest
       valid_lft forever preferred_lft forever
    inet6 fe80::5ccd:45ff:fe3a:6b2/64 scope link
       valid_lft forever preferred_lft forever
----------------------------------------------------
6: vethpeer@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether ba:04:4a:ad:ae:dc brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.51/24 scope global vethpeer
       valid_lft forever preferred_lft forever
    inet6 fe80::b804:4aff:fead:aedc/64 scope link
       valid_lft forever preferred_lft forever
---- *****----------- start failover ----------*****-----------
----------------------------------------------------
Floated IP 192.168.111.52 should be at slave which is now master vethpeer
    link/ether 5e:cd:45:3a:06:b2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.50/24 scope global vethtest
       valid_lft forever preferred_lft forever
    inet 192.168.111.52/32 scope global vethtest
       valid_lft forever preferred_lft forever
    inet6 fe80::5ccd:45ff:fe3a:6b2/64 scope link
       valid_lft forever preferred_lft forever
----------------------------------------------------
6: vethpeer@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether ba:04:4a:ad:ae:dc brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.51/24 scope global vethpeer
       valid_lft forever preferred_lft forever
    inet6 fe80::b804:4aff:fead:aedc/64 scope link
       valid_lft forever preferred_lft forever
---- *****----------- start failover ----------*****-----------
----------------------------------------------------
Floated IP 192.168.111.52 should be at slave which is now master vethpeer
----------------------------------------------------
6: vethpeer@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether ba:04:4a:ad:ae:dc brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.51/24 scope global vethpeer
       valid_lft forever preferred_lft forever
    inet 192.168.111.52/32 scope global vethpeer
       valid_lft forever preferred_lft forever
    inet6 fe80::b804:4aff:fead:aedc/64 scope link
       valid_lft forever preferred_lft forever
---------------Floated ip 192.168.111.52 should not be at vethtest-----------
----------------------------------------------------------------------------
7: vethtest@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 5e:cd:45:3a:06:b2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.111.50/24 scope global vethtest
       valid_lft forever preferred_lft forever
    inet6 fe80::5ccd:45ff:fe3a:6b2/64 scope link
       valid_lft forever preferred_lft forever
ok

----------------------------------------------------------------------
Ran 1 test in 37.688s

OK
:: [ 19:09:24 ] :: [   PASS   ] :: Command '/usr/bin/python3 /usr/bin/keepalived-tests.py' (Expected 0, got 0)
:: [ 19:09:24 ] :: [   PASS   ] :: Command '/usr/bin/python3 /usr/bin/keepalived-tests.py' (Expected 0, got 0)
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
::   Duration: 38s
::   Duration: 38s
::   Assertions: 1 good, 0 bad
::   Assertions: 1 good, 0 bad
::   RESULT: PASS
::   RESULT: PASS

::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
::   Cleanup
::   Cleanup
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

rebased onto 7148053

a year ago

rebased onto 07b28f7

a year ago

rebased onto fb831b9

a year ago

rebased onto 66af7a4

a year ago

rebased onto eecc58e

a year ago