#!/usr/bin/python
# Copyright (c) 2017 Red Hat, Inc. All rights reserved. This copyrighted material
# is made available to anyone wishing to use, modify, copy, or
# redistribute it subject to the terms and conditions of the GNU General
# Public License v.2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; If not, see http://www.gnu.org/licenses/.
#
# Author: Jakub Krysl <jkrysl@redhat.com>
"""dmpd_library.py: Complete library providing functionality for device-mapper-persistent-data upstream test."""
import platform
from os.path import expanduser
import re #regex
import sys, os
import subprocess
import time
import fileinput
def _print(string):
module_name = __name__
string = re.sub("DEBUG:", "DEBUG:("+ module_name + ") ", string)
string = re.sub("FAIL:", "FAIL:("+ module_name + ") ", string)
string = re.sub("FATAL:", "FATAL:("+ module_name + ") ", string)
string = re.sub("WARN:", "WARN:("+ module_name + ") ", string)
print(string)
return
def run(cmd, return_output=False, verbose=True, force_flush=False):
"""Run a command line specified as cmd.
The arguments are:
\tcmd (str): Command to be executed
\tverbose: if we should show command output or not
\tforce_flush: if we want to show command output while command is being executed. eg. hba_test run
\treturn_output (Boolean): Set to True if want output result to be returned as tuple. Default is False
Returns:
\tint: Return code of the command executed
\tstr: As tuple of return code if return_output is set to True
"""
#by default print command output
if (verbose == True):
#Append time information to command
date = "date \"+%Y-%m-%d %H:%M:%S\""
p = subprocess.Popen(date, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
stdout, stderr = p.communicate()
stdout = stdout.rstrip("\n")
_print("INFO: [%s] Running: '%s'..." % (stdout, cmd))
#enabling shell=True, because was the only way I found to run command with '|'
if not force_flush:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
stdout, stderr = p.communicate()
sys.stdout.flush()
sys.stderr.flush()
else:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
stdout = ""
stderr = ""
while p.poll() is None:
new_data = p.stdout.readline()
stdout += new_data
if verbose:
sys.stdout.write(new_data)
sys.stdout.flush()
retcode = p.returncode
output = stdout + stderr
#remove new line from last line
output = output.rstrip()
#by default print command output
#if force_flush we already printed it
if verbose == True and not force_flush:
print(output)
if return_output == False:
return retcode
else:
return retcode, output
def atomic_run(message, success=True, return_output=False, **kwargs):
errors = kwargs.pop("errors")
command = kwargs.pop("command")
params = []
for a in kwargs:
params.append(str(a) + " = " + str(kwargs[a]))
params = ", ".join([str(i) for i in params])
_print("\nINFO: " + message + " with params %s" % params)
if return_output:
kwargs["return_output"] = True
ret, output = command(**kwargs)
else:
ret = command(**kwargs)
expected_break = {True: False, False: True}
print("(Returned, Expected)")
if command == run:
expected_break = {True: 1, False: 0}
if ret in expected_break:
print(not expected_break[ret], success)
else:
print(ret, success)
else:
print(ret, success)
if ret == expected_break[success]:
error = "FAIL: " + message + " with params %s failed" % params
_print(error)
errors.append(error)
sleep(0.2)
if return_output:
return output
else:
return ret
def sleep(duration):
"""
It basically call sys.sleep, but as stdout and stderr can be buffered
We flush them before sleep
"""
sys.stdout.flush()
sys.stderr.flush()
time.sleep(duration)
return
def mkdir(new_dir):
if os.path.isdir(new_dir):
_print("INFO: %s already exist" % new_dir)
return True
cmd = "mkdir -p %s" % new_dir
retcode, output = run(cmd, return_output=True, verbose=False)
if retcode != 0:
_print("WARN: could create directory %s" % new_dir)
print(output)
return False
return True
def dist_release():
"""
Find out the release number of Linux distribution.
"""
dist = platform.linux_distribution()
if not dist or dist[1] == "":
_print("WARN: dist_release() - Could not determine dist release")
return None
return dist[1]
def dist_ver():
"""
Check the Linux distribution version.
"""
release = dist_release()
if not release:
return None
m = re.match("(\d+).\d+", release)
if m:
return int(m.group(1))
# See if it is only digits, in that case return it
m = re.match("(\d+)", release)
if m:
return int(m.group(1))
_print("WARN: dist_ver() - Invalid release output %s" % release)
return None
def show_sys_info():
print("### Kernel Info: ###")
ret, kernel = run ("uname -a", return_output=True, verbose=False)
ret, taint_val = run("cat /proc/sys/kernel/tainted", return_output=True, verbose=False)
print("Kernel version: %s" % kernel)
print("Kernel tainted: %s" % taint_val)
print("### IP settings: ###")
run("ip a")
if run("rpm -q device-mapper-multipath") == 0:
#Abort test execution if multipath is not working well
if run("multipath -l 2>/dev/null") != 0:
sys.exit(1)
#Flush all unused multipath devices before starting the test
run("multipath -F")
run("multipath -r")
def get_free_space(path):
"""
Get free space of a path.
Path could be:
\t/dev/sda
\t/root
\t./
"""
if not path:
return None
cmd = "df -B 1 %s" % (path)
retcode, output = run(cmd, return_output=True, verbose=False)
if retcode != 0:
_print("WARN: get_free_space() - could not run %s" % (cmd))
print(output)
return None
fs_list = output.split("\n")
# delete the header info
del fs_list[0]
if len(fs_list) > 1:
#Could be the information was too long and splited in lines
tmp_info = "".join(fs_list)
fs_list[0] = tmp_info
#expected order
#Filesystem 1B-blocks Used Available Use% Mounted on
free_space_regex = re.compile("\S+\s+\d+\s+\d+\s+(\d+)")
m = free_space_regex.search(fs_list[0])
if m:
return int(m.group(1))
return None
def size_human_2_size_bytes(size_human):
"""
Usage
size_human_2_size_bytes(size_human)
Purpose
Convert human readable stander size to B
Parameter
size_human # like '1KiB'
Returns
size_bytes # like 1024
"""
if not size_human:
return None
# make sure size_human is a string, could be only numbers, for example
size_human = str(size_human)
if not re.search("\d", size_human):
# Need at least 1 digit
return None
size_human_regex = re.compile("([\-0-9\.]+)(Ki|Mi|Gi|Ti|Ei|Zi){0,1}B$")
m = size_human_regex.match(size_human)
if not m:
if re.match("^\d+$", size_human):
# Assume size is already in bytes
return size_human
_print("WARN: '%s' is an invalid human size format" % size_human)
return None
number = None
fraction = 0
# check if number is fractional
f = re.match("(\d+)\.(\d+)", m.group(1))
if f:
number = int(f.group(1))
fraction = int(f.group(2))
else:
number = int(m.group(1))
unit = m.group(2)
if not unit:
unit = 'B'
for valid_unit in ['B', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if unit == valid_unit:
if unit == 'B':
# cut any fraction if was given, as it is not valid
return str(number)
return int(number + fraction)
number *= 1024
fraction *= 1024
fraction /= 10
return int(number + fraction)
def size_bytes_2_size_human(num):
if not num:
return None
#Even if we receive string we convert so we can process it
num = int(num)
for unit in ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB']:
if abs(num) < 1024.0:
size_human = "%3.1f%s" % (num, unit)
#round it down removing decimal numbers
size_human = re.sub("\.\d+", "", size_human)
return size_human
num /= 1024.0
#Very big number!!
size_human = "%.1f%s" % (num, 'Yi')
#round it down removing decimal numbers
size_human = re.sub("\.\d+", "", size_human)
return size_human
def install_package(pack):
"""
Install a package "pack" via `yum install -y `
"""
#Check if package is already installed
ret, ver = run("rpm -q %s" % pack, verbose=False, return_output=True)
if ret == 0:
_print("INFO: %s is already installed (%s)" % (pack, ver))
return True
if run("yum install -y %s" % pack) != 0:
msg = "FAIL: Could not install %s" % pack
_print(msg)
return False
_print("INFO: %s was successfully installed" % pack)
return True
def create_filesystem(vg_name, lv_name, filesystem="xfs"):
if filesystem not in ["xfs", "ext4", "btrfs"]:
_print("WARN: Unknown filesystem.")
return False
if run("mkfs.%s /dev/%s/%s" % (filesystem, vg_name, lv_name), verbose=True) != 0:
_print("WARN: Could not create filesystem %s on %s/%s" % (filesystem, vg_name, lv_name))
return False
return True
def metadata_snapshot(vg_name, lv_name):
if run("dmsetup suspend /dev/mapper/%s-%s-tpool" % (vg_name, lv_name), verbose=True) != 0:
_print("WARN: Device mapper could not suspend /dev/mapper/%s-%s-tpool" % (vg_name, lv_name))
return False
if run("dmsetup message /dev/mapper/%s-%s-tpool 0 reserve_metadata_snap" % (vg_name, lv_name), verbose=True) != 0:
_print("WARN: Device mapper could not create metadata snaphot on /dev/mapper/%s-%s-tpool" % (vg_name, lv_name))
return False
if run("dmsetup resume /dev/mapper/%s-%s-tpool" % (vg_name, lv_name), verbose=True) != 0:
_print("WARN: Device mapper could not resume /dev/mapper/%s-%s-tpool" % (vg_name, lv_name))
return False
return True
class LogChecker:
def __init__(self):
segfault_msg = " segfault "
calltrace_msg = "Call Trace:"
self.error_mgs = [segfault_msg, calltrace_msg]
def check_all(self):
"""Check for error on the system
Returns:
\tBoolean:
\t\tTrue is no error was found
\t\tFalse if some error was found
"""
_print("INFO: Checking for error on the system")
error = 0
if not self.kernel_check():
error += 1
if not self.abrt_check():
error += 1
if not self.messages_dump_check():
error += 1
if not self.dmesg_check():
error += 1
if not self.console_log_check():
error += 1
if not self.kdump_check():
error += 1
if error:
log_messages = "/var/log/messages"
if os.path.isfile(log_messages):
print("submit %s, named messages.log" % log_messages)
run("cp %s messages.log" % log_messages)
run("rhts-submit-log -l messages.log")
_print("INFO: Umounting NFS to avoid sosreport being hang there")
run("umount /var/crash")
ret_code = run("which sosreport", verbose=False)
if ret_code != 0:
_print("WARN: sosreport is not installed")
_print("INFO: Mounting NFS again")
run("mount /var/crash")
return False
print("Generating sosreport log")
disable_plugin = ""
if run("sosreport --list-plugins | grep emc") == 0:
disable_plugin = "-n emc"
ret_code, sosreport_log = run("sosreport --batch %s" % disable_plugin, return_output=True)
if ret_code != 0:
_print("WARN: sosreport command failed")
_print("INFO: Mounting NFS again")
run("mount /var/crash")
return False
sos_lines = sosreport_log.split("\n")
sos_file = None
for line in sos_lines:
#In RHEL7 sosreport is saving under /var/tmp while RHEL6 uses /tmp...
m = re.match(r"\s+((\/var)?\/tmp\/sosreport\S+)", line)
if m:
sos_file = m.group(1)
if not sos_file:
_print("WARN: could not save sosreport log")
_print("INFO: Mounting NFS again")
run("mount /var/crash")
return False
run("rhts-submit-log -l %s" % sos_file)
_print("INFO: Mounting NFS again")
run("mount /var/crash")
return False
return True
@staticmethod
def abrt_check():
"""Check if abrtd found any issue
Returns:
\tBoolean:
\t\tTrue no error was found
\t\tFalse some error was found
"""
_print("INFO: Checking abrt for error")
if run("rpm -q abrt", verbose=False) != 0:
_print("WARN: abrt tool does not seem to be installed")
_print("WARN: skipping abrt check")
return True
if run("pidof abrtd", verbose=False) != 0:
_print("WARN: abrtd is not running")
return False
ret, log = run("abrt-cli list", return_output=True)
if ret != 0:
_print("WARN: abrt-cli command failed")
return False
# We try to match for "Directory" to check if
# abrt-cli list is actually listing any issue
error = False
if log:
lines = log.split("\n")
for line in lines:
m = re.match(r"Directory:\s+(\S+)", line)
if m:
directory = m.group(1)
filename = directory
filename = filename.replace(":", "-")
filename += ".tar.gz"
run("tar cfzP %s %s" % (filename, directory))
run("rhts-submit-log -l %s" % filename)
# if log is saved on beaker, it can be deleted from server
# it avoids next test from detecting this failure
run("abrt-cli rm %s" % directory)
error = True
if error:
_print("WARN: Found abrt error")
return False
_print("PASS: no Abrt entry has been found.")
return True
@staticmethod
def kernel_check():
"""
Check if kernel got tainted.
It checks /proc/sys/kernel/tainted which returns a bitmask.
The values are defined in the kernel source file include/linux/kernel.h,
and explained in kernel/panic.c
cd /usr/src/kernels/`uname -r`/
Sources are provided by kernel-devel
Returns:
\tBoolean:
\t\tTrue if did not find any issue
\t\tFalse if found some issue
"""
_print("INFO: Checking for tainted kernel")
previous_tainted_file = "/tmp/previous-tainted"
ret, tainted = run("cat /proc/sys/kernel/tainted", return_output=True)
tainted_val = int(tainted)
if tainted_val == 0:
run("echo %d > %s" % (tainted_val, previous_tainted_file), verbose=False)
_print("PASS: Kernel is not tainted.")
return True
_print("WARN: Kernel is tainted!")
if not os.path.isfile(previous_tainted_file):
run("echo 0 > %s" % previous_tainted_file, verbose=False)
ret, prev_taint = run("cat %s" % previous_tainted_file, return_output=True)
prev_taint_val = int(prev_taint)
if prev_taint_val == tainted_val:
_print("INFO: Kernel tainted has already been handled")
return True
run("echo %d > %s" % (tainted_val, previous_tainted_file), verbose=False)
# check all bits that are set
bit = 0
while tainted_val != 0:
if tainted_val & 1:
_print("\tTAINT bit %d is set\n" % bit)
bit += 1
# shift tainted value
tainted_val /= 2
# List all tainted bits that are defined
print("List bit definition for tainted kernel")
run("cat /usr/src/kernels/`uname -r`/include/linux/kernel.h | grep TAINT_")
found_issue = False
# try to find the module which tainted the kernel, tainted module have a mark between '('')'
ret, output = run("cat /proc/modules | grep -e '(.*)' | cut -d' ' -f1", return_output=True)
tainted_mods = output.split("\n")
# For example during iscsi async_events scst tool loads an unsigned module
# just ignores it, so we will ignore this tainted if there is no tainted
# modules loaded
if not tainted_mods:
_print("INFO: ignoring tainted as the module is not loaded anymore")
else:
ignore_modules = ["ocrdma", "nvme_fc", "nvmet_fc"]
for tainted_mod in tainted_mods:
if tainted_mod:
_print("INFO: The following module got tainted: %s" % tainted_mod)
run("modinfo %s" % tainted_mod)
# we are ignoring ocrdma module
if tainted_mod in ignore_modules:
_print("INFO: ignoring tainted on %s" % tainted_mod)
run("echo %d > %s" % (tainted_val, previous_tainted_file), verbose=False)
continue
found_issue = True
run("echo %s > %s" % (tainted, previous_tainted_file), verbose=False)
if found_issue:
return False
return True
@staticmethod
def _date2num(date):
date_map = {"Jan": "1",
"Feb": "2",
"Mar": "3",
"Apr": "4",
"May": "5",
"Jun": "6",
"Jul": "7",
"Aug": "8",
"Sep": "9",
"Oct": "10",
"Nov": "11",
"Dec": "12"}
date_regex = r"(\S\S\S)\s(\d+)\s(\d\d:\d\d:\d\d)"
m = re.match(date_regex, date)
month = date_map[m.group(1)]
day = str(m.group(2))
# if day is a single digit, add '0' to begin
if len(day) == 1:
day = "0" + day
hour = m.group(3)
hour = hour.replace(":", "")
value = month + day + hour
return value
@staticmethod
def clear_dmesg():
cmd = "dmesg --clear"
if dist_ver() < 7:
cmd = "dmesg -c"
run(cmd, verbose=False)
return True
def messages_dump_check(self):
previous_time_file = "/tmp/previous-dump-check"
log_msg_file = "/var/log/messages"
if not os.path.isfile(log_msg_file):
_print("WARN: Could not open %s" % log_msg_file)
return True
log_file = open(log_msg_file)
log = log_file.read()
begin_tag = "\\[ cut here \\]"
end_tag = "\\[ end trace "
if not os.path.isfile(previous_time_file):
first_time = "Jan 01 00:00:00"
time = self._date2num(first_time)
run("echo %s > %s" % (time, previous_time_file))
# Read the last time test ran
ret, last_run = run("cat %s" % previous_time_file, return_output=True)
_print("INFO: Checking for stack dump messages after: %s" % last_run)
# Going to search the file for text that matches begin_tag until end_tag
dump_regex = begin_tag + "(.*?)" + end_tag
m = re.findall(dump_regex, log, re.MULTILINE)
if m:
_print("INFO: Checking if it is newer than: %s" % last_run)
print(m.group(1))
# TODO
_print("PASS: No recent dump messages has been found.")
return True
def dmesg_check(self):
"""
Check for error messages on dmesg ("Call Trace and segfault")
"""
_print("INFO: Checking for errors on dmesg.")
error = 0
for msg in self.error_mgs:
ret, output = run("dmesg | grep -i '%s'" % msg, return_output=True)
if output:
_print("WARN: found %s on dmesg" % msg)
run("echo '\nINFO found %s Saving it\n'>> dmesg.log" % msg)
run("dmesg >> dmesg.log")
run("rhts-submit-log -l dmesg.log")
error = + 1
self.clear_dmesg()
if error:
return False
_print("PASS: No errors on dmesg have been found.")
return True
def console_log_check(self):
"""
Checks for error messages on console log ("Call Trace and segfault")
"""
error = 0
console_log_file = "/root/console.log"
prev_console_log_file = "/root/console.log.prev"
new_console_log_file = "/root/console.log.new"
if not os.environ.get('LAB_CONTROLLER'):
_print("WARN: Could not find lab controller")
return True
if not os.environ.get('RECIPEID'):
_print("WARN: Could not find recipe ID")
return True
lab_controller = os.environ['LAB_CONTROLLER']
recipe_id = os.environ['RECIPEID']
# get current console log
url = "http://%s:8000/recipes/%s/logs/console.log" % (lab_controller, recipe_id)
if (run("wget -q %s -O %s" % (url, new_console_log_file)) != 0):
_print("INFO: Could not get console log")
# return sucess if could not get console.log
return True
# if there was previous console log, we just check the new part
run("diff -N -n --unidirectional-new-file %s %s > %s" % (
prev_console_log_file, new_console_log_file, console_log_file))
# backup the current full console.log
# next time we run the test we will compare just
# what has been appended to console.log
run("mv -f %s %s" % (new_console_log_file, prev_console_log_file))
_print("INFO: Checking for errors on %s" % console_log_file)
for msg in self.error_mgs:
ret, output = run("cat %s | grep -i '%s'" % (console_log_file, msg), return_output=True)
if output:
_print("INFO found %s on %s" % (msg, console_log_file))
run("rhts-submit-log -l %s" % console_log_file)
error = + 1
if error:
return False
_print("PASS: No errors on %s have been found." % console_log_file)
return True
@staticmethod
def kdump_check():
"""
Check for kdump error messages.
It assumes kdump is configured on /var/crash
"""
error = 0
previous_kdump_check_file = "/tmp/previous-kdump-check"
kdump_dir = "/var/crash"
ret, hostname = run("hostname", verbose=False, return_output=True)
if not os.path.exists("%s/%s" % (kdump_dir, hostname)):
_print("INFO: No kdump log found for this server")
return True
ret, output = run("ls -l %s/%s | awk '{print$9}'" % (kdump_dir, hostname), return_output=True)
kdumps = output.split("\n")
kdump_dates = []
for kdump in kdumps:
if kdump == "":
continue
# parse on the date, remove the ip of the uploader
m = re.match(".*?-(.*)", kdump)
if not m:
_print("WARN: unexpected format for kdump (%s)" % kdump)
continue
date = m.group(1)
# Old dump were using "."
date = date.replace(r"\.", "-")
# replace last "-" with space to format date properly
index = date.rfind("-")
date = date[:index] + " " + date[index + 1:]
_print("INFO: Found kdump from %s" % date)
kdump_dates.append(date)
# checks if a file to store last run exists, if not create it
if not os.path.isfile("%s" % previous_kdump_check_file):
# time in seconds
ret, time = run(r"date +\"\%s\"", verbose=False, return_output=True)
run("echo -n %s > %s" % (time, previous_kdump_check_file), verbose=False)
_print("INFO: kdump check is executing for the first time.")
_print("INFO: doesn't know from which date should check files.")
_print("PASS: Returning success.")
return True
# Read the last time test ran
ret, previous_check_time = run("cat %s" % previous_kdump_check_file, return_output=True)
# just add new line to terminal because the file should not have already new line character
print("")
for date in kdump_dates:
# Note %% is escape form to use % in a string
ret, kdump_time = run("date --date=\"%s\" +%%s" % date, return_output=True)
if ret != 0:
_print("WARN: Could not convert date %s" % date)
continue
if not kdump_time:
continue
if (int(kdump_time) > int(previous_check_time)):
_print("WARN: Found a kdump log from %s (more recent than %s)" % (date, previous_check_time))
_print("WARN: Check %s/%s" % (kdump_dir, hostname))
error += 1
ret, time = run(r"date +\"\%s\"", verbose=False, return_output=True)
run("echo -n %s > %s" % (time, previous_kdump_check_file), verbose=False)
if error:
return False
_print("PASS: No errors on kdump have been found.")
return True
class TestClass:
#we currently support these exit code for a test case
tc_sup_status = {"pass" : "PASS: ",
"fail" : "ERROR: ",
"skip" : "SKIP: "}
tc_pass = []
tc_fail = []
tc_skip = [] #For some reason it did not execute
tc_results = [] #Test results stored in a list
test_dir = "%s/.stqe-test" % expanduser("~")
test_log = "%s/test.log" % test_dir
def __init__(self):
print("################################## Test Init ###################################")
self.log_checker = LogChecker()
if not os.path.isdir(self.test_dir):
mkdir(self.test_dir)
# read entries on test.log, there will be entries if tend was not called
# before starting a TC class again, usually if the test case reboots the server
if not os.path.isfile(self.test_log):
#running the test for the first time
show_sys_info()
#Track memory usage during test
run("free -b > init_mem.txt", verbose=False)
run("top -b -n 1 > init_top.txt", verbose=False)
else:
try:
f = open(self.test_log)
file_data = f.read()
f.close()
except:
_print("WARN: TestClass() could not read %s" % self.test_log)
return
finally:
f.close()
log_entries = file_data.split("\n")
#remove the file, once tlog is ran it will add the entries again...
run("rm -f %s" % (self.test_log), verbose=False)
if log_entries:
_print("INFO: Loading test result from previous test run...")
for entry in log_entries:
self.tlog(entry)
print("################################################################################")
return
def tlog(self, string):
"""print message, if message begins with supported message status
the test message will be added to specific test result array
"""
print(string)
if re.match(self.tc_sup_status["pass"], string):
self.tc_pass.append(string)
self.tc_results.append(string)
run("echo '%s' >> %s" % (string, self.test_log), verbose=False)
if re.match(self.tc_sup_status["fail"], string):
self.tc_fail.append(string)
self.tc_results.append(string)
run("echo '%s' >> %s" % (string, self.test_log), verbose=False)
if re.match(self.tc_sup_status["skip"], string):
self.tc_skip.append(string)
self.tc_results.append(string)
run("echo '%s' >> %s" % (string, self.test_log), verbose=False)
return None
@staticmethod
def trun(cmd, return_output=False):
"""Run the cmd and format the log. return the exitint status of cmd
The arguments are:
\tCommand to run
\treturn_output: if should return command output as well (Boolean)
Returns:
\tint: Command exit code
\tstr: command output (optional)
"""
return run(cmd, return_output)
def tok(self, cmd, return_output=False):
"""Run the cmd and expect it to pass.
The arguments are:
\tCommand to run
\treturn_output: if should return command output as well (Boolean)
Returns:
\tBoolean: return_code
\t\tTrue: If command excuted successfully
\t\tFalse: Something went wrong
\tstr: command output (optional)
"""
cmd_code = None
ouput = None
ret_code = None
if not return_output:
cmd_code = run(cmd)
else:
cmd_code, output = run(cmd, return_output)
if cmd_code == 0:
self.tpass(cmd)
ret_code = True
else:
self.tfail(cmd)
ret_code = False
if return_output:
return ret_code, output
else:
return ret_code
def tnok(self, cmd, return_output=False):
"""Run the cmd and expect it to fail.
The arguments are:
\tCommand to run
\treturn_output: if should return command output as well (Boolean)
Returns:
\tBoolean: return_code
\t\tFalse: If command excuted successfully
\t\tTrue: Something went wrong
\tstr: command output (optional)
"""
cmd_code = None
ouput = None
ret_code = None
if not return_output:
cmd_code = run(cmd)
else:
cmd_code, output = run(cmd, return_output)
if cmd_code != 0:
self.tpass(cmd + " [exited with error, as expected]")
ret_code = True
else:
self.tfail(cmd + " [expected to fail, but it did not]")
ret_code = False
if return_output:
return ret_code, output
else:
return ret_code
def tpass(self, string):
"""Will add PASS + string to test log summary
"""
self.tlog(self.tc_sup_status["pass"] + string)
return None
def tfail(self, string):
"""Will add ERROR + string to test log summary
"""
self.tlog(self.tc_sup_status["fail"] + string)
return None
def tskip(self, string):
"""Will add SKIP + string to test log summary
"""
self.tlog(self.tc_sup_status["skip"] + string)
return None
def tend(self):
"""It checks for error in the system and print test summary
Returns:
\tBoolean
\t\tTrue if all test passed and no error was found on server
\t\tFalse if some test failed or found error on server
"""
if self.log_checker.check_all():
self.tpass("Search for error on the server")
else:
self.tfail("Search for error on the server")
print("################################ Test Summary ##################################")
#Will print test results in order and not by test result order
for tc in self.tc_results:
print(tc)
n_tc_pass = len(self.tc_pass)
n_tc_fail = len(self.tc_fail)
n_tc_skip = len(self.tc_skip)
print("#############################")
print("Total tests that passed: " + str(n_tc_pass))
print("Total tests that failed: " + str(n_tc_fail))
print("Total tests that skipped: " + str(n_tc_skip))
print("################################################################################")
sys.stdout.flush()
#Added this sleep otherwise some of the prints were not being shown....
sleep(1)
run("rm -f %s" % (self.test_log), verbose=False)
run("rmdir %s" % (self.test_dir), verbose=False)
#If at least one test failed, return error
if n_tc_fail > 0:
return False
return True
class LoopDev:
def __init__(self):
self.image_path = "/tmp"
@staticmethod
def _get_loop_path(name):
loop_path = name
if "/dev/" not in name:
loop_path = "/dev/" + name
return loop_path
@staticmethod
def _get_image_file(name, image_path):
image_file = "%s/%s.img" % (image_path, name)
return image_file
@staticmethod
def _standardize_name(name):
"""
Make sure use same standard for name, for example remove /dev/ from it if exists
"""
if not name:
_print("WARN: _standardize_name() - requires name as parameter")
return None
return name.replace("/dev/", "")
def create_loopdev(self, name=None, size=1024):
"""
Create a loop device
Parameters:
\tname: eg. loop0 (optional)
\tsize: Size in MB (default: 1024MB)
"""
ret_fail = False
if not name:
cmd = "losetup -f"
retcode, output = run(cmd, return_output=True, verbose=False)
if retcode != 0:
_print("WARN: Could not find free loop device")
print(output)
return None
name = output
ret_fail = None
name = self._standardize_name(name)
fname = self._get_image_file(name, self.image_path)
_print("INFO: Creating loop device %s with size %d" % (fname, size))
_print("INFO: Checking if %s exists" % fname)
if not os.path.isfile(fname):
# make sure we have enough space to create the file
free_space_bytes = get_free_space(self.image_path)
# Convert the size given in megabytes to bytes
size_bytes = int(size_human_2_size_bytes("%sMiB" % size))
if free_space_bytes <= size_bytes:
_print("WARN: Not enough space to create loop device with size %s"
% size_bytes_2_size_human(size_bytes))
_print("available space: %s" % size_bytes_2_size_human(free_space_bytes))
return ret_fail
_print("INFO: Creating file %s" % fname)
# cmd = "dd if=/dev/zero of=%s seek=%d bs=1M count=0" % (fname, size)
cmd = "fallocate -l %sM %s" % (size, fname)
try:
# We are just creating the file, not writting zeros to it
retcode = run(cmd)
if retcode != 0:
_print("command failed with code %s" % retcode)
_print("WARN: Could not create loop device image file")
return ret_fail
except OSError as e:
print >> sys.err, "command failed: ", e
return ret_fail
loop_path = self._get_loop_path(name)
# detach loop device if it exists
self.detach_loopdev(loop_path)
# Going to associate the file to the loopdevice
cmd = "losetup %s %s" % (loop_path, fname)
retcode = run(cmd)
if retcode != 0:
_print("WARN: Could not create loop device")
return ret_fail
if ret_fail is None:
return loop_path
return True
def delete_loopdev(self, name):
"""
Delete a loop device
Parameters:
\tname: eg. loop0 or /dev/loop0
"""
if not name:
_print("WARN: delete_loopdev() - requires name parameter")
return False
_print("INFO: Deleting loop device %s" % name)
name = self._standardize_name(name)
loop_path = self._get_loop_path(name)
# detach loop device if it exists
if not self.detach_loopdev(name):
_print("WARN: could not detach %s" % loop_path)
return False
fname = self._get_image_file(name, self.image_path)
if os.path.isfile(fname):
cmd = "rm -f %s" % fname
retcode = run(cmd)
if retcode != 0:
_print("WARN: Could not delete loop device %s" % name)
return False
# check if loopdev file is deleted as it sometimes remains
if os.path.isfile(fname):
_print("WARN: Deleted loop device file %s but it is still there" % fname)
return False
return True
@staticmethod
def get_loopdev():
# example of output on rhel-6.7
# /dev/loop0: [fd00]:396428 (/tmp/loop0.img)
retcode, output = run("losetup -a | awk '{print$1}'", return_output=True, verbose=False)
# retcode, output = run("losetup -l | tail -n +2", return_output=True, verbose=False)
if (retcode != 0):
_print("WARN: get_loopdev failed to execute")
print(output)
return None
devs = None
if output:
devs = output.split("\n")
# remove the ":" character from all devices
devs = [d.replace(':', "") for d in devs]
return devs
def detach_loopdev(self, name=None):
cmd = "losetup -D"
if name:
devs = self.get_loopdev()
if not devs:
# No device was found
return False
name = self._standardize_name(name)
# Just try to detach if device is connected, otherwise ignore
# print "INFO: Checking if ", loop_path, " exists, to be detached"
dev_path = self._get_loop_path(name)
if dev_path in devs:
cmd = "losetup -d %s" % dev_path
else:
# if loop device does not exist just ignore it
return True
# run losetup -D or -d <device>
retcode = run(cmd)
if retcode != 0:
_print("WARN: Could not detach loop device")
return False
return True
class LVM:
###########################################
# VG section
###########################################
@staticmethod
def vg_query(verbose=False):
"""Query Volume Groups and return a dictonary with VG information for each VG.
The arguments are:
\tNone
Returns:
\tdict: Return a dictionary with VG info for each VG
"""
cmd = "vgs --noheadings --separator \",\""
retcode, output = run(cmd, return_output=True, verbose=verbose)
if (retcode != 0):
_print("INFO: there is no VGs")
return None
vgs = output.split("\n")
# format of VG info: name #PV #LV #SN Attr VSize VFree
vg_info_regex = "\s+(\S+),(\S+),(\S+),(.*),(.*),(.*),(.*)$"
vg_dict = {}
for vg in vgs:
m = re.match(vg_info_regex, vg)
if not m:
continue
vg_info_dict = {"num_pvs": m.group(2),
"num_lvs": m.group(3),
"num_sn": m.group(4), # not sure what it is
"attr": m.group(5),
"vsize": m.group(6),
"vfree": m.group(7)}
vg_dict[m.group(1)] = vg_info_dict
return vg_dict
@staticmethod
def vg_create(vg_name, pv_name, force=False, verbose=True):
"""Create a Volume Group.
The arguments are:
\tPV name
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not vg_name or not pv_name:
_print("WARN: vg_create requires vg_name and pv_name")
return False
options = ""
if force:
options += "--force"
cmd = "vgcreate %s %s %s" % (options, vg_name, pv_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
# _print ("WARN: Could not create %s" % vg_name)
return False
return True
def vg_remove(self, vg_name, force=False, verbose=True):
"""Delete a Volume Group.
The arguments are:
\tVG name
\tforce (boolean)
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not vg_name:
_print("WARN: vg_remove requires vg_name")
return False
vg_dict = self.vg_query()
if vg_name not in vg_dict.keys():
_print("INFO: vg_remove - %s does not exist. Skipping..." % vg_name)
return True
options = ""
if force:
options += "--force"
cmd = "vgremove %s %s" % (options, vg_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
return False
return True
###########################################
# LV section
###########################################
@staticmethod
def lv_query(options=None, verbose=False):
"""Query Logical Volumes and return a dictonary with LV information for each LV.
The arguments are:
\toptions: If not want to use default lvs output. Use -o for no default fields
Returns:
\tdict: Return a list with LV info for each LV
"""
# Use \",\" as separator, as some output might contain ','
# For example, lvs -o modules on thin device returns "thin,thin-pool"
cmd = "lvs -a --noheadings --separator \\\",\\\""
# format of LV info: Name VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert
lv_info_regex = "\s+(\S+)\",\"(\S+)\",\"(\S+)\",\"(\S+)\",\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)$"
# default parameters returned by lvs -a
param_names = ["name", "vg_name", "attr", "size", "pool", "origin", "data_per", "meta_per", "move", "log",
"copy_per", "convert"]
if options:
param_names = ["name", "vg_name"]
# need to change default regex
lv_info_regex = "\s+(\S+)\",\"(\S+)"
parameters = options.split(",")
for param in parameters:
lv_info_regex += "\",\"(.*)"
param_names.append(param)
lv_info_regex += "$"
cmd += " -o lv_name,vg_name,%s" % options
retcode, output = run(cmd, return_output=True, verbose=verbose)
if (retcode != 0):
_print("INFO: there is no LVs")
return None
lvs = output.split("\n")
lv_list = []
for lv in lvs:
m = re.match(lv_info_regex, lv)
if not m:
_print("WARN: (%s) does not match lvs output format" % lv)
continue
lv_info_dict = {}
for index in xrange(len(param_names)):
lv_info_dict[param_names[index]] = m.group(index + 1)
lv_list.append(lv_info_dict)
return lv_list
def lv_info(self, lv_name, vg_name, options=None, verbose=False):
"""
Show information of specific LV
"""
if not lv_name or not vg_name:
_print("WARN: lv_info() - requires lv_name and vg_name as parameters")
return None
lvs = self.lv_query(options=options, verbose=verbose)
if not lvs:
return None
for lv in lvs:
if (lv["name"] == lv_name and
lv["vg_name"] == vg_name):
return lv
return None
@staticmethod
def lv_create(vg_name, lv_name, options=(""), verbose=True):
"""Create a Logical Volume.
The arguments are:
\tVG name
\tLV name
\toptions
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not vg_name or not lv_name:
_print("WARN: lv_create requires vg_name and lv_name")
return False
cmd = "lvcreate %s %s -n %s" % (" ".join(str(i) for i in options), vg_name, lv_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
# _print ("WARN: Could not create %s" % lv_name)
return False
return True
@staticmethod
def lv_activate(lv_name, vg_name, verbose=True):
"""Activate a Logical Volume
The arguments are:
\tLV name
\tVG name
Returns:
\tBoolean:
\t\tTrue in case of success
\t\tFalse if something went wrong
"""
if not lv_name or not vg_name:
_print("WARN: lv_activate requires lv_name and vg_name")
return False
cmd = "lvchange -ay %s/%s" % (vg_name, lv_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
_print("WARN: Could not activate LV %s" % lv_name)
return False
# Maybe we should query the LVs and make sure it is really activated
return True
@staticmethod
def lv_deactivate(lv_name, vg_name, verbose=True):
"""Deactivate a Logical Volume
The arguments are:
\tLV name
\tVG name
Returns:
\tBoolean:
\t\tTrue in case of success
\t\tFalse if something went wrong
"""
if not lv_name or not vg_name:
_print("WARN: lv_deactivate requires lv_name and vg_name")
return False
cmd = "lvchange -an %s/%s" % (vg_name, lv_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
_print("WARN: Could not deactivate LV %s" % lv_name)
return False
# Maybe we should query the LVs and make sure it is really deactivated
return True
def lv_remove(self, lv_name, vg_name, verbose=True):
"""Remove an LV from a VG
The arguments are:
\tLV name
\tVG name
Returns:
\tBoolean:
\t\tTrue in case of success
\t\tFalse if something went wrong
"""
if not lv_name or not vg_name:
_print("WARN: lv_remove requires lv_name and vg_name")
return False
lvs = self.lv_query()
lv_names = lv_name.split()
for lv_name in lv_names:
if not self.lv_info(lv_name, vg_name):
_print("INFO: lv_remove - LV %s does not exist. Skipping" % lv_name)
continue
cmd = "lvremove --force %s/%s" % (vg_name, lv_name)
retcode = run(cmd, verbose=verbose)
if (retcode != 0):
_print("WARN: Could not remove LV %s" % lv_name)
return False
if self.lv_info(lv_name, vg_name):
_print("INFO: lv_remove - LV %s still exists." % lv_name)
return False
return True
@staticmethod
def lv_convert(vg_name, lv_name, options, verbose=True):
"""Change Logical Volume layout.
The arguments are:
\tVG name
\tLV name
\toptions
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not options:
_print("WARN: lv_convert requires at least some options specified.")
return False
if not lv_name or not vg_name:
_print("WARN: lv_convert requires vg_name and lv_name")
return False
cmd = "lvconvert %s %s/%s" % (" ".join(options), vg_name, lv_name)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not convert %s" % lv_name)
return False
return True
###########################################
# Config file
###########################################
@staticmethod
def get_config_file_path():
return "/etc/lvm/lvm.conf"
def update_config(self, key, value):
config_file = self.get_config_file_path()
search_regex = re.compile("(\s*)%s(\s*)=(\s*)\S*" % key)
for line in fileinput.input(config_file, inplace=1):
m = search_regex.match(line)
if m:
line = "%s%s = %s" % (m.group(1), key, value)
# print saves the line to the file
# need to remove new line character as print will add it
line = line.rstrip('\n')
print(line)
class DMPD:
def __init__(self):
self.lvm = LVM()
def _get_devices(self):
lv_list = self.lvm.lv_query()
return lv_list
@staticmethod
def _get_active_devices():
cmd = "ls /dev/mapper/"
retcode, output = run(cmd, return_output=True, verbose=False)
if retcode != 0:
_print("WARN: Could not find active dm devices")
return False
devices = output.split()
return devices
@staticmethod
def _get_device_path(vg_name, lv_name):
device_path = vg_name + "-" + lv_name
if "/dev/mapper/" not in device_path:
device_path = "/dev/mapper/" + device_path
return device_path
def _check_device(self, vg_name, lv_name):
devices = self._get_devices()
device_list = [x["name"] for x in devices]
if lv_name not in device_list:
_print("WARN: %s is not a device" % lv_name)
return False
for x in devices:
if x["name"] == lv_name and x["vg_name"] == vg_name:
_print("INFO: Found device %s in group %s" % (lv_name, vg_name))
return True
return False
def _activate_device(self, vg_name, lv_name):
devices_active = self._get_active_devices()
if vg_name + "-" + lv_name not in devices_active:
ret = self.lvm.lv_activate(lv_name, vg_name)
if not ret:
_print("WARN: Could not activate device %s" % lv_name)
return False
_print("INFO: device %s was activated" % lv_name)
_print("INFO: device %s is active" % lv_name)
return True
@staticmethod
def _fallocate(_file, size, command_message):
cmd = "fallocate -l %sM %s" % (size, _file)
try:
retcode = run(cmd)
if retcode != 0:
_print("WARN: Command failed with code %s." % retcode)
_print("WARN: Could not create file to %s metadata to." % command_message)
return False
except OSError as e:
print >> sys.err, "command failed: ", e
return False
return True
@staticmethod
def get_help(cmd):
commands = ["cache_check", "cache_dump", "cache_metadata_size", "cache_repair", "cache_restore", "era_check",
"era_dump", "era_invalidate", "era_restore", "thin_check", "thin_delta", "thin_dump", "thin_ls",
"thin_metadata_size", "thin_repair", "thin_restore", "thin_rmap", "thin_show_duplicates",
"thin_trim"]
if cmd not in commands:
_print("WARN: Unknown command %s" % cmd)
return False
command = "%s -h" % cmd
retcode = run(command, verbose=True)
if retcode != 0:
_print("WARN: Could not get help for %s." % cmd)
return False
return True
@staticmethod
def get_version(cmd):
commands = ["cache_check", "cache_dump", "cache_metadata_size", "cache_repair", "cache_restore", "era_check",
"era_dump", "era_invalidate", "era_restore", "thin_check", "thin_delta", "thin_dump", "thin_ls",
"thin_metadata_size", "thin_repair", "thin_restore", "thin_rmap", "thin_show_duplicates",
"thin_trim"]
if cmd not in commands:
_print("WARN: Unknown command %s" % cmd)
return False
command = "%s -V" % cmd
retcode = run(command, verbose=True)
if retcode != 0:
_print("WARN: Could not get version of %s." % cmd)
return False
return True
def _get_dev_id(self, dev_id, path=None, lv_name=None, vg_name=None):
dev_ids = []
if path is None:
retcode, data = self.thin_dump(source_vg=vg_name, source_lv=lv_name, formatting="xml", return_output=True)
if not retcode:
_print("WARN: Could not dump metadata from %s/%s" % (vg_name, lv_name))
return False
data_lines = data.splitlines()
for line in data_lines:
blocks = line.split()
for block in blocks:
if not block.startswith("dev_"):
continue
else:
dev_ids.append(int(block[8:-1]))
else:
with open(path, "r") as meta:
for line in meta:
blocks = line.split()
for block in blocks:
if not block.startswith("dev_"):
continue
else:
dev_ids.append(int(block[8:-1]))
if dev_id in dev_ids:
return True
return False
@staticmethod
def _metadata_size(source=None, lv_name=None, vg_name=None):
if source is None:
cmd = "lvs -a --units m"
ret, data = run(cmd, return_output=True)
if ret != 0:
_print("WARN: Could not list LVs")
data_line = data.splitlines()
for line in data_line:
cut = line.split()
if not cut or lv_name != cut[0] and vg_name != cut[1]:
continue
cut = cut[3]
cut = cut.split("m")
size = float(cut[0])
cmd = "rm -f /tmp/meta_size"
run(cmd)
return int(size)
_print("WARN: Could not find %s %s in lvs, setting size to 100m" % (lv_name, vg_name))
return 100
else:
return int(os.stat(source).st_size) / 1000000
###########################################
# cache section
###########################################
def cache_check(self, source_file=None, source_vg=None, source_lv=None, quiet=False, super_block_only=False,
clear_needs_check_flag=False, skip_mappings=False, skip_hints=False, skip_discards=False,
verbose=True):
"""Check cache pool metadata from either file or device.
The arguments are:
\tsource_file
\tsource_vg VG name
\tsource_lv LV name
\tquiet Mute STDOUT
\tsuper_block_only
\tclear_needs_check_flag
\tskip_mappings
\tskip_hints
\tskip_discards
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if not source_file and (not source_vg or not source_lv):
_print("WARN: cache_check requires either source_file OR source_vg and source_lv.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
device = source_file
if quiet:
options += "--quiet "
if super_block_only:
options += "--super-block-only "
if clear_needs_check_flag:
options += "--clear-needs-check-flag "
if skip_mappings:
options += "--skip-mappings "
if skip_hints:
options += "--skip-hints "
if skip_discards:
options += "--skip-discards "
cmd = "cache_check %s %s" % (device, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not check %s metadata" % device)
return False
return True
def cache_dump(self, source_file=None, source_vg=None, source_lv=None, output=None, repair=False, verbose=True,
return_output=False):
"""Dumps cache metadata from device of source file to standard output or file.
The arguments are:
\tsource_file
\tsource_vg VG name
\tsource_lv LV name
\toutput specify output xml file
\treturn_output see 'Returns', not usable with output=True
\trepair Repair the metadata while dumping it
Returns:
\tOnly Boolean if return_output False:
\t\tTrue if success
\t'tFalse in case of failure
\tBoolean and data if return_output True
"""
options = ""
if return_output and output:
_print("INFO: Cannot return to both STDOUT and file, returning only to file.")
return_output = False
if return_output:
ret_fail = (False, None)
else:
ret_fail = False
if not source_file and (not source_vg or not source_lv):
_print("WARN: cache_dump requires either source_file OR source_vg and source_lv.")
return ret_fail
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return ret_fail
ret = self._activate_device(source_vg, source_lv)
if not ret:
return ret_fail
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return ret_fail
device = source_file
if output:
if not os.path.isfile(output):
size = self._metadata_size(source_file, source_lv, source_vg)
ret = self._fallocate(output, size + 1, "dump")
if not ret:
return ret_fail
options += "-o %s " % output
if repair:
options += "--repair"
cmd = "cache_dump %s %s" % (device, options)
if return_output:
retcode, data = run(cmd, return_output=True, verbose=verbose)
else:
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not dump %s metadata." % device)
return ret_fail
if return_output:
return True, data
return True
def cache_repair(self, source_file=None, source_vg=None, source_lv=None, target_file=None, target_vg=None,
target_lv=None, verbose=True):
"""Repairs cache metadata from source file/device to target file/device
The arguments are:
\tsource as either source_file OR source_vg and source_lv
\ttarget as either target_file OR target_vg and target_lv
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not source_file and (not source_vg or not source_lv):
_print("WARN: cache_repair requires either source_file OR source_vg and source_lv as source.")
return False
if not target_file and (not target_vg or not target_lv):
_print("WARN: cache_repair requires either target_file OR target_vg and target_lv as target.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
source = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
source = source_file
if not target_file:
ret = self._check_device(target_vg, target_lv)
if not ret:
return False
ret = self._activate_device(target_vg, target_lv)
if not ret:
return False
target = self._get_device_path(target_vg, target_lv)
else:
if not os.path.isfile(target_file):
size = self._metadata_size(source_file, source_lv, source_vg)
ret = self._fallocate(target_file, size + 1, "repair")
if not ret:
return False
target = target_file
cmd = "cache_repair -i %s -o %s" % (source, target)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not repair metadata from %s to %s" % (source, target))
return False
return True
def cache_restore(self, source_file, target_vg=None, target_lv=None, target_file=None, quiet=False,
metadata_version=None, omit_clean_shutdown=False, override_metadata_version=None, verbose=True):
"""Restores cache metadata from source xml file to target device/file
The arguments are:
\tsource_file Source xml file
\ttarget as either target_file OR target_vg and target_lv
\tquiet Mute STDOUT
\tmetadata_version Specify metadata version to restore
\tomit_clean_shutdown Disable clean shutdown
\toverride_metadata_version DEBUG option to override metadata version without checking
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if source_file is None:
_print("WARN: cache_restore requires source file.")
return False
if not target_file and (not target_vg or not target_lv):
_print("WARN: cache_restore requires either target_file OR target_vg and target_lv as target.")
return False
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
if not target_file:
ret = self._check_device(target_vg, target_lv)
if not ret:
return False
ret = self._activate_device(target_vg, target_lv)
if not ret:
return False
target = self._get_device_path(target_vg, target_lv)
else:
if not os.path.isfile(target_file):
size = self._metadata_size(source_file)
ret = self._fallocate(target_file, size + 1, "restore")
if not ret:
return False
target = target_file
if quiet:
options += "--quiet "
if metadata_version:
options += "--metadata-version %s " % metadata_version
if omit_clean_shutdown:
options += "--omit-clean-shutdown "
if override_metadata_version:
options += "--debug-override-metadata-version %s" % override_metadata_version
cmd = "cache_restore -i %s -o %s %s" % (source_file, target, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not restore metadata from %s to %s" % (source_file, target))
return False
return True
###########################################
# thinp section
###########################################
def thin_check(self, source_file=None, source_vg=None, source_lv=None, quiet=False, super_block_only=False,
clear_needs_check_flag=False, skip_mappings=False, ignore_non_fatal_errors=False, verbose=True):
"""Check thin pool metadata from either file or device.
The arguments are:
\tsource_file
\tsource_vg VG name
\tsource_lv LV name
\tquiet Mute STDOUT
\tsuper_block_only
\tclear_needs_check_flag
\tskip_mappings
\tignore_non_fatal_errors
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if not source_file and (not source_vg or not source_lv):
_print("WARN: thin_check requires either source_file OR source_vg and source_lv.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
device = source_file
if quiet:
options += "--quiet "
if super_block_only:
options += "--super-block-only "
if clear_needs_check_flag:
options += "--clear-needs-check-flag "
if skip_mappings:
options += "--skip-mappings "
if ignore_non_fatal_errors:
options += "--ignore-non-fatal-errors "
cmd = "thin_check %s %s" % (device, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not check %s metadata" % device)
return False
return True
def thin_ls(self, source_vg, source_lv, no_headers=False, fields=None, snapshot=False, verbose=True):
"""List information about thin LVs on thin pool.
The arguments are:
\tsource_vg VG name
\tsource_lv LV name
\tfields list of fields to output, default is all
\tsnapshot If use metadata snapshot, able to run on live snapshotted pool
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if not source_vg or not source_lv:
_print("WARN: thin_ls requires source_vg and source_lv.")
return False
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
device = self._get_device_path(source_vg, source_lv)
if no_headers:
options += "--no-headers "
fields_possible = ["DEV", "MAPPED_BLOCKS", "EXCLUSIVE_BLOCKS", "SHARED_BLOCKS", "MAPPED_SECTORS",
"EXCLUSIVE_SECTORS", "SHARED_SECTORS", "MAPPED_BYTES", "EXCLUSIVE_BYTES", "SHARED_BYTES",
"MAPPED", "EXCLUSIVE", "TRANSACTION", "CREATE_TIME", "SHARED", "SNAP_TIME"]
if fields is None:
options += " --format \"%s\" " % ",".join([str(i) for i in fields_possible])
else:
for field in fields:
if field not in fields_possible:
_print("WARN: Unknown field %s specified." % field)
_print("INFO: Possible fields are: %s" % ", ".join([str(i) for i in fields_possible]))
return False
options += " --format \"%s\" " % ",".join([str(i) for i in fields])
if snapshot:
options += "--metadata-snap"
cmd = "thin_ls %s %s" % (device, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not list %s metadata" % device)
return False
return True
def thin_dump(self, source_file=None, source_vg=None, source_lv=None, output=None, repair=False, formatting=None,
snapshot=None, dev_id=None, skip_mappings=False, verbose=True, return_output=False):
"""Dumps thin metadata from device of source file to standard output or file.
The arguments are:
\tsource_file
\tsource_vg VG name
\tsource_lv LV name
\toutput specify output xml file
\treturn_output see 'Returns', not usable with output=True
\trepair Repair the metadata while dumping it
\tformatting Specify output format [xml, human_readable, custom='file']
\tsnapshot (Boolean/Int) Use metadata snapshot. If Int provided, specifies block number
\tdev_id ID of the device
Returns:
\tOnly Boolean if return_output False:
\t\tTrue if success
\t'tFalse in case of failure
\tBoolean and data if return_output True
"""
options = ""
if return_output and output:
_print("INFO: Cannot return to both STDOUT and file, returning only to file.")
return_output = False
if return_output:
ret_fail = (False, None)
else:
ret_fail = False
if not source_file and (not source_vg or not source_lv):
_print("WARN: thin_dump requires either source_file OR source_vg and source_lv.")
return ret_fail
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return ret_fail
ret = self._activate_device(source_vg, source_lv)
if not ret:
return ret_fail
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return ret_fail
device = source_file
if output:
if not os.path.isfile(output):
size = self._metadata_size(source_file, source_lv, source_vg)
ret = self._fallocate(output, size + 1, "dump")
if not ret:
return ret_fail
options += "-o %s " % output
if repair:
options += "--repair "
if snapshot:
if isinstance(snapshot, bool):
options += "--metadata-snap "
elif isinstance(snapshot, int):
options += "--metadata-snap %s " % snapshot
else:
_print("WARN: Unknown snapshot value, use either Boolean or Int.")
return ret_fail
if formatting:
if formatting in ["xml", "human_readable"]:
options += "--format %s " % formatting
elif formatting.startswith("custom="):
if not os.path.isfile(formatting[8:-1]):
_print("WARN: Specified custom formatting file is not a file.")
return ret_fail
options += "--format %s " % formatting
else:
_print("WARN: Unknown formatting specified, please use one of [xml, human_readable, custom='file'].")
return ret_fail
if dev_id:
if isinstance(dev_id, int):
if self._get_dev_id(dev_id, source_file, source_lv, source_vg):
options += "--dev-id %s " % dev_id
else:
_print("WARN: Unknown dev_id value, device with ID %s does not exist." % dev_id)
return ret_fail
else:
_print("WARN: Unknown dev_id value, must be Int.")
return ret_fail
if skip_mappings:
options += "--skip-mappings "
cmd = "thin_dump %s %s" % (device, options)
if return_output:
retcode, data = run(cmd, return_output=True, verbose=verbose)
else:
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not dump %s metadata." % device)
return ret_fail
if return_output:
return True, data
return True
def thin_restore(self, source_file, target_vg=None, target_lv=None, target_file=None, quiet=False, verbose=True):
"""Restores thin metadata from source xml file to target device/file
The arguments are:
\tsource_file Source xml file
\ttarget as either target_file OR target_vg and target_lv
\tquiet Mute STDOUT
\tmetadata_version Specify metadata version to restore
\tomit_clean_shutdown Disable clean shutdown
\toverride_metadata_version DEBUG option to override metadata version without checking
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if source_file is None:
_print("WARN: thin_restore requires source file.")
return False
if not target_file and (not target_vg or not target_lv):
_print("WARN: thin_restore requires either target_file OR target_vg and target_lv as target.")
return False
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
if not target_file:
ret = self._check_device(target_vg, target_lv)
if not ret:
return False
ret = self._activate_device(target_vg, target_lv)
if not ret:
return False
target = self._get_device_path(target_vg, target_lv)
else:
if not os.path.isfile(target_file):
size = self._metadata_size(source_file)
ret = self._fallocate(target_file, size + 1, "restore")
if not ret:
return False
target = target_file
if quiet:
options += "--quiet"
cmd = "thin_restore -i %s -o %s %s" % (source_file, target, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not restore metadata from %s to %s" % (source_file, target))
return False
return True
def thin_repair(self, source_file=None, source_vg=None, source_lv=None, target_file=None, target_vg=None,
target_lv=None, verbose=True):
"""Repairs thin metadata from source file/device to target file/device
The arguments are:
\tsource as either source_file OR source_vg and source_lv
\ttarget as either target_file OR target_vg and target_lv
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not source_file and (not source_vg or not source_lv):
_print("WARN: thin_repair requires either source_file OR source_vg and source_lv as source.")
return False
if not target_file and (not target_vg or not target_lv):
_print("WARN: thin_repair requires either target_file OR target_vg and target_lv as target.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
source = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
source = source_file
if not target_file:
ret = self._check_device(target_vg, target_lv)
if not ret:
return False
ret = self._activate_device(target_vg, target_lv)
if not ret:
return False
target = self._get_device_path(target_vg, target_lv)
else:
if not os.path.isfile(target_file):
size = self._metadata_size(source_file, source_lv, source_vg)
ret = self._fallocate(target_file, size + 1, "repair")
if not ret:
return False
target = target_file
cmd = "thin_repair -i %s -o %s" % (source, target)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not repair metadata from %s to %s" % (source, target))
return False
return True
def thin_rmap(self, region, source_file=None, source_vg=None, source_lv=None, verbose=True):
"""Output reverse map of a thin provisioned region of blocks from metadata device.
The arguments are:
\tsource_vg VG name
\tsource_lv LV name
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
if not source_file and (not source_vg or not source_lv):
_print("WARN: thin_rmap requires either source_file OR source_vg and source_lv as source.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
device = source_file
regions = region.split(".")
try:
int(regions[0])
if regions[1] != '':
raise ValueError
int(regions[2])
if regions[3] is not None:
raise ValueError
except ValueError:
_print("WARN: Region must be in format 'INT..INT'")
return False
except IndexError:
pass
# region 1..-1 must be valid, using usigned 32bit ints
if int(regions[0]) & 0xffffffff >= int(regions[2]) & 0xffffffff:
_print("WARN: Beginning of the region must be before its end.")
return False
options = "--region %s" % region
cmd = "thin_rmap %s %s" % (device, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not output reverse map from %s metadata device" % device)
return False
return True
def thin_trim(self, target_vg, target_lv, force=True, verbose=True):
"""Issue discard requests for free pool space.
The arguments are:
\ttarget_vg VG name
\ttarget_lv LV name
\tforce suppress warning message and disable prompt, default True
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if force:
options += " --pool-inactive"
if not target_vg or not target_lv:
_print("WARN: thin_trim requires target_vg and target_lv.")
return False
ret = self._check_device(target_vg, target_lv)
if not ret:
return False
ret = self._activate_device(target_vg, target_lv)
if not ret:
return False
device = self._get_device_path(target_vg, target_lv)
cmd = "thin_trim %s %s" % (device, options)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not discard free pool space on device %s." % device)
return False
return True
def thin_delta(self, thin1, thin2, source_file=None, source_vg=None, source_lv=None, snapshot=False,
verbosity=False, verbose=True):
"""Print the differences in the mappings between two thin devices..
The arguments are:
\tsource_vg VG name
\tsource_lv LV name
\tthin1 numeric identificator of first thin volume
\tthin2 numeric identificator of second thin volume
\tsnapshot (Boolean/Int) Use metadata snapshot. If Int provided, specifies block number
\tverbosity Provide extra information on the mappings
Returns:
\tBoolean:
\t\tTrue if success
\t'tFalse in case of failure
"""
options = ""
if not source_file and (not source_vg or not source_lv):
_print("WARN: thin_delta requires either source_file OR source_vg and source_lv.")
return False
if not source_file:
ret = self._check_device(source_vg, source_lv)
if not ret:
return False
ret = self._activate_device(source_vg, source_lv)
if not ret:
return False
device = self._get_device_path(source_vg, source_lv)
else:
if not os.path.isfile(source_file):
_print("WARN: Source file is not a file.")
return False
device = source_file
if snapshot:
if isinstance(snapshot, bool):
options += "--metadata-snap "
elif isinstance(snapshot, int):
options += "--metadata-snap %s " % snapshot
else:
_print("WARN: Unknown snapshot value, use either Boolean or Int.")
return False
if verbosity:
options += "--verbose"
if self._get_dev_id(thin1, source_file, source_lv, source_vg) and \
self._get_dev_id(thin2, source_file, source_lv, source_vg):
cmd = "thin_delta %s --thin1 %s --thin2 %s %s" % (options, thin1, thin2, device)
retcode = run(cmd, verbose=verbose)
if retcode != 0:
_print("WARN: Could not get differences in mappings between two thin LVs.")
return False
else:
_print("WARN: Specified ID does not exist.")
return False
return True