| #!/usr/bin/python -tt |
| # |
| # rteval - script for evaluating platform suitability for RT Linux |
| # |
| # This program is used to determine the suitability of |
| # a system for use in a Real Time Linux environment. |
| # It starts up various system loads and measures event |
| # latency while the loads are running. A report is generated |
| # to show the latencies encountered during the run. |
| # |
| # Copyright 2009,2010,2011,2012 Clark Williams <williams@redhat.com> |
| # Copyright 2009,2010,2011,2012 David Sommerseth <davids@redhat.com> |
| # |
| # This program is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License as published by |
| # the Free Software Foundation; either version 2 of the License, or |
| # (at your option) any later version. |
| # |
| # This program is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| # GNU General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License |
| # along with this program; if not, write to the Free Software |
| # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| # |
| # For the avoidance of doubt the "preferred form" of this code is one which |
| # is in an open unpatent encumbered format. Where cryptographic key signing |
| # forms part of the process of creating an executable the information |
| # including keys needed to generate an equivalently functional executable |
| # are deemed to be part of the source code. |
| # |
| |
| import sys |
| import os |
| import os.path |
| import time |
| import string |
| import threading |
| import subprocess |
| import socket |
| import optparse |
| import tempfile |
| import statvfs |
| import shutil |
| import signal |
| import rtevalclient |
| import ethtool |
| import xmlrpclib |
| import platform |
| import fnmatch |
| import glob |
| from datetime import datetime |
| from distutils import sysconfig |
| |
| # put local path at start of list to overide installed methods |
| sys.path.insert(0, "./rteval") |
| import util |
| import load |
| import cyclictest |
| import xmlout |
| import dmi |
| import rtevalConfig |
| import rtevalMailer |
| from cputopology import CPUtopology |
| |
| |
| pathSave={} |
| def getcmdpath(which): |
| """ |
| getcmdpath is a method which allows finding an executable in the PATH |
| directories to call it from full path |
| """ |
| if not pathSave.has_key(which): |
| for path in os.environ['PATH'].split(':'): |
| cmdfile = os.path.join(path, which) |
| if os.path.isfile(cmdfile) and os.access(cmdfile, os.X_OK): |
| pathSave[which] = cmdfile |
| break |
| if not pathSave[which]: |
| raise RuntimeError, "Command '%s' is unknown on this system" % which |
| return pathSave[which] |
| |
| |
| sigint_received = False |
| def sigint_handler(signum, frame): |
| global sigint_received |
| sigint_received = True |
| print "*** SIGINT received - stopping rteval run ***" |
| |
| def sigterm_handler(signum, frame): |
| raise RuntimeError, "SIGTERM received!" |
| |
| class RtEval(object): |
| def __init__(self, cmdargs): |
| self.version = "1.41" |
| self.load_modules = [] |
| self.workdir = os.getcwd() |
| self.reportdir = os.getcwd() |
| self.inifile = None |
| self.cmd_options = {} |
| self.start = datetime.now() |
| self.init = 'unknown' |
| |
| default_config = { |
| 'rteval': { |
| 'verbose' : False, |
| 'keepdata' : True, |
| 'debugging' : False, |
| 'duration' : '60', |
| 'sysreport' : False, |
| 'reportdir' : None, |
| 'reportfile' : None, |
| 'installdir' : '/usr/share/rteval', |
| 'srcdir' : '/usr/share/rteval/loadsource', |
| 'xmlrpc' : None, |
| 'xslt_report': '/usr/share/rteval/rteval_text.xsl', |
| 'report_interval': '600', |
| 'logging' : False, |
| }, |
| 'loads' : { |
| 'kcompile' : 'module', |
| 'hackbench' : 'module', |
| }, |
| 'kcompile' : { |
| 'source' : 'linux-2.6.39.tar.bz2', |
| 'jobspercore': '2', |
| }, |
| 'hackbench' : { |
| 'source' : 'hackbench.tar.bz2', |
| 'jobspercore': '5', |
| }, |
| 'cyclictest' : { |
| 'interval' : '100', |
| 'buckets' : '2000', |
| } |
| } |
| |
| # setup initial configuration |
| self.config = rtevalConfig.rtevalConfig(default_config, logfunc=self.info) |
| |
| # parse command line options |
| self.parse_options(cmdargs) |
| |
| # read in config file info |
| self.inifile = self.config.Load(self.cmd_options.inifile) |
| |
| # copy the command line options into the rteval config section |
| # (cmd line overrides config file values) |
| self.config.AppendConfig('rteval', self.cmd_options) |
| |
| if self.cmd_options.cyclictest_interval != None: |
| self.config.AppendConfig('cyclictest', { "interval":self.cmd_options.cyclictest_interval }) |
| |
| if self.cmd_options.cyclictest_distance != None: |
| self.config.AppendConfig('cyclictest', { "distance":self.cmd_options.cyclictest_distance }) |
| |
| if self.cmd_options.cyclictest_buckets != None: |
| self.config.AppendConfig('cyclictest', { "buckets":self.cmd_options.cyclictest_distance }) |
| |
| if self.cmd_options.cyclictest_priority != None: |
| self.config.AppendConfig('cyclictest', { "priority":self.cmd_options.cyclictest_priority }) |
| |
| if self.cmd_options.hackbench_jobspercore != None: |
| self.config.AppendConfig('hackbench', { "jobspercore":self.cmd_options.hackbench_jobspercore }) |
| |
| if self.cmd_options.kcompile_jobspercore != None: |
| self.config.AppendConfig('kcompile', { "jobspercore":self.cmd_options.kcompile_jobspercore }) |
| |
| self.debug("workdir: %s" % self.workdir) |
| |
| # prepare a mailer, if that's configured |
| if self.config.HasSection('smtp'): |
| self.mailer = rtevalMailer.rtevalMailer(self.config.GetSection('smtp')) |
| else: |
| self.mailer = None |
| |
| self.loads = [] |
| self.cputopology = None |
| self.numcores = None |
| self.memsize = None |
| self.current_clocksource = None |
| self.available_clocksource = None |
| self.services = None |
| self.kthreads = None |
| self.xml = None |
| self.baseos = "unknown" |
| self.annotate = self.cmd_options.annotate |
| |
| if not self.config.xslt_report.startswith(self.config.installdir): |
| self.config.xslt_report = os.path.join(self.config.installdir, "rteval_text.xsl") |
| |
| if not os.path.exists(self.config.xslt_report): |
| raise RuntimeError, "can't find XSL template (%s)!" % self.config.xslt_report |
| |
| # Add rteval directory into module search path |
| sys.path.insert(0, '%s/rteval' % sysconfig.get_python_lib()) |
| |
| # generate a set of "junk" characters to use for filtering later |
| self.junk = "" |
| for c in range(0, 0xff): |
| s = chr(c) |
| if s not in string.printable: |
| self.junk += s |
| self.transtable = string.maketrans("", "") |
| |
| # If --xmlrpc-submit is given, check that we can access the server |
| res = None |
| if self.config.xmlrpc: |
| self.debug("Checking if XML-RPC server '%s' is reachable" % self.config.xmlrpc) |
| attempt = 0 |
| warning_sent = False |
| ping_failed = False |
| while attempt < 6: |
| try: |
| client = rtevalclient.rtevalclient("http://%s/rteval/API1/" % self.config.xmlrpc) |
| res = client.Hello() |
| attempt = 10 |
| ping_failed = False |
| except xmlrpclib.ProtocolError: |
| # Server do not support Hello(), but is reachable |
| self.info("Got XML-RPC connection with %s but it did not support Hello()" |
| % self.config.xmlrpc) |
| res = None |
| except socket.error, err: |
| self.info("Could not establish XML-RPC contact with %s\n%s" |
| % (self.config.xmlrpc, str(err))) |
| |
| if (self.mailer is not None) and (not warning_sent): |
| self.mailer.SendMessage("[RTEVAL:WARNING] Failed to ping XML-RPC server", |
| "Server %s did not respond. Not giving up yet." |
| % self.config.xmlrpc) |
| warning_sent = True |
| |
| # Do attempts handling |
| attempt += 1 |
| if attempt > 5: |
| break # To avoid sleeping before we abort |
| |
| print "Failed pinging XML-RPC server. Doing another attempt(%i) " % attempt |
| time.sleep(attempt*15) # Incremental sleep - sleep attempts*15 seconds |
| ping_failed = True |
| |
| if ping_failed: |
| if not self.cmd_options.xmlrpc_noabort: |
| print "ERROR: Could not reach XML-RPC server '%s'. Aborting." % self.config.xmlrpc |
| sys.exit(2) |
| else: |
| print "WARNING: Could not ping the XML-RPC server. Will continue anyway." |
| |
| if res: |
| self.info("Verified XML-RPC connection with %s (XML-RPC API version: %i)" |
| % (res["server"], res["APIversion"])) |
| self.debug("Recieved greeting: %s" % res["greeting"]) |
| |
| |
| def get_cpu_topology(self): |
| ''' figure out how many processors we have available''' |
| |
| topology = CPUtopology() |
| topology.parse() |
| |
| self.numcores = topology.getCPUcores(True) |
| self.debug("counted %d cores (%d online) and %d sockets" % |
| (topology.getCPUcores(False), self.numcores, |
| topology.getCPUsockets())) |
| return topology.getXMLdata() |
| |
| def __get_services_sysvinit(self): |
| reject = ('functions', 'halt', 'killall', 'single', 'linuxconf', 'kudzu', |
| 'skeleton', 'README', '*.dpkg-dist', '*.dpkg-old', 'rc', 'rcS', |
| 'single', 'reboot', 'bootclean.sh') |
| for sdir in ('/etc/init.d', '/etc/rc.d/init.d'): |
| if os.path.isdir(sdir): |
| servicesdir = sdir |
| break |
| if not servicesdir: |
| raise RuntimeError, "No services dir (init.d) found on your system" |
| self.debug("Services located in %s, going through each service file to check status" % servicesdir) |
| ret_services = {} |
| for service in glob.glob(os.path.join(servicesdir, '*')): |
| servicename = os.path.basename(service) |
| if not [1 for p in reject if fnmatch.fnmatch(servicename, p)] and os.access(service, os.X_OK): |
| cmd = '%s -qs "\(^\|\W\)status)" %s' % (getcmdpath('grep'), service) |
| c = subprocess.Popen(cmd, shell=True) |
| c.wait() |
| if c.returncode == 0: |
| cmd = ['env', '-i', 'LANG="%s"' % os.environ['LANG'], 'PATH="%s"' % os.environ['PATH'], 'TERM="%s"' % os.environ['TERM'], service, 'status'] |
| c = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| c.wait() |
| if c.returncode == 0 and (c.stdout.read() or c.stderr.read()): |
| ret_services[servicename] = 'running' |
| else: |
| ret_services[servicename] = 'not running' |
| else: |
| ret_services[servicename] = 'unknown' |
| return ret_services |
| |
| def __get_services_systemd(self): |
| ret_services = {} |
| cmd = '%s list-unit-files -t service --no-legend' % getcmdpath('systemctl') |
| self.debug("cmd: %s" % cmd) |
| c = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| for p in c.stdout: |
| # p are lines like "servicename.service status" |
| v = p.strip().split() |
| ret_services[v[0].split('.')[0]] = v[1] |
| return ret_services |
| |
| def get_services(self): |
| cmd = [getcmdpath('ps'), '-ocomm=', '1'] |
| c = subprocess.Popen(cmd, stdout=subprocess.PIPE) |
| self.init = c.stdout.read().strip() |
| if self.init == 'systemd': |
| self.debug("Using systemd to get services status") |
| return self.__get_services_systemd() |
| elif self.init == 'init': |
| self.init = 'sysvinit' |
| self.debug("Using sysvinit to get services status") |
| return self.__get_services_sysvinit() |
| else: |
| raise RuntimeError, "Unknown init system (%s)" % self.init |
| return {} |
| |
| def get_kthreads(self): |
| policies = {'FF':'fifo', 'RR':'rrobin', 'TS':'other', '?':'unknown' } |
| ret_kthreads = {} |
| self.debug("getting kthread status") |
| cmd = '%s -eocommand,pid,policy,rtprio,comm' % getcmdpath('ps') |
| self.debug("cmd: %s" % cmd) |
| c = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) |
| for p in c.stdout: |
| v = p.strip().split() |
| kcmd = v.pop(0) |
| try: |
| if int(v[0]) > 0 and kcmd.startswith('[') and kcmd.endswith(']'): |
| ret_kthreads[v[0]] = {'policy' : policies[v[1]], |
| 'priority' : v[2], 'name' : v[3] } |
| except ValueError: |
| pass # Ignore lines which don't have a number in the first row |
| return ret_kthreads |
| |
| def get_modules(self): |
| modlist = [] |
| try: |
| fp = open('/proc/modules', 'r') |
| line = fp.readline() |
| while line: |
| mod = line.split() |
| modlist.append({"modname": mod[0], |
| "modsize": mod[1], |
| "numusers": mod[2], |
| "usedby": mod[3], |
| "modstate": mod[4]}) |
| line = fp.readline() |
| fp.close() |
| except Exception, err: |
| raise err |
| return modlist |
| |
| def parse_options(self, cmdargs): |
| '''parse the command line arguments''' |
| parser = optparse.OptionParser() |
| parser.add_option("-d", "--duration", dest="duration", |
| type="string", default=self.config.duration, |
| help="specify length of test run (default: %default)") |
| parser.add_option("-v", "--verbose", dest="verbose", |
| action="store_true", default=self.config.verbose, |
| help="turn on verbose prints (default: %default)") |
| parser.add_option("-w", "--workdir", dest="workdir", |
| type="string", default=self.workdir, |
| help="top directory for rteval data (default: %default)") |
| parser.add_option("-l", "--loaddir", dest="srcdir", |
| type="string", default=self.config.srcdir, |
| help="directory for load source tarballs (default: %default)") |
| parser.add_option("-i", "--installdir", dest="installdir", |
| type="string", default=self.config.installdir, |
| help="place to locate installed templates (default: %default)") |
| parser.add_option("-s", "--sysreport", dest="sysreport", |
| action="store_true", default=self.config.sysreport, |
| help='run sysreport to collect system data (default: %default)') |
| parser.add_option("-D", '--debug', dest='debugging', |
| action='store_true', default=self.config.debugging, |
| help='turn on debug prints (default: %default)') |
| parser.add_option("-X", '--xmlrpc-submit', dest='xmlrpc', |
| action='store', default=self.config.xmlrpc, metavar='HOST', |
| help='Hostname to XML-RPC server to submit reports') |
| parser.add_option("-P", "--xmlrpc-no-abort", dest="xmlrpc_noabort", |
| action='store_true', default=False, |
| help="Do not abort if XML-RPC server do not respond to ping request"); |
| parser.add_option("-Z", '--summarize', dest='summarize', |
| action='store_true', default=False, |
| help='summarize an already existing XML report') |
| parser.add_option("-H", '--raw-histogram', dest='rawhistogram', |
| action='store_true', default=False, |
| help='Generate raw histogram data for an already existing XML report') |
| parser.add_option("-f", "--inifile", dest="inifile", |
| type='string', default=None, |
| help="initialization file for configuring loads and behavior") |
| parser.add_option("-a", "--annotate", dest="annotate", |
| type="string", default=None, |
| help="Add a little annotation which is stored in the report") |
| parser.add_option("-L", "--logging", dest="logging", |
| action='store_true', default=False, |
| help='log the output of the loads in the report directory') |
| parser.add_option("-O", "--onlyload", dest="onlyload", |
| action='store_true', default=False, |
| help="only run the loads (don't run measurement threads)") |
| |
| # module options |
| parser.add_option("", "--cyclictest-interval", dest="cyclictest_interval", |
| action="store", type="int", |
| help="cyclictest measurement interval in microseconds") |
| parser.add_option("", "--cyclictest-distance", dest="cyclictest_distance", |
| action="store", type="int", |
| help="cyclictest measurement interval increment in microseconds") |
| parser.add_option("", "--cyclictest-buckets", dest="cyclictest_buckets", |
| action="store", type="int", |
| help="number of cyclictest 1 microsecond histogram buckets") |
| parser.add_option("", "--cyclictest-priority", dest="cyclictest_priority", |
| action="store", type="int", |
| help="SCHED_FIFO priority of measurement threads") |
| |
| parser.add_option("", "--hackbench-jobspercore", dest="hackbench_jobspercore", |
| action="store", type="int", |
| help="number of hackbench jobs per-core") |
| parser.add_option("", "--kcompile-jobspercore", dest="kcompile_jobspercore", |
| action="store", type="int", |
| help="number of kernel compile jobs per-core") |
| |
| |
| (self.cmd_options, self.cmd_arguments) = parser.parse_args(args = cmdargs) |
| if self.cmd_options.duration: |
| mult = 1.0 |
| v = self.cmd_options.duration.lower() |
| if v.endswith('s'): |
| v = v[:-1] |
| elif v.endswith('m'): |
| v = v[:-1] |
| mult = 60.0 |
| elif v.endswith('h'): |
| v = v[:-1] |
| mult = 3600.0 |
| elif v.endswith('d'): |
| v = v[:-1] |
| mult = 3600.0 * 24.0 |
| self.cmd_options.duration = float(v) * mult |
| self.workdir = os.path.abspath(self.cmd_options.workdir) |
| |
| |
| def debug(self, str): |
| if self.config.debugging is True: |
| print "rteval: %s" % str |
| |
| def info(self, str): |
| if self.config.verbose is True: |
| print str |
| |
| def run_sysreport(self): |
| import glob |
| if os.path.exists('/usr/sbin/sosreport'): |
| exe = '/usr/sbin/sosreport' |
| elif os.path.exists('/usr/sbin/sysreport'): |
| exe = '/usr/sbin/sysreport' |
| else: |
| raise RuntimeError, "Can't find sosreport/sysreport" |
| |
| self.debug("report tool: %s" % exe) |
| options = ['-k', 'rpm.rpmva=off', |
| '--name=rteval', |
| '--batch', |
| '--no-progressbar'] |
| |
| self.info("Generating SOS report") |
| self.info("using command %s" % " ".join([exe]+options)) |
| subprocess.call([exe] + options) |
| for s in glob.glob('/tmp/s?sreport-rteval-*'): |
| self.debug("moving %s to %s" % (s, self.reportdir)) |
| shutil.move(s, self.reportdir) |
| |
| |
| def genxml(self, duration, accum, samples, xslt = None): |
| seconds = duration.seconds |
| hours = seconds / 3600 |
| if hours: seconds -= (hours * 3600) |
| minutes = seconds / 60 |
| if minutes: seconds -= (minutes * 60) |
| (sys, node, release, ver, machine) = os.uname() |
| |
| # Start new XML report |
| self.xmlreport = xmlout.XMLOut('rteval', self.version) |
| self.xmlreport.NewReport() |
| |
| self.xmlreport.openblock('run_info', {'days': duration.days, |
| 'hours': hours, |
| 'minutes': minutes, |
| 'seconds': seconds}) |
| self.xmlreport.taggedvalue('date', self.start.strftime('%Y-%m-%d')) |
| self.xmlreport.taggedvalue('time', self.start.strftime('%H:%M:%S')) |
| if self.annotate: |
| self.xmlreport.taggedvalue('annotate', self.annotate) |
| self.xmlreport.closeblock() |
| self.xmlreport.openblock('uname') |
| self.xmlreport.taggedvalue('node', node) |
| isrt = 1 |
| if ver.find(' RT ') == -1: |
| isrt = 0 |
| self.xmlreport.taggedvalue('kernel', release, {'is_RT':isrt}) |
| self.xmlreport.taggedvalue('arch', machine) |
| self.xmlreport.taggedvalue('baseos', self.baseos) |
| self.xmlreport.closeblock() |
| |
| self.xmlreport.openblock("clocksource") |
| self.xmlreport.taggedvalue('current', self.current_clocksource) |
| self.xmlreport.taggedvalue('available', self.available_clocksource) |
| self.xmlreport.closeblock() |
| |
| self.xmlreport.openblock('hardware') |
| self.xmlreport.AppendXMLnodes(self.cputopology) |
| self.xmlreport.taggedvalue('numa_nodes', self.numanodes) |
| self.xmlreport.taggedvalue('memory_size', "%.3f" % self.memsize[0], {"unit": self.memsize[1]}) |
| self.xmlreport.closeblock() |
| |
| self.xmlreport.openblock('services', {'init': self.init}) |
| for s in self.services: |
| self.xmlreport.taggedvalue("service", self.services[s], {"name": s}) |
| self.xmlreport.closeblock() |
| |
| keys = self.kthreads.keys() |
| if len(keys): |
| keys.sort() |
| self.xmlreport.openblock('kthreads') |
| for pid in keys: |
| self.xmlreport.taggedvalue('thread', self.kthreads[pid]['name'], |
| { 'policy' : self.kthreads[pid]['policy'], |
| 'priority' : self.kthreads[pid]['priority'], |
| }) |
| self.xmlreport.closeblock() |
| |
| modlist = util.get_modules() |
| if len(modlist): |
| self.xmlreport.openblock('kernelmodules') |
| for mod in modlist: |
| self.xmlreport.openblock('module') |
| self.xmlreport.taggedvalue('info', mod['modname'], |
| {'size': mod['modsize'], |
| 'state': mod['modstate'], |
| 'numusers': mod['numusers']}) |
| if mod['usedby'] != '-': |
| self.xmlreport.openblock('usedby') |
| for ub in mod['usedby'].split(','): |
| if len(ub): |
| self.xmlreport.taggedvalue('module', ub, None) |
| self.xmlreport.closeblock() |
| self.xmlreport.closeblock() |
| self.xmlreport.closeblock() |
| |
| # |
| # Retrieve configured IP addresses |
| # |
| self.xmlreport.openblock('network_config') |
| |
| # Get the interface name for the IPv4 default gw |
| route = open('/proc/net/route') |
| defgw4 = None |
| if route: |
| rl = route.readline() |
| while rl != '' : |
| rl = route.readline() |
| splt = rl.split("\t") |
| # Only catch default route |
| if len(splt) > 2 and splt[2] != '00000000' and splt[1] == '00000000': |
| defgw4 = splt[0] |
| break |
| route.close() |
| |
| # Make an interface tag for each device found |
| if hasattr(ethtool, 'get_interfaces_info'): |
| # Using the newer python-ethtool API (version >= 0.4) |
| for dev in ethtool.get_interfaces_info(ethtool.get_devices()): |
| if cmp(dev.device,'lo') == 0: |
| continue |
| |
| self.xmlreport.openblock('interface', |
| {'device': dev.device, |
| 'hwaddr': dev.mac_address} |
| ) |
| |
| # Protcol configurations |
| if dev.ipv4_address: |
| self.xmlreport.openblock('IPv4', |
| {'ipaddr': dev.ipv4_address, |
| 'netmask': dev.ipv4_netmask, |
| 'broadcast': dev.ipv4_broadcast, |
| 'defaultgw': (defgw4 == dev.device) and '1' or '0'} |
| ) |
| self.xmlreport.closeblock() |
| |
| for ip6 in dev.get_ipv6_addresses(): |
| self.xmlreport.openblock('IPv6', |
| {'ipaddr': ip6.address, |
| 'netmask': ip6.netmask, |
| 'scope': ip6.scope} |
| ) |
| self.xmlreport.closeblock() |
| self.xmlreport.closeblock() |
| else: # Fall back to older python-ethtool API (version < 0.4) |
| ifdevs = ethtool.get_active_devices() |
| ifdevs.remove('lo') |
| ifdevs.sort() |
| |
| for dev in ifdevs: |
| self.xmlreport.openblock('interface', |
| {'device': dev, |
| 'hwaddr': ethtool.get_hwaddr(dev)} |
| ) |
| self.xmlreport.openblock('IPv4', |
| {'ipaddr': ethtool.get_ipaddr(dev), |
| 'netmask': ethtool.get_netmask(dev), |
| 'defaultgw': (defgw4 == dev) and '1' or '0'} |
| ) |
| self.xmlreport.closeblock() |
| self.xmlreport.closeblock() |
| self.xmlreport.closeblock() |
| |
| self.xmlreport.openblock('loads', {'load_average':str(accum / samples)}) |
| for load in self.loads: |
| load.genxml(self.xmlreport) |
| self.xmlreport.closeblock() |
| self.cyclictest.genxml(self.xmlreport) |
| |
| # now generate the dmidecode data for this host |
| d = dmi.DMIinfo(self.config.GetSection('rteval')) |
| d.genxml(self.xmlreport) |
| |
| # Close the report - prepare for return the result |
| self.xmlreport.close() |
| |
| # Write XML (or write XSLT parsed XML if xslt != None) |
| if self.xml != None: |
| self.xmlreport.Write(self.xml, xslt) |
| else: |
| # If no file is set, use stdout |
| self.xmlreport.Write("-", xslt) # libxml2 defines a filename as "-" to be stdout |
| |
| |
| def report(self): |
| "Create a screen report, based on a predefined XSLT template" |
| self.xmlreport.Write("-", self.config.xslt_report) |
| |
| def XMLreport(self): |
| "Retrieves the complete rteval XML report as a libxml2.xmlDoc object" |
| return self.xmlreport.GetXMLdocument() |
| |
| def show_report(self, xmlfile, xsltfile): |
| '''summarize a previously generated xml file''' |
| print "Loading %s for summarizing" % xmlfile |
| |
| xsltfullpath = os.path.join(self.config.installdir, xsltfile) |
| if not os.path.exists(xsltfullpath): |
| raise RuntimeError, "can't find XSL template (%s)!" % xsltfullpath |
| |
| xmlreport = xmlout.XMLOut('rteval', self.version) |
| xmlreport.LoadReport(xmlfile) |
| xmlreport.Write('-', xsltfullpath) |
| del xmlreport |
| |
| def start_loads(self): |
| if len(self.loads) == 0: |
| raise RuntimeError, "start_loads: No loads defined!" |
| self.info ("starting loads:") |
| for l in self.loads: |
| l.start() |
| # now wait until they're all ready |
| self.info("waiting for ready from all loads") |
| ready=False |
| while not ready: |
| busy = 0 |
| for l in self.loads: |
| if not l.isAlive(): |
| raise RuntimeError, "%s died" % l.name |
| if not l.isReady(): |
| busy += 1 |
| self.debug("waiting for %s" % l.name) |
| if busy: |
| time.sleep(1.0) |
| else: |
| ready = True |
| |
| def stop_loads(self): |
| if len(self.loads) == 0: |
| raise RuntimeError, "stop_loads: No loads defined!" |
| self.info("stopping loads: ") |
| for l in self.loads: |
| self.info("\t%s" % l.name) |
| l.stopevent.set() |
| l.join(2.0) |
| |
| def make_report_dir(self): |
| t = self.start |
| i = 1 |
| self.reportdir = os.path.join(self.workdir, |
| t.strftime("rteval-%Y%m%d-"+str(i))) |
| while os.path.exists(self.reportdir): |
| i += 1 |
| self.reportdir = os.path.join(self.workdir, |
| t.strftime('rteval-%Y%m%d-'+str(i))) |
| if not os.path.isdir(self.reportdir): |
| os.mkdir(self.reportdir) |
| os.mkdir(os.path.join(self.reportdir, "logs")) |
| return self.reportdir |
| |
| def get_dmesg(self): |
| dpath = "/var/log/dmesg" |
| if not os.path.exists(dpath): |
| print "dmesg file not found at %s" % dpath |
| return |
| shutil.copyfile(dpath, os.path.join(self.reportdir, "dmesg")) |
| |
| |
| def show_remaining_time(self, remaining): |
| r = int(remaining) |
| days = r / 86400 |
| if days: r = r - (days * 86400) |
| hours = r / 3600 |
| if hours: r = r - (hours * 3600) |
| minutes = r / 60 |
| if minutes: r = r - (minutes * 60) |
| print "rteval time remaining: %d days, %d hours, %d minutes, %d seconds" % (days, hours, minutes, r) |
| |
| |
| def measure(self): |
| # Collect misc system info |
| self.baseos = util.get_base_os() |
| self.cputopology = self.get_cpu_topology() |
| self.numanodes = util.get_num_nodes() |
| self.memsize = util.get_memory_size() |
| (self.current_clocksource, self.available_clocksource) = util.get_clocksources() |
| self.services = self.get_services() |
| self.kthreads = self.get_kthreads() |
| |
| onlyload = self.cmd_options.onlyload |
| |
| builddir = os.path.join(self.workdir, 'rteval-build') |
| if not os.path.isdir(builddir): os.mkdir(builddir) |
| self.reportfile = os.path.join(self.reportdir, "summary.rpt") |
| self.xml = os.path.join(self.reportdir, "summary.xml") |
| |
| # read in loads from the ini file |
| self.load_modules = [] |
| loads = self.config.GetSection("loads") |
| for l in loads: |
| # hope to eventually have different kinds but module is only on |
| # for now (jcw) |
| if l[1].lower() == 'module': |
| self.info("importing load module %s" % l[0]) |
| self.load_modules.append(__import__(l[0])) |
| |
| self.info("setting up loads") |
| self.loads = [] |
| params = {'workdir':self.workdir, |
| 'reportdir':self.reportdir, |
| 'builddir':builddir, |
| 'srcdir':self.config.srcdir, |
| 'verbose': self.config.verbose, |
| 'debugging': self.config.debugging, |
| 'numcores':self.numcores, |
| 'logging':self.config.logging, |
| 'memsize':self.memsize, |
| 'numanodes':self.numanodes, |
| 'duration':self.config.duration, |
| } |
| |
| for m in self.load_modules: |
| self.config.AppendConfig(m.__name__, params) |
| self.info("creating load instance for %s" % m.__name__) |
| self.loads.append(m.create(self.config.GetSection(m.__name__))) |
| |
| if not onlyload: |
| self.config.AppendConfig('cyclictest', params) |
| self.info("setting up cyclictest") |
| self.cyclictest = cyclictest.Cyclictest(params=self.config.GetSection('cyclictest')) |
| |
| nthreads = 0 |
| try: |
| # start the loads |
| self.start_loads() |
| |
| print "rteval run on %s started at %s" % (os.uname()[2], time.asctime()) |
| print "started %d loads on %d cores" % (len(self.loads), self.numcores), |
| if self.numanodes > 1: |
| print " with %d numa nodes" % self.numanodes |
| else: |
| print "" |
| print "Run duration: %d seconds" % self.config.duration |
| |
| start = datetime.now() |
| |
| if not onlyload: |
| # start the cyclictest thread |
| self.info("starting cyclictest") |
| self.cyclictest.start() |
| |
| # turn loose the loads |
| self.info("sending start event to all loads") |
| for l in self.loads: |
| l.startevent.set() |
| nthreads += 1 |
| |
| accum = 0.0 |
| samples = 0 |
| |
| report_interval = int(self.config.GetSection('rteval').report_interval) |
| |
| # wait for time to expire or thread to die |
| signal.signal(signal.SIGINT, sigint_handler) |
| signal.signal(signal.SIGTERM, sigterm_handler) |
| self.info("waiting for duration (%f)" % self.config.duration) |
| stoptime = (time.time() + self.config.duration) |
| currtime = time.time() |
| rpttime = currtime + report_interval |
| loadcount = 5 |
| while (currtime <= stoptime) and not sigint_received: |
| time.sleep(1.0) |
| if not onlyload and not self.cyclictest.isAlive(): |
| raise RuntimeError, "cyclictest thread died!" |
| if len(threading.enumerate()) < nthreads: |
| raise RuntimeError, "load thread died!" |
| if not loadcount: |
| # open the loadavg /proc entry |
| p = open("/proc/loadavg") |
| load = float(p.readline().split()[0]) |
| p.close() |
| accum += load |
| samples += 1 |
| loadcount = 5 |
| #self.debug("current loadavg: %f, running avg: %f (load: %f, samples: %d)" % \ |
| # (load, accum/samples, load, samples)) |
| else: |
| loadcount -= 1 |
| if currtime >= rpttime: |
| left_to_run = stoptime - currtime |
| self.show_remaining_time(left_to_run) |
| rpttime = currtime + report_interval |
| print "load average: %.2f" % (accum / samples) |
| currtime = time.time() |
| self.debug("out of measurement loop") |
| signal.signal(signal.SIGINT, signal.SIG_DFL) |
| signal.signal(signal.SIGTERM, signal.SIG_DFL) |
| |
| except RuntimeError, e: |
| print "Runtime error during measurement: %s", e |
| raise |
| |
| finally: |
| if not onlyload: |
| # stop cyclictest |
| self.cyclictest.stopevent.set() |
| |
| # stop the loads |
| self.stop_loads() |
| |
| print "stopping run at %s" % time.asctime() |
| if not onlyload: |
| # wait for cyclictest to finish calculating stats |
| self.cyclictest.finished.wait() |
| self.genxml(datetime.now() - start, accum, samples) |
| self.report() |
| if self.config.sysreport: |
| self.run_sysreport() |
| |
| |
| def XMLRPC_Send(self): |
| "Sends the report to a given XML-RPC host. Returns 0 on success or 2 on submission failure." |
| |
| if not self.config.xmlrpc: |
| return 2 |
| |
| url = "http://%s/rteval/API1/" % self.config.xmlrpc |
| attempt = 0 |
| exitcode = 2 # Presume failure |
| warning_sent = False |
| while attempt < 6: |
| try: |
| client = rtevalclient.rtevalclient(url) |
| print "Submitting report to %s" % url |
| rterid = client.SendReport(self.xmlreport.GetXMLdocument()) |
| print "Report registered with submission id %i" % rterid |
| attempt = 10 |
| exitcode = 0 # Success |
| except socket.error: |
| if (self.mailer is not None) and (not warning_sent): |
| self.mailer.SendMessage("[RTEVAL:WARNING] Failed to submit report to XML-RPC server", |
| "Server %s did not respond. Not giving up yet." |
| % self.config.xmlrpc) |
| warning_sent = True |
| |
| attempt += 1 |
| if attempt > 5: |
| break # To avoid sleeping before we abort |
| |
| print "Failed sending report. Doing another attempt(%i) " % attempt |
| time.sleep(attempt*5*60) # Incremental sleep - sleep attempts*5 minutes |
| |
| except Exception, err: |
| raise err |
| |
| if (self.mailer is not None): |
| # Send final result messages |
| if exitcode == 2: |
| self.mailer.SendMessage("[RTEVAL:FAILURE] Failed to submit report to XML-RPC server", |
| "Server %s did not respond at all after %i attempts." |
| % (self.config.xmlrpc, attempt - 1)) |
| elif (exitcode == 0) and warning_sent: |
| self.mailer.SendMessage("[RTEVAL:SUCCESS] XML-RPC server available again", |
| "Succeeded to submit the report to %s in the end." |
| % (self.config.xmlrpc)) |
| return exitcode |
| |
| |
| def tar_results(self): |
| if not os.path.isdir(self.reportdir): |
| raise RuntimeError, "no such directory: %s" % self.reportdir |
| import tarfile |
| dirname = os.path.dirname(self.reportdir) |
| rptdir = os.path.basename(self.reportdir) |
| cwd = os.getcwd() |
| os.chdir(dirname) |
| try: |
| t = tarfile.open(rptdir + ".tar.bz2", "w:bz2") |
| t.add(rptdir) |
| t.close() |
| except: |
| os.chdir(cwd) |
| |
| def summarize(self, file): |
| isarchive = False |
| summary = file |
| if file.endswith(".tar.bz2"): |
| import tarfile |
| try: |
| t = tarfile.open(file) |
| except: |
| print "Don't know how to summarize %s (tarfile open failed)" % file |
| return |
| element = None |
| for f in t.getnames(): |
| if f.find('summary.xml') != -1: |
| element = f |
| break |
| if element == None: |
| print "No summary.xml found in tar archive %s" % file |
| return |
| tmp = tempfile.gettempdir() |
| self.debug("extracting %s from %s for summarizing" % (element, file)) |
| t.extract(element, path=tmp) |
| summary = os.path.join(tmp, element) |
| isarchive = True |
| self.show_report(summary, 'rteval_text.xsl') |
| if isarchive: |
| os.unlink(summary) |
| |
| def rteval(self): |
| ''' main function for rteval''' |
| retval = 0; |
| |
| # Parse initial DMI decoding errors |
| dmi.ProcessWarnings() |
| |
| # if --summarize was specified then just parse the XML, print it and exit |
| if self.cmd_options.summarize or self.cmd_options.rawhistogram: |
| if len(self.cmd_arguments) < 1: |
| raise RuntimeError, "Must specify at least one XML file with --summarize!" |
| |
| for x in self.cmd_arguments: |
| if self.cmd_options.summarize: |
| self.summarize(x) |
| elif self.cmd_options.rawhistogram: |
| self.show_report(x, 'rteval_histogram_raw.xsl') |
| |
| sys.exit(0) |
| |
| if os.getuid() != 0: |
| print "Must be root to run rteval!" |
| sys.exit(-1) |
| |
| self.debug('''rteval options: |
| workdir: %s |
| loaddir: %s |
| reportdir: %s |
| verbose: %s |
| debugging: %s |
| logging: %s |
| duration: %f |
| sysreport: %s |
| inifile: %s''' % (self.workdir, self.config.srcdir, self.reportdir, self.config.verbose, |
| self.config.debugging, self.config.logging, self.config.duration, |
| self.config.sysreport, self.inifile)) |
| |
| if not os.path.isdir(self.workdir): |
| raise RuntimeError, "work directory %d does not exist" % self.workdir |
| |
| # create our report directory |
| try: |
| self.make_report_dir() |
| except: |
| print "Cannot create the report dir!" |
| print "(is this an NFS filesystem with rootsquash turned on?)" |
| sys.exit(-1) |
| |
| self.measure() |
| |
| # if --xmlrpc-submit | -X was given, send our report to this host |
| if self.config.xmlrpc: |
| retval = self.XMLRPC_Send() |
| |
| self.get_dmesg() |
| self.tar_results() |
| |
| return retval |
| |
| if __name__ == '__main__': |
| import pwd, grp |
| |
| try: |
| # Parse initial DMI decoding errors |
| dmi.ProcessWarnings() |
| |
| rteval = RtEval(sys.argv[1:]) |
| ec = rteval.rteval() |
| rteval.debug("exiting with exit code: %d" % ec) |
| sys.exit(ec) |
| except KeyboardInterrupt: |
| sys.exit(0) |
