| #!/usr/bin/env python3 |
| # SPDX-License-Identifier: GPL-2.0-only |
| # |
| # Copyright (c) 2020 Western Digital Corporation or its affiliates. |
| # |
| """ |
| # latency_percentiles.py |
| # |
| # Test the code that produces latency percentiles |
| # This is mostly to test the code changes to allow reporting |
| # of slat, clat, and lat percentiles |
| # |
| # USAGE |
| # python3 latency-tests.py [-f fio-path] [-a artifact-root] [--debug] |
| # |
| # |
| # Test scenarios: |
| # |
| # - DONE json |
| # unified rw reporting |
| # compare with latency log |
| # try various combinations of the ?lat_percentile options |
| # null, aio |
| # r, w, t |
| # - DONE json+ |
| # check presence of latency bins |
| # if the json percentiles match those from the raw data |
| # then the latency bin values and counts are probably ok |
| # - DONE terse |
| # produce both terse, JSON output and confirm that they match |
| # lat only; both lat and clat |
| # - DONE sync_lat |
| # confirm that sync_lat data appears |
| # - MANUAL TESTING normal output: |
| # null ioengine |
| # enable all, but only clat and lat appear |
| # enable subset of latency types |
| # read, write, trim, unified |
| # libaio ioengine |
| # enable all latency types |
| # enable subset of latency types |
| # read, write, trim, unified |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=null --slat_percentiles=1 --clat_percentiles=1 --lat_percentiles=1 |
| # echo confirm that clat and lat percentiles appear |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=null --slat_percentiles=0 --clat_percentiles=0 --lat_percentiles=1 |
| # echo confirm that only lat percentiles appear |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=null --slat_percentiles=0 --clat_percentiles=1 --lat_percentiles=0 |
| # echo confirm that only clat percentiles appear |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=libaio --slat_percentiles=1 --clat_percentiles=1 --lat_percentiles=1 |
| # echo confirm that slat, clat, lat percentiles appear |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=libaio --slat_percentiles=0 --clat_percentiles=1 --lat_percentiles=1 |
| # echo confirm that clat and lat percentiles appear |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=libaio -rw=randrw |
| # echo confirm that clat percentiles appear for reads and writes |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=libaio --slat_percentiles=1 --clat_percentiles=0 --lat_percentiles=0 --rw=randrw |
| # echo confirm that slat percentiles appear for both reads and writes |
| # ./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=libaio --slat_percentiles=1 --clat_percentiles=1 --lat_percentiles=1 \ |
| # --rw=randrw --unified_rw_reporting=1 |
| # echo confirm that slat, clat, and lat percentiles appear for 'mixed' IOs |
| #./fio/fio --name=test --randrepeat=0 --norandommap --time_based --runtime=2s --size=512M \ |
| # --ioengine=null --slat_percentiles=1 --clat_percentiles=1 --lat_percentiles=1 \ |
| # --rw=randrw --fsync=32 |
| # echo confirm that fsync latencies appear |
| """ |
| |
| import os |
| import csv |
| import sys |
| import json |
| import math |
| import time |
| import argparse |
| import platform |
| import subprocess |
| from collections import Counter |
| from pathlib import Path |
| |
| |
| class FioLatTest(): |
| """fio latency percentile test.""" |
| |
| def __init__(self, artifact_root, test_options, debug): |
| """ |
| artifact_root root directory for artifacts (subdirectory will be created under here) |
| test test specification |
| """ |
| self.artifact_root = artifact_root |
| self.test_options = test_options |
| self.debug = debug |
| self.filename = None |
| self.json_data = None |
| self.terse_data = None |
| |
| self.test_dir = os.path.join(self.artifact_root, |
| "{:03d}".format(self.test_options['test_id'])) |
| if not os.path.exists(self.test_dir): |
| os.mkdir(self.test_dir) |
| |
| self.filename = "latency{:03d}".format(self.test_options['test_id']) |
| |
| def run_fio(self, fio_path): |
| """Run a test.""" |
| |
| fio_args = [ |
| "--max-jobs=16", |
| "--name=latency", |
| "--randrepeat=0", |
| "--norandommap", |
| "--time_based", |
| "--size=16M", |
| "--rwmixread=50", |
| "--group_reporting=1", |
| "--write_lat_log={0}".format(self.filename), |
| "--output={0}.out".format(self.filename), |
| "--ioengine={ioengine}".format(**self.test_options), |
| "--rw={rw}".format(**self.test_options), |
| "--runtime={runtime}".format(**self.test_options), |
| "--output-format={output-format}".format(**self.test_options), |
| ] |
| for opt in ['slat_percentiles', 'clat_percentiles', 'lat_percentiles', |
| 'unified_rw_reporting', 'fsync', 'fdatasync', 'numjobs', |
| 'cmdprio_percentage', 'bssplit', 'cmdprio_bssplit']: |
| if opt in self.test_options: |
| option = '--{0}={{{0}}}'.format(opt) |
| fio_args.append(option.format(**self.test_options)) |
| |
| command = [fio_path] + fio_args |
| with open(os.path.join(self.test_dir, "{0}.command".format(self.filename)), "w+") as \ |
| command_file: |
| command_file.write("%s\n" % command) |
| |
| passed = True |
| stdout_file = open(os.path.join(self.test_dir, "{0}.stdout".format(self.filename)), "w+") |
| stderr_file = open(os.path.join(self.test_dir, "{0}.stderr".format(self.filename)), "w+") |
| exitcode_file = open(os.path.join(self.test_dir, |
| "{0}.exitcode".format(self.filename)), "w+") |
| try: |
| proc = None |
| # Avoid using subprocess.run() here because when a timeout occurs, |
| # fio will be stopped with SIGKILL. This does not give fio a |
| # chance to clean up and means that child processes may continue |
| # running and submitting IO. |
| proc = subprocess.Popen(command, |
| stdout=stdout_file, |
| stderr=stderr_file, |
| cwd=self.test_dir, |
| universal_newlines=True) |
| proc.communicate(timeout=300) |
| exitcode_file.write('{0}\n'.format(proc.returncode)) |
| passed &= (proc.returncode == 0) |
| except subprocess.TimeoutExpired: |
| proc.terminate() |
| proc.communicate() |
| assert proc.poll() |
| print("Timeout expired") |
| passed = False |
| except Exception: |
| if proc: |
| if not proc.poll(): |
| proc.terminate() |
| proc.communicate() |
| print("Exception: %s" % sys.exc_info()) |
| passed = False |
| finally: |
| stdout_file.close() |
| stderr_file.close() |
| exitcode_file.close() |
| |
| if passed: |
| if 'json' in self.test_options['output-format']: |
| if not self.get_json(): |
| print('Unable to decode JSON data') |
| passed = False |
| if 'terse' in self.test_options['output-format']: |
| if not self.get_terse(): |
| print('Unable to decode terse data') |
| passed = False |
| |
| return passed |
| |
| def get_json(self): |
| """Convert fio JSON output into a python JSON object""" |
| |
| filename = os.path.join(self.test_dir, "{0}.out".format(self.filename)) |
| with open(filename, 'r') as file: |
| file_data = file.read() |
| |
| # |
| # Sometimes fio informational messages are included at the top of the |
| # JSON output, especially under Windows. Try to decode output as JSON |
| # data, lopping off up to the first four lines |
| # |
| lines = file_data.splitlines() |
| for i in range(5): |
| file_data = '\n'.join(lines[i:]) |
| try: |
| self.json_data = json.loads(file_data) |
| except json.JSONDecodeError: |
| continue |
| else: |
| return True |
| |
| return False |
| |
| def get_terse(self): |
| """Read fio output and return terse format data.""" |
| |
| filename = os.path.join(self.test_dir, "{0}.out".format(self.filename)) |
| with open(filename, 'r') as file: |
| file_data = file.read() |
| |
| # |
| # Read the first few lines and see if any of them begin with '3;' |
| # If so, the line is probably terse output. Obviously, this only |
| # works for fio terse version 3 and it does not work for |
| # multi-line terse output |
| # |
| lines = file_data.splitlines() |
| for i in range(8): |
| file_data = lines[i] |
| if file_data.startswith('3;'): |
| self.terse_data = file_data.split(';') |
| return True |
| |
| return False |
| |
| def check_latencies(self, jsondata, ddir, slat=True, clat=True, tlat=True, plus=False, |
| unified=False): |
| """Check fio latency data. |
| |
| ddir data direction to check (0=read, 1=write, 2=trim) |
| slat True if submission latency data available to check |
| clat True if completion latency data available to check |
| tlat True of total latency data available to check |
| plus True if we actually have json+ format data where additional checks can |
| be carried out |
| unified True if fio is reporting unified r/w data |
| """ |
| |
| types = { |
| 'slat': slat, |
| 'clat': clat, |
| 'lat': tlat |
| } |
| |
| retval = True |
| |
| for lat in ['slat', 'clat', 'lat']: |
| this_iter = True |
| if not types[lat]: |
| if 'percentile' in jsondata[lat+'_ns']: |
| this_iter = False |
| print('unexpected %s percentiles found' % lat) |
| else: |
| print("%s percentiles skipped" % lat) |
| continue |
| else: |
| if 'percentile' not in jsondata[lat+'_ns']: |
| this_iter = False |
| print('%s percentiles not found in fio output' % lat) |
| |
| # |
| # Check only for the presence/absence of json+ |
| # latency bins. Future work can check the |
| # accuracy of the bin values and counts. |
| # |
| # Because the latency percentiles are based on |
| # the bins, we can be confident that the bin |
| # values and counts are correct if fio's |
| # latency percentiles match what we compute |
| # from the raw data. |
| # |
| if plus: |
| if 'bins' not in jsondata[lat+'_ns']: |
| print('bins not found with json+ output format') |
| this_iter = False |
| else: |
| if not self.check_jsonplus(jsondata[lat+'_ns']): |
| this_iter = False |
| else: |
| if 'bins' in jsondata[lat+'_ns']: |
| print('json+ bins found with json output format') |
| this_iter = False |
| |
| latencies = [] |
| for i in range(10): |
| lat_file = os.path.join(self.test_dir, "%s_%s.%s.log" % (self.filename, lat, i+1)) |
| if not os.path.exists(lat_file): |
| break |
| with open(lat_file, 'r', newline='') as file: |
| reader = csv.reader(file) |
| for line in reader: |
| if unified or int(line[2]) == ddir: |
| latencies.append(int(line[1])) |
| |
| if int(jsondata['total_ios']) != len(latencies): |
| this_iter = False |
| print('%s: total_ios = %s, latencies logged = %d' % \ |
| (lat, jsondata['total_ios'], len(latencies))) |
| elif self.debug: |
| print("total_ios %s match latencies logged" % jsondata['total_ios']) |
| |
| latencies.sort() |
| ptiles = jsondata[lat+'_ns']['percentile'] |
| |
| for percentile in ptiles.keys(): |
| # |
| # numpy.percentile(latencies, float(percentile), |
| # interpolation='higher') |
| # produces values that mostly match what fio reports |
| # however, in the tails of the distribution, the values produced |
| # by fio's and numpy.percentile's algorithms are occasionally off |
| # by one latency measurement. So instead of relying on the canned |
| # numpy.percentile routine, implement here fio's algorithm |
| # |
| rank = math.ceil(float(percentile)/100 * len(latencies)) |
| if rank > 0: |
| index = rank - 1 |
| else: |
| index = 0 |
| value = latencies[int(index)] |
| fio_val = int(ptiles[percentile]) |
| # The theory in stat.h says that the proportional error will be |
| # less than 1/128 |
| if not self.similar(fio_val, value): |
| delta = abs(fio_val - value) / value |
| print("Error with %s %sth percentile: " |
| "fio: %d, expected: %d, proportional delta: %f" % |
| (lat, percentile, fio_val, value, delta)) |
| print("Rank: %d, index: %d" % (rank, index)) |
| this_iter = False |
| elif self.debug: |
| print('%s %sth percentile values match: %d, %d' % |
| (lat, percentile, fio_val, value)) |
| |
| if this_iter: |
| print("%s percentiles match" % lat) |
| else: |
| retval = False |
| |
| return retval |
| |
| @staticmethod |
| def check_empty(job): |
| """ |
| Make sure JSON data is empty. |
| |
| Some data structures should be empty. This function makes sure that they are. |
| |
| job JSON object that we need to check for emptiness |
| """ |
| |
| return job['total_ios'] == 0 and \ |
| job['slat_ns']['N'] == 0 and \ |
| job['clat_ns']['N'] == 0 and \ |
| job['lat_ns']['N'] == 0 |
| |
| def check_nocmdprio_lat(self, job): |
| """ |
| Make sure no per priority latencies appear. |
| |
| job JSON object to check |
| """ |
| |
| for ddir in ['read', 'write', 'trim']: |
| if ddir in job: |
| if 'prios' in job[ddir]: |
| print("Unexpected per priority latencies found in %s output" % ddir) |
| return False |
| |
| if self.debug: |
| print("No per priority latencies found") |
| |
| return True |
| |
| @staticmethod |
| def similar(approximation, actual): |
| """ |
| Check whether the approximate values recorded by fio are within the theoretical bound. |
| |
| Since it is impractical to store exact latency measurements for each and every IO, fio |
| groups similar latency measurements into variable-sized bins. The theory in stat.h says |
| that the proportional error will be less than 1/128. This function checks whether this |
| is true. |
| |
| TODO This test will fail when comparing a value from the largest latency bin against its |
| actual measurement. Find some way to detect this and avoid failing. |
| |
| approximation value of the bin used by fio to store a given latency |
| actual actual latency value |
| """ |
| |
| # Avoid a division by zero. The smallest latency values have no error. |
| if actual == 0: |
| return approximation == 0 |
| |
| delta = abs(approximation - actual) / actual |
| return delta <= 1/128 |
| |
| def check_jsonplus(self, jsondata): |
| """Check consistency of json+ data |
| |
| When we have json+ data we can check the min value, max value, and |
| sample size reported by fio |
| |
| jsondata json+ data that we need to check |
| """ |
| |
| retval = True |
| |
| keys = [int(k) for k in jsondata['bins'].keys()] |
| values = [int(jsondata['bins'][k]) for k in jsondata['bins'].keys()] |
| smallest = min(keys) |
| biggest = max(keys) |
| sampsize = sum(values) |
| |
| if not self.similar(jsondata['min'], smallest): |
| retval = False |
| print('reported min %d does not match json+ min %d' % (jsondata['min'], smallest)) |
| elif self.debug: |
| print('json+ min values match: %d' % jsondata['min']) |
| |
| if not self.similar(jsondata['max'], biggest): |
| retval = False |
| print('reported max %d does not match json+ max %d' % (jsondata['max'], biggest)) |
| elif self.debug: |
| print('json+ max values match: %d' % jsondata['max']) |
| |
| if sampsize != jsondata['N']: |
| retval = False |
| print('reported sample size %d does not match json+ total count %d' % \ |
| (jsondata['N'], sampsize)) |
| elif self.debug: |
| print('json+ sample sizes match: %d' % sampsize) |
| |
| return retval |
| |
| def check_sync_lat(self, jsondata, plus=False): |
| """Check fsync latency percentile data. |
| |
| All we can check is that some percentiles are reported, unless we have json+ data. |
| If we actually have json+ data then we can do more checking. |
| |
| jsondata JSON data for fsync operations |
| plus True if we actually have json+ data |
| """ |
| retval = True |
| |
| if 'percentile' not in jsondata['lat_ns']: |
| print("Sync percentile data not found") |
| return False |
| |
| if int(jsondata['total_ios']) != int(jsondata['lat_ns']['N']): |
| retval = False |
| print('Mismatch between total_ios and lat_ns sample size') |
| elif self.debug: |
| print('sync sample sizes match: %d' % jsondata['total_ios']) |
| |
| if not plus: |
| if 'bins' in jsondata['lat_ns']: |
| print('Unexpected json+ bin data found') |
| return False |
| |
| if not self.check_jsonplus(jsondata['lat_ns']): |
| retval = False |
| |
| return retval |
| |
| def check_terse(self, terse, jsondata): |
| """Compare terse latencies with JSON latencies. |
| |
| terse terse format data for checking |
| jsondata JSON format data for checking |
| """ |
| |
| retval = True |
| |
| for lat in terse: |
| split = lat.split('%') |
| pct = split[0] |
| terse_val = int(split[1][1:]) |
| json_val = math.floor(jsondata[pct]/1000) |
| if terse_val != json_val: |
| retval = False |
| print('Mismatch with %sth percentile: json value=%d,%d terse value=%d' % \ |
| (pct, jsondata[pct], json_val, terse_val)) |
| elif self.debug: |
| print('Terse %sth percentile matches JSON value: %d' % (pct, terse_val)) |
| |
| return retval |
| |
| def check_prio_latencies(self, jsondata, clat=True, plus=False): |
| """Check consistency of per priority latencies. |
| |
| clat True if we should check clat data; other check lat data |
| plus True if we have json+ format data where additional checks can |
| be carried out |
| unified True if fio is reporting unified r/w data |
| """ |
| |
| if clat: |
| obj = combined = 'clat_ns' |
| else: |
| obj = combined = 'lat_ns' |
| |
| if not 'prios' in jsondata or not combined in jsondata: |
| print("Error identifying per priority latencies") |
| return False |
| |
| sum_sample_size = sum([x[obj]['N'] for x in jsondata['prios']]) |
| if sum_sample_size != jsondata[combined]['N']: |
| print("Per prio sample size sum %d != combined sample size %d" % |
| (sum_sample_size, jsondata[combined]['N'])) |
| return False |
| elif self.debug: |
| print("Per prio sample size sum %d == combined sample size %d" % |
| (sum_sample_size, jsondata[combined]['N'])) |
| |
| min_val = min([x[obj]['min'] for x in jsondata['prios']]) |
| if min_val != jsondata[combined]['min']: |
| print("Min per prio min latency %d does not match min %d from combined data" % |
| (min_val, jsondata[combined]['min'])) |
| return False |
| elif self.debug: |
| print("Min per prio min latency %d matches min %d from combined data" % |
| (min_val, jsondata[combined]['min'])) |
| |
| max_val = max([x[obj]['max'] for x in jsondata['prios']]) |
| if max_val != jsondata[combined]['max']: |
| print("Max per prio max latency %d does not match max %d from combined data" % |
| (max_val, jsondata[combined]['max'])) |
| return False |
| elif self.debug: |
| print("Max per prio max latency %d matches max %d from combined data" % |
| (max_val, jsondata[combined]['max'])) |
| |
| weighted_vals = [x[obj]['mean'] * x[obj]['N'] for x in jsondata['prios']] |
| weighted_avg = sum(weighted_vals) / jsondata[combined]['N'] |
| delta = abs(weighted_avg - jsondata[combined]['mean']) |
| if (delta / jsondata[combined]['mean']) > 0.0001: |
| print("Difference between merged per prio weighted average %f mean " |
| "and actual mean %f exceeds 0.01%%" % (weighted_avg, jsondata[combined]['mean'])) |
| return False |
| elif self.debug: |
| print("Merged per prio weighted average %f mean matches actual mean %f" % |
| (weighted_avg, jsondata[combined]['mean'])) |
| |
| if plus: |
| for prio in jsondata['prios']: |
| if not self.check_jsonplus(prio[obj]): |
| return False |
| |
| counter = Counter() |
| for prio in jsondata['prios']: |
| counter.update(prio[obj]['bins']) |
| |
| bins = dict(counter) |
| |
| if len(bins) != len(jsondata[combined]['bins']): |
| print("Number of merged bins %d does not match number of overall bins %d" % |
| (len(bins), len(jsondata[combined]['bins']))) |
| return False |
| elif self.debug: |
| print("Number of merged bins %d matches number of overall bins %d" % |
| (len(bins), len(jsondata[combined]['bins']))) |
| |
| for duration in bins.keys(): |
| if bins[duration] != jsondata[combined]['bins'][duration]: |
| print("Merged per prio count does not match overall count for duration %d" % |
| duration) |
| return False |
| |
| print("Merged per priority latency data match combined latency data") |
| return True |
| |
| def check(self): |
| """Check test output.""" |
| |
| raise NotImplementedError() |
| |
| |
| class Test001(FioLatTest): |
| """Test object for Test 1.""" |
| |
| def check(self): |
| """Check Test 1 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, slat=False) |
| |
| return retval |
| |
| |
| class Test002(FioLatTest): |
| """Test object for Test 2.""" |
| |
| def check(self): |
| """Check Test 2 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['read']): |
| print("Unexpected read data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['write'], 1, slat=False, clat=False) |
| |
| return retval |
| |
| |
| class Test003(FioLatTest): |
| """Test object for Test 3.""" |
| |
| def check(self): |
| """Check Test 3 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['read']): |
| print("Unexpected read data found in output") |
| retval = False |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['trim'], 2, slat=False, tlat=False) |
| |
| return retval |
| |
| |
| class Test004(FioLatTest): |
| """Test object for Tests 4, 13.""" |
| |
| def check(self): |
| """Check Test 4, 13 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, plus=True) |
| |
| return retval |
| |
| |
| class Test005(FioLatTest): |
| """Test object for Test 5.""" |
| |
| def check(self): |
| """Check Test 5 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['read']): |
| print("Unexpected read data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['write'], 1, slat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test006(FioLatTest): |
| """Test object for Test 6.""" |
| |
| def check(self): |
| """Check Test 6 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, slat=False, tlat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test007(FioLatTest): |
| """Test object for Test 7.""" |
| |
| def check(self): |
| """Check Test 7 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, clat=False, tlat=False, plus=True) |
| retval &= self.check_latencies(job['write'], 1, clat=False, tlat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test008(FioLatTest): |
| """Test object for Tests 8, 14.""" |
| |
| def check(self): |
| """Check Test 8, 14 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if 'read' in job or 'write' in job or 'trim' in job: |
| print("Unexpected data direction found in fio output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['mixed'], 0, plus=True, unified=True) |
| |
| return retval |
| |
| |
| class Test009(FioLatTest): |
| """Test object for Test 9.""" |
| |
| def check(self): |
| """Check Test 9 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['read']): |
| print("Unexpected read data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_sync_lat(job['sync'], plus=True): |
| print("Error checking fsync latency data") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['write'], 1, slat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test010(FioLatTest): |
| """Test object for Test 10.""" |
| |
| def check(self): |
| """Check Test 10 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, plus=True) |
| retval &= self.check_latencies(job['write'], 1, plus=True) |
| retval &= self.check_terse(self.terse_data[17:34], job['read']['lat_ns']['percentile']) |
| retval &= self.check_terse(self.terse_data[58:75], job['write']['lat_ns']['percentile']) |
| # Terse data checking only works for default percentiles. |
| # This needs to be changed if something other than the default is ever used. |
| |
| return retval |
| |
| |
| class Test011(FioLatTest): |
| """Test object for Test 11.""" |
| |
| def check(self): |
| """Check Test 11 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| if not self.check_nocmdprio_lat(job): |
| print("Unexpected per priority latencies found") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, slat=False, clat=False, plus=True) |
| retval &= self.check_latencies(job['write'], 1, slat=False, clat=False, plus=True) |
| retval &= self.check_terse(self.terse_data[17:34], job['read']['lat_ns']['percentile']) |
| retval &= self.check_terse(self.terse_data[58:75], job['write']['lat_ns']['percentile']) |
| # Terse data checking only works for default percentiles. |
| # This needs to be changed if something other than the default is ever used. |
| |
| return retval |
| |
| |
| class Test015(FioLatTest): |
| """Test object for Test 15.""" |
| |
| def check(self): |
| """Check Test 15 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, plus=True) |
| retval &= self.check_prio_latencies(job['read'], clat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test016(FioLatTest): |
| """Test object for Test 16.""" |
| |
| def check(self): |
| """Check Test 16 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['read']): |
| print("Unexpected read data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| |
| retval &= self.check_latencies(job['write'], 1, slat=False, plus=True) |
| retval &= self.check_prio_latencies(job['write'], clat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test017(FioLatTest): |
| """Test object for Test 17.""" |
| |
| def check(self): |
| """Check Test 17 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['write']): |
| print("Unexpected write data found in output") |
| retval = False |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, slat=False, tlat=False, plus=True) |
| retval &= self.check_prio_latencies(job['read'], plus=True) |
| |
| return retval |
| |
| |
| class Test018(FioLatTest): |
| """Test object for Test 18.""" |
| |
| def check(self): |
| """Check Test 18 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, clat=False, tlat=False, plus=True) |
| retval &= self.check_latencies(job['write'], 1, clat=False, tlat=False, plus=True) |
| |
| # We actually have json+ data but setting plus=False below avoids checking the |
| # json+ bins which did not exist for clat and lat because this job is run with |
| # clat_percentiles=0, lat_percentiles=0, However, we can still check the summary |
| # statistics |
| retval &= self.check_prio_latencies(job['write'], plus=False) |
| retval &= self.check_prio_latencies(job['read'], plus=False) |
| |
| return retval |
| |
| |
| class Test019(FioLatTest): |
| """Test object for Tests 19, 20.""" |
| |
| def check(self): |
| """Check Test 19, 20 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if 'read' in job or 'write' in job or 'trim' in job: |
| print("Unexpected data direction found in fio output") |
| retval = False |
| |
| retval &= self.check_latencies(job['mixed'], 0, plus=True, unified=True) |
| retval &= self.check_prio_latencies(job['mixed'], clat=False, plus=True) |
| |
| return retval |
| |
| |
| class Test021(FioLatTest): |
| """Test object for Test 21.""" |
| |
| def check(self): |
| """Check Test 21 output.""" |
| |
| job = self.json_data['jobs'][0] |
| |
| retval = True |
| if not self.check_empty(job['trim']): |
| print("Unexpected trim data found in output") |
| retval = False |
| |
| retval &= self.check_latencies(job['read'], 0, slat=False, tlat=False, plus=True) |
| retval &= self.check_latencies(job['write'], 1, slat=False, tlat=False, plus=True) |
| retval &= self.check_prio_latencies(job['read'], clat=True, plus=True) |
| retval &= self.check_prio_latencies(job['write'], clat=True, plus=True) |
| |
| return retval |
| |
| |
| def parse_args(): |
| """Parse command-line arguments.""" |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument('-f', '--fio', help='path to file executable (e.g., ./fio)') |
| parser.add_argument('-a', '--artifact-root', help='artifact root directory') |
| parser.add_argument('-d', '--debug', help='enable debug output', action='store_true') |
| parser.add_argument('-s', '--skip', nargs='+', type=int, |
| help='list of test(s) to skip') |
| parser.add_argument('-o', '--run-only', nargs='+', type=int, |
| help='list of test(s) to run, skipping all others') |
| args = parser.parse_args() |
| |
| return args |
| |
| |
| def main(): |
| """Run tests of fio latency percentile reporting""" |
| |
| args = parse_args() |
| |
| artifact_root = args.artifact_root if args.artifact_root else \ |
| "latency-test-{0}".format(time.strftime("%Y%m%d-%H%M%S")) |
| os.mkdir(artifact_root) |
| print("Artifact directory is %s" % artifact_root) |
| |
| if args.fio: |
| fio = str(Path(args.fio).absolute()) |
| else: |
| fio = 'fio' |
| print("fio path is %s" % fio) |
| |
| if platform.system() == 'Linux': |
| aio = 'libaio' |
| elif platform.system() == 'Windows': |
| aio = 'windowsaio' |
| else: |
| aio = 'posixaio' |
| |
| test_list = [ |
| { |
| # randread, null |
| # enable slat, clat, lat |
| # only clat and lat will appear because |
| # because the null ioengine is synchronous |
| "test_id": 1, |
| "runtime": 2, |
| "output-format": "json", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": 'null', |
| 'rw': 'randread', |
| "test_obj": Test001, |
| }, |
| { |
| # randwrite, null |
| # enable lat only |
| "test_id": 2, |
| "runtime": 2, |
| "output-format": "json", |
| "slat_percentiles": 0, |
| "clat_percentiles": 0, |
| "lat_percentiles": 1, |
| "ioengine": 'null', |
| 'rw': 'randwrite', |
| "test_obj": Test002, |
| }, |
| { |
| # randtrim, null |
| # enable clat only |
| "test_id": 3, |
| "runtime": 2, |
| "output-format": "json", |
| "slat_percentiles": 0, |
| "clat_percentiles": 1, |
| "lat_percentiles": 0, |
| "ioengine": 'null', |
| 'rw': 'randtrim', |
| "test_obj": Test003, |
| }, |
| { |
| # randread, aio |
| # enable slat, clat, lat |
| # all will appear because libaio is asynchronous |
| "test_id": 4, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randread', |
| "test_obj": Test004, |
| }, |
| { |
| # randwrite, aio |
| # enable only clat, lat |
| "test_id": 5, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 0, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randwrite', |
| "test_obj": Test005, |
| }, |
| { |
| # randread, aio |
| # by default only clat should appear |
| "test_id": 6, |
| "runtime": 5, |
| "output-format": "json+", |
| "ioengine": aio, |
| 'rw': 'randread', |
| "test_obj": Test006, |
| }, |
| { |
| # 50/50 r/w, aio |
| # enable only slat |
| "test_id": 7, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 0, |
| "lat_percentiles": 0, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| "test_obj": Test007, |
| }, |
| { |
| # 50/50 r/w, aio, unified_rw_reporting |
| # enable slat, clat, lat |
| "test_id": 8, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'unified_rw_reporting': 1, |
| "test_obj": Test008, |
| }, |
| { |
| # randwrite, null |
| # enable slat, clat, lat |
| # fsync |
| "test_id": 9, |
| "runtime": 2, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": 'null', |
| 'rw': 'randwrite', |
| 'fsync': 32, |
| "test_obj": Test009, |
| }, |
| { |
| # 50/50 r/w, aio |
| # enable slat, clat, lat |
| "test_id": 10, |
| "runtime": 5, |
| "output-format": "terse,json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| "test_obj": Test010, |
| }, |
| { |
| # 50/50 r/w, aio |
| # enable only lat |
| "test_id": 11, |
| "runtime": 5, |
| "output-format": "terse,json+", |
| "slat_percentiles": 0, |
| "clat_percentiles": 0, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| "test_obj": Test011, |
| }, |
| { |
| # randread, null |
| # enable slat, clat, lat |
| # only clat and lat will appear because |
| # because the null ioengine is synchronous |
| # same as Test 1 except add numjobs = 4 to test |
| # sum_thread_stats() changes |
| "test_id": 12, |
| "runtime": 2, |
| "output-format": "json", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": 'null', |
| 'rw': 'randread', |
| 'numjobs': 4, |
| "test_obj": Test001, |
| }, |
| { |
| # randread, aio |
| # enable slat, clat, lat |
| # all will appear because libaio is asynchronous |
| # same as Test 4 except add numjobs = 4 to test |
| # sum_thread_stats() changes |
| "test_id": 13, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randread', |
| 'numjobs': 4, |
| "test_obj": Test004, |
| }, |
| { |
| # 50/50 r/w, aio, unified_rw_reporting |
| # enable slat, clat, lata |
| # same as Test 8 except add numjobs = 4 to test |
| # sum_thread_stats() changes |
| "test_id": 14, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'unified_rw_reporting': 1, |
| 'numjobs': 4, |
| "test_obj": Test008, |
| }, |
| { |
| # randread, aio |
| # enable slat, clat, lat |
| # all will appear because libaio is asynchronous |
| # same as Test 4 except add cmdprio_percentage |
| "test_id": 15, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randread', |
| 'cmdprio_percentage': 50, |
| "test_obj": Test015, |
| }, |
| { |
| # randwrite, aio |
| # enable only clat, lat |
| # same as Test 5 except add cmdprio_percentage |
| "test_id": 16, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 0, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randwrite', |
| 'cmdprio_percentage': 50, |
| "test_obj": Test016, |
| }, |
| { |
| # randread, aio |
| # by default only clat should appear |
| # same as Test 6 except add cmdprio_percentage |
| "test_id": 17, |
| "runtime": 5, |
| "output-format": "json+", |
| "ioengine": aio, |
| 'rw': 'randread', |
| 'cmdprio_percentage': 50, |
| "test_obj": Test017, |
| }, |
| { |
| # 50/50 r/w, aio |
| # enable only slat |
| # same as Test 7 except add cmdprio_percentage |
| "test_id": 18, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 0, |
| "lat_percentiles": 0, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'cmdprio_percentage': 50, |
| "test_obj": Test018, |
| }, |
| { |
| # 50/50 r/w, aio, unified_rw_reporting |
| # enable slat, clat, lat |
| # same as Test 8 except add cmdprio_percentage |
| "test_id": 19, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'unified_rw_reporting': 1, |
| 'cmdprio_percentage': 50, |
| "test_obj": Test019, |
| }, |
| { |
| # 50/50 r/w, aio, unified_rw_reporting |
| # enable slat, clat, lat |
| # same as Test 19 except add numjobs = 4 to test |
| # sum_thread_stats() changes |
| "test_id": 20, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 1, |
| "clat_percentiles": 1, |
| "lat_percentiles": 1, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'unified_rw_reporting': 1, |
| 'cmdprio_percentage': 50, |
| 'numjobs': 4, |
| "test_obj": Test019, |
| }, |
| { |
| # r/w, aio |
| # enable only clat |
| # test bssplit and cmdprio_bssplit |
| "test_id": 21, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 0, |
| "clat_percentiles": 1, |
| "lat_percentiles": 0, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'bssplit': '64k/40:1024k/60', |
| 'cmdprio_bssplit': '64k/25/1/1:64k/75/3/2:1024k/0', |
| "test_obj": Test021, |
| }, |
| { |
| # r/w, aio |
| # enable only clat |
| # same as Test 21 except add numjobs = 4 to test |
| # sum_thread_stats() changes |
| "test_id": 22, |
| "runtime": 5, |
| "output-format": "json+", |
| "slat_percentiles": 0, |
| "clat_percentiles": 1, |
| "lat_percentiles": 0, |
| "ioengine": aio, |
| 'rw': 'randrw', |
| 'bssplit': '64k/40:1024k/60', |
| 'cmdprio_bssplit': '64k/25/1/1:64k/75/3/2:1024k/0', |
| 'numjobs': 4, |
| "test_obj": Test021, |
| }, |
| ] |
| |
| passed = 0 |
| failed = 0 |
| skipped = 0 |
| |
| for test in test_list: |
| if (args.skip and test['test_id'] in args.skip) or \ |
| (args.run_only and test['test_id'] not in args.run_only): |
| skipped = skipped + 1 |
| outcome = 'SKIPPED (User request)' |
| elif (platform.system() != 'Linux' or os.geteuid() != 0) and \ |
| ('cmdprio_percentage' in test or 'cmdprio_bssplit' in test): |
| skipped = skipped + 1 |
| outcome = 'SKIPPED (Linux root required for cmdprio tests)' |
| else: |
| test_obj = test['test_obj'](artifact_root, test, args.debug) |
| status = test_obj.run_fio(fio) |
| if status: |
| status = test_obj.check() |
| if status: |
| passed = passed + 1 |
| outcome = 'PASSED' |
| else: |
| failed = failed + 1 |
| outcome = 'FAILED' |
| |
| print("**********Test {0} {1}**********".format(test['test_id'], outcome)) |
| |
| print("{0} tests passed, {1} failed, {2} skipped".format(passed, failed, skipped)) |
| |
| sys.exit(failed) |
| |
| |
| if __name__ == '__main__': |
| main() |
