Handle traces to more than once device at a time
Signed-off-by: Chris Mason <chris.mason@fusionio.com>
diff --git a/blkparse.c b/blkparse.c
index f81b831..43630be 100644
--- a/blkparse.c
+++ b/blkparse.c
@@ -30,6 +30,7 @@
#include <sys/mman.h>
#include <time.h>
#include <math.h>
+#include <dirent.h>
#include "plot.h"
#include "blkparse.h"
@@ -48,6 +49,9 @@
extern int plot_io_action;
extern int io_per_process;
+static const int line_len = 1024;
+static char line[1024];
+
/*
* Trace categories
*/
@@ -158,6 +162,9 @@
/* sector offset of this IO */
u64 sector;
+ /* dev_t for this IO */
+ u32 device;
+
/* time this IO was dispatched */
u64 dispatch_time;
/* time this IO was finished */
@@ -219,20 +226,21 @@
static int io_hash_table_insert(struct pending_io *ins_pio)
{
u64 sector = ins_pio->sector;
+ u32 dev = ins_pio->device;
int slot = hash_sector(sector);
struct list_head *head;
struct pending_io *pio;
head = io_hash_table + slot;
list_for_each_entry(pio, head, hash_list) {
- if (pio->sector == sector)
+ if (pio->sector == sector && pio->device == dev)
return -EEXIST;
}
list_add_tail(&ins_pio->hash_list, head);
return 0;
}
-static struct pending_io *io_hash_table_search(u64 sector)
+static struct pending_io *io_hash_table_search(u64 sector, u32 dev)
{
int slot = hash_sector(sector);
struct list_head *head;
@@ -240,7 +248,7 @@
head = io_hash_table + slot;
list_for_each_entry(pio, head, hash_list) {
- if (pio->sector == sector)
+ if (pio->sector == sector && pio->device == dev)
return pio;
}
return NULL;
@@ -253,6 +261,7 @@
pio = calloc(1, sizeof(*pio));
pio->sector = io->sector;
+ pio->device = io->device;
pio->pid = io->pid;
ret = io_hash_table_insert(pio);
@@ -268,7 +277,7 @@
{
struct pending_io *pio;
- pio = io_hash_table_search(io->sector);
+ pio = io_hash_table_search(io->sector, io->device);
if (!pio) {
pio = hash_queued_io(io);
if (!pio)
@@ -282,7 +291,7 @@
{
struct pending_io *pio;
- pio = io_hash_table_search(io->sector);
+ pio = io_hash_table_search(io->sector, io->device);
if (!pio)
return NULL;
@@ -513,6 +522,66 @@
return -1;
}
+static struct dev_info *lookup_dev(struct trace *trace, struct blk_io_trace *io)
+{
+ u32 dev = io->device;
+ int i;
+ struct dev_info *di = NULL;
+
+ for (i = 0; i < trace->num_devices; i++) {
+ if (trace->devices[i].device == dev) {
+ di = trace->devices + i;
+ goto found;
+ }
+ }
+ i = trace->num_devices++;
+ if (i >= MAX_DEVICES_PER_TRACE) {
+ fprintf(stderr, "Trace contains too many devices (%d)\n", i);
+ exit(1);
+ }
+ di = trace->devices + i;
+ di->device = dev;
+found:
+ return di;
+}
+
+static void map_devices(struct trace *trace)
+{
+ struct dev_info *di;
+ u64 found;
+ u64 map_start = 0;
+ int i;
+
+ first_record(trace);
+ while (1) {
+ if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
+ di = lookup_dev(trace, trace->io);
+ found = trace->io->sector << 9;
+ if (found < di->min)
+ di->min = found;
+
+ found += trace->io->bytes;
+ if (di->max < found)
+ di->max = found;
+ }
+ if (next_record(trace))
+ break;
+ }
+ first_record(trace);
+ for (i = 0; i < trace->num_devices; i++) {
+ di = trace->devices + i;
+ di->map = map_start;
+ map_start += di->max - di->min;
+ }
+}
+
+u64 map_io(struct trace *trace, struct blk_io_trace *io)
+{
+ struct dev_info *di = lookup_dev(trace, io);
+ u64 val = trace->io->sector << 9;
+ return di->map + val - di->min;
+}
+
void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret, u64 *max_bank_ret,
u64 *max_offset_ret)
{
@@ -521,10 +590,13 @@
u64 max_bank = 0;
u64 max_bank_offset = 0;
u64 num_banks = 0;
+
+ map_devices(trace);
+
first_record(trace);
while (1) {
if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
- found = trace->io->sector << 9;
+ found = map_io(trace, trace->io);
if (found < min)
min = found;
@@ -591,7 +663,7 @@
check_io_types(trace);
if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY)) &&
(trace->io->action & BLK_TA_MASK) == __BLK_TA_QUEUE) {
- u64 off = (trace->io->sector << 9) - min_offset;
+ u64 off = map_io(trace, trace->io) - min_offset;
slot = (int)(off / bytes_per_bucket);
hits[slot]++;
@@ -636,33 +708,139 @@
return 0;
}
+static char footer[] = ".blktrace.0";
+static int footer_len = sizeof(footer);
+
+static void match_trace(char *name, char **traces)
+{
+ int match_len;
+ char *match;
+ int footer_start;
+
+ match_len = strlen(name);
+ if (match_len <= footer_len)
+ return;
+
+ footer_start = match_len - footer_len;
+ if (strcmp(name + footer_start + 1, footer) != 0)
+ return;
+
+ match = strdup(name);
+ if (!match)
+ goto enomem;
+
+ match[footer_start + 1] = '\0';
+ snprintf(line, line_len, "%s -i '%s'", *traces ? *traces : "", match);
+ free(match);
+
+ match = strdup(line);
+ if (!match)
+ goto enomem;
+
+ free(*traces);
+ *traces = match;
+ return;
+
+enomem:
+ perror("memory allocation failed");
+ exit(1);
+ return;
+}
+
+static char *combine_blktrace_devs(char *dir_name)
+{
+ DIR *dir;
+ char *traces = NULL;
+ struct dirent *d;
+ int len;
+ int ret;
+
+ dir = opendir(dir_name);
+ if (!dir)
+ return NULL;
+
+ while (1) {
+ d = readdir(dir);
+ if (!d)
+ break;
+
+ len = strlen(d->d_name);
+ if (len > footer_len)
+ match_trace(d->d_name, &traces);
+ }
+
+ closedir(dir);
+
+ if (!traces)
+ return NULL;
+
+ snprintf(line, line_len, "blkparse -O %s -D %s -d '%s.%s'",
+ traces, dir_name, dir_name, "dump");
+
+ ret = system(line);
+ if (ret) {
+ fprintf(stderr, "blkparse failure %s\n", line);
+ exit(1);
+ }
+ snprintf(line, line_len, "%s.%s", dir_name, "dump");
+ return strdup(line);
+}
+
static char *find_trace_file(char *filename)
{
int ret;
struct stat st;
- char line[1024];
char *dot;
char *try;
+ int found_dir = 0;
+ /* look for an exact match of whatever they pass in.
+ * If it is a file, assume it is the dump file.
+ * If a directory, remember that it existed so we
+ * can combine traces in that directory later
+ */
ret = stat(filename, &st);
- if (ret == 0)
- return strdup(filename);
+ if (ret == 0) {
+ if (S_ISREG(st.st_mode))
+ return strdup(filename);
- snprintf(line, 1024, "%s.%s", filename, "dump");
+ if (S_ISDIR(st.st_mode))
+ found_dir = 1;
+ }
+
+ /*
+ * try tacking .dump onto the end and see if that already
+ * has been generated
+ */
+ snprintf(line, line_len, "%s.%s", filename, "dump");
ret = stat(line, &st);
if (ret == 0)
return strdup(line);
+ /*
+ * try to generate the .dump from all the traces in
+ * a single dir.
+ */
+ if (found_dir) {
+ try = combine_blktrace_devs(filename);
+ if (try)
+ return try;
+ }
+
+ /*
+ * try to generate the .dump from all the blktrace
+ * files for a named trace
+ */
try = strdup(filename);
dot = strrchr(try, '.');
if (!dot || strcmp(".dump", dot) != 0) {
if (dot && dot != try)
*dot = '\0';
- snprintf(line, 1024, "%s%s", try, ".blktrace.0");
+ snprintf(line, line_len, "%s%s", try, ".blktrace.0");
ret = stat(line, &st);
if (ret == 0) {
blktrace_to_dump(try);
- snprintf(line, 1024, "%s.%s", try, "dump");
+ snprintf(line, line_len, "%s.%s", try, "dump");
ret = stat(line, &st);
if (ret == 0) {
free(try);
@@ -762,9 +940,11 @@
return __BLK_TA_COMPLETE;
}
-void add_tput(struct trace *trace, struct graph_line_data *gld)
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+ struct graph_line_data *reads_gld)
{
struct blk_io_trace *io = trace->io;
+ struct graph_line_data *gld;
int action = io->action & BLK_TA_MASK;
int seconds;
@@ -774,11 +954,17 @@
if (action != tput_event(trace))
return;
+ if (BLK_DATADIR(io->action) & BLK_TC_READ)
+ gld = reads_gld;
+ else
+ gld = writes_gld;
+
seconds = SECONDS(io->time);
if (seconds > gld->max_seconds)
return;
gld->data[seconds].sum += io->bytes;
+
gld->data[seconds].count = 1;
if (gld->data[seconds].sum > gld->max)
gld->max = gld->data[seconds].sum;
@@ -834,7 +1020,7 @@
if (action != io_event(trace))
return;
- offset = io->sector << 9;
+ offset = map_io(trace, io);
pm = get_pid_map(tf, io->pid);
if (!pm) {
diff --git a/blkparse.h b/blkparse.h
index 0222e5e..d435a87 100644
--- a/blkparse.h
+++ b/blkparse.h
@@ -25,6 +25,15 @@
#define DOUBLE_TO_NANO_ULL(d) ((unsigned long long)((d) * 1000000000))
#define CHECK_MAGIC(t) (((t)->magic & 0xffffff00) == BLK_IO_TRACE_MAGIC)
+struct dev_info {
+ u32 device;
+ u64 min;
+ u64 max;
+ u64 map;
+};
+
+#define MAX_DEVICES_PER_TRACE 64
+
struct trace {
int fd;
u64 len;
@@ -48,6 +57,8 @@
int mpstat_fd;
int mpstat_seconds;
int mpstat_num_cpus;
+ int num_devices;
+ struct dev_info devices[MAX_DEVICES_PER_TRACE];
};
struct trace_file {
@@ -61,9 +72,12 @@
u64 min_offset;
u64 max_offset;
+ char *reads_color;
+ char *writes_color;
char *line_color;
- struct graph_line_data *tput_gld;
+ struct graph_line_data *tput_writes_gld;
+ struct graph_line_data *tput_reads_gld;
struct graph_line_data *iop_gld;
struct graph_line_data *latency_gld;
struct graph_line_data *queue_depth_gld;
@@ -104,7 +118,8 @@
void add_completed_io(struct trace *trace,
struct graph_line_data *latency_gld);
void add_io(struct trace *trace, struct trace_file *tf);
-void add_tput(struct trace *trace, struct graph_line_data *gld);
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+ struct graph_line_data *reads_gld);
void add_pending_io(struct trace *trace, struct graph_line_data *gld);
int next_record(struct trace *trace);
void first_record(struct trace *trace);
diff --git a/main.c b/main.c
index 41b2828..fbd8a88 100644
--- a/main.c
+++ b/main.c
@@ -139,7 +139,8 @@
static char *graph_title = "";
static char *output_filename = "trace.svg";
-static char *blktrace_device = NULL;
+static char *blktrace_devices[MAX_DEVICES_PER_TRACE];
+static int num_blktrace_devices = 0;
static char *blktrace_outfile = "trace";
static char *blktrace_dest_dir = ".";
static char *program_to_run = NULL;
@@ -257,7 +258,8 @@
else
alloc_ptrs = 1;
list_for_each_entry(tf, &all_traces, list) {
- tf->tput_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+ tf->tput_reads_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+ tf->tput_writes_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
@@ -300,6 +302,7 @@
tf->trace = trace;
tf->max_seconds = SECONDS(last_time) + 1;
tf->stop_seconds = SECONDS(last_time) + 1;
+
find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
&max_bank, &max_bank_offset);
filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
@@ -324,13 +327,19 @@
for (i = 0; i < tf->io_plots; i++) {
if (tf->gdd_reads[i]) {
tf->line_color = tf->gdd_reads[i]->color;
- break;
+ tf->reads_color = tf->gdd_reads[i]->color;
}
if (tf->gdd_writes[i]) {
tf->line_color = tf->gdd_writes[i]->color;
- break;
+ tf->writes_color = tf->gdd_writes[i]->color;
}
+ if (tf->writes_color && tf->reads_color)
+ break;
}
+ if (!tf->reads_color)
+ tf->reads_color = tf->line_color;
+ if (!tf->writes_color)
+ tf->writes_color = tf->line_color;
}
}
@@ -351,7 +360,7 @@
first_record(trace);
while (1) {
check_record(trace);
- add_tput(trace, tf->tput_gld);
+ add_tput(trace, tf->tput_writes_gld, tf->tput_reads_gld);
add_iop(trace, tf->iop_gld);
add_io(trace, tf);
add_pending_io(trace, tf->queue_depth_gld);
@@ -663,6 +672,12 @@
pos = label + strlen(label);
for (i = 0; i < tf->io_plots; i++) {
+ if (tf->gdd_writes[i]) {
+ svg_io_graph(plot, tf->gdd_writes[i]);
+ if (io_per_process)
+ strcpy(pos, tf->gdd_writes[i]->label);
+ svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
+ }
if (tf->gdd_reads[i]) {
svg_io_graph(plot, tf->gdd_reads[i]);
if (io_per_process)
@@ -670,12 +685,6 @@
svg_add_legend(plot, label, " Reads", tf->gdd_reads[i]->color);
}
- if (tf->gdd_writes[i]) {
- svg_io_graph(plot, tf->gdd_writes[i]);
- if (io_per_process)
- strcpy(pos, tf->gdd_writes[i]->label);
- svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
- }
}
}
if (plot->add_xlabel)
@@ -694,14 +703,20 @@
if (active_graphs[TPUT_GRAPH_INDEX] == 0)
return;
- if (num_traces > 1)
- svg_alloc_legend(plot, num_traces);
+ svg_alloc_legend(plot, num_traces * 2);
+
list_for_each_entry(tf, &all_traces, list) {
- if (tf->tput_gld->max > max)
- max = tf->tput_gld->max;
+ if (tf->tput_writes_gld->max > max)
+ max = tf->tput_writes_gld->max;
+ if (tf->tput_reads_gld->max > max)
+ max = tf->tput_reads_gld->max;
}
- list_for_each_entry(tf, &all_traces, list)
- tf->tput_gld->max = max;
+ list_for_each_entry(tf, &all_traces, list) {
+ if (tf->tput_writes_gld->max > 0)
+ tf->tput_writes_gld->max = max;
+ if (tf->tput_reads_gld->max > 0)
+ tf->tput_reads_gld->max = max;
+ }
setup_axis(plot);
set_plot_label(plot, "Throughput");
@@ -715,15 +730,20 @@
set_xticks(plot, num_xticks, min_seconds, max_seconds);
list_for_each_entry(tf, &all_traces, list) {
- svg_line_graph(plot, tf->tput_gld, tf->line_color, 0, 0);
- if (num_traces > 1)
- svg_add_legend(plot, tf->label, "", tf->line_color);
+ if (tf->tput_writes_gld->max > 0) {
+ svg_line_graph(plot, tf->tput_writes_gld, tf->writes_color, 0, 0);
+ svg_add_legend(plot, tf->label, "Writes", tf->writes_color);
+ }
+ if (tf->tput_reads_gld->max > 0) {
+ svg_line_graph(plot, tf->tput_reads_gld, tf->reads_color, 0, 0);
+ svg_add_legend(plot, tf->label, "Reads", tf->reads_color);
+ }
}
if (plot->add_xlabel)
set_xlabel(plot, "Time (seconds)");
- if (num_traces > 1)
- svg_write_legend(plot);
+
+ svg_write_legend(plot);
close_plot(plot);
total_graphs_written++;
}
@@ -1300,7 +1320,11 @@
disable_one_graph(optarg);
break;
case 'd':
- blktrace_device = strdup(optarg);
+ if (num_blktrace_devices == MAX_DEVICES_PER_TRACE - 1) {
+ fprintf(stderr, "Too many blktrace devices provided\n");
+ exit(1);
+ }
+ blktrace_devices[num_blktrace_devices++] = strdup(optarg);
break;
case 'D':
blktrace_dest_dir = strdup(optarg);
@@ -1386,11 +1410,11 @@
return 0;
}
-static void dest_mkdir()
+static void dest_mkdir(char *dir)
{
int ret;
- ret = mkdir(blktrace_dest_dir, 0777);
+ ret = mkdir(dir, 0777);
if (ret && errno != EEXIST) {
fprintf(stderr, "failed to mkdir error %s\n", strerror(errno));
exit(errno);
@@ -1440,14 +1464,23 @@
exit(1);
}
- if (blktrace_device) {
+ if (num_blktrace_devices) {
char *path;
- dest_mkdir();
+ dest_mkdir(blktrace_dest_dir);
+ if (num_blktrace_devices > 1) {
+ snprintf(line, line_len, "%s/%s", blktrace_dest_dir,
+ blktrace_outfile);
+ dest_mkdir(line);
+ }
+
path = join_path(blktrace_dest_dir, blktrace_outfile);
- ret = start_blktrace(blktrace_device, blktrace_outfile,
- blktrace_dest_dir);
+ snprintf(line, line_len, "%s.dump", path);
+ unlink(line);
+
+ ret = start_blktrace(blktrace_devices, num_blktrace_devices,
+ blktrace_outfile, blktrace_dest_dir);
if (ret) {
fprintf(stderr, "exiting due to blktrace failure\n");
exit(1);
@@ -1462,7 +1495,6 @@
exit(1);
}
wait_for_tracers();
- blktrace_to_dump(path);
} else {
/* no program specified, just wait for
* blktrace to exit
diff --git a/mpstat.c b/mpstat.c
index f2292b9..022f589 100644
--- a/mpstat.c
+++ b/mpstat.c
@@ -38,8 +38,11 @@
char line[1024];
int line_len = 1024;
-char *record_header = "CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle\n";
-int record_header_len = 0;
+static char record_header[] = "CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle\n";
+static char record_header_v2[] = "CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle\n";
+
+int record_header_len = sizeof(record_header);
+int record_header_v2_len = sizeof(record_header_v2);
static int past_eof(struct trace *trace, char *cur)
{
@@ -67,11 +70,17 @@
{
char *cur = trace->mpstat_cur;
- cur = strstr(cur, record_header);
+ cur = strstr(trace->mpstat_cur, record_header);
+ if (cur) {
+ cur += record_header_len;
+ } else {
+ cur = strstr(trace->mpstat_cur, record_header_v2);
+ if (cur)
+ cur += record_header_v2_len;
+ }
if (!cur)
return NULL;
- cur += record_header_len;
if (past_eof(trace, cur))
return NULL;
trace->mpstat_cur = cur;
@@ -111,8 +120,7 @@
char *cur = trace->mpstat_start;
char *cpu;
char *record;
- int len;
- char *line;
+ int len; char *line;
first_mpstat(trace);
@@ -134,6 +142,7 @@
trace->mpstat_num_cpus = atoi(cur);
first_mpstat(trace);
free(line);
+
return trace->mpstat_num_cpus;
}
@@ -176,7 +185,6 @@
}
snprintf(line, line_len, "%s.mpstat", guess_filename(trace_name));
-
fd = open(line, O_RDONLY);
if (fd < 0)
return 0;
@@ -191,7 +199,6 @@
fprintf(stderr, "Unable to mmap trace file %s, err %s\n", line, strerror(errno));
goto fail_fd;
}
-
trace->mpstat_start = p;
trace->mpstat_len = st.st_size;
trace->mpstat_cur = p;
@@ -211,6 +218,11 @@
/*
* 09:56:26 AM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle
*
+ * or
+ *
+ * 10:18:51 AM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle
+ *
+ *
* this reads just one line in the mpstat
*/
int read_mpstat_event(struct trace *trace, double *user,
diff --git a/plot.c b/plot.c
index 7b4e18c..e5af3ad 100644
--- a/plot.c
+++ b/plot.c
@@ -71,8 +71,10 @@
static char *colors[] = {
"blue", "darkgreen",
- "red", "aqua",
- "orange", "darkviolet",
+ "red",
+ "darkviolet",
+ "orange",
+ "aqua",
"brown", "#00FF00",
"yellow", "coral",
"black", "darkred",
diff --git a/tracers.c b/tracers.c
index a995e11..433b530 100644
--- a/tracers.c
+++ b/tracers.c
@@ -42,16 +42,14 @@
static pid_t blktrace_pid = 0;
static pid_t mpstat_pid = 0;
-char *blktrace_args[] = {
+char *blktrace_args[17 + MAX_DEVICES_PER_TRACE * 2] = {
"blktrace",
- "-d", NULL,
- "-b", "16384",
- "-o", "trace",
- "-D", ".",
+ "-b", "8192",
"-a", "queue",
"-a", "complete",
"-a", "issue",
"-a", "notify",
+ "-D", ".",
NULL,
};
@@ -61,9 +59,8 @@
NULL,
};
-#define DEVICE_INDEX 2
-#define DEST_DIR_INDEX 8
-#define TRACE_NAME_INDEX 6
+#define DEST_DIR_INDEX 12
+#define LAST_ARG 13
int stop_tracer(pid_t *tracer_pid)
{
@@ -103,18 +100,42 @@
}
-int start_blktrace(char *device, char *trace_name, char *dest)
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest)
{
pid_t pid;
int ret;
char **arg = blktrace_args;
- blktrace_args[DEVICE_INDEX] = device;
+ int i;
+ int arg_index;
fprintf(stderr, "running blktrace");
- if (dest)
- blktrace_args[DEST_DIR_INDEX] = dest;
- if (trace_name)
- blktrace_args[TRACE_NAME_INDEX] = trace_name;
+ if (!trace_name)
+ trace_name = "trace";
+
+ arg_index = LAST_ARG;
+ for (i = 0; i < num_devices; i++) {
+ blktrace_args[arg_index++] = "-d";
+ blktrace_args[arg_index++] = devices[i];
+ }
+
+ /*
+ * single device traces use -o and are put into
+ * the dest dir if provided
+ */
+ if (num_devices == 1) {
+ blktrace_args[arg_index++] = "-o";
+ blktrace_args[arg_index++] = trace_name;
+ if (dest)
+ blktrace_args[DEST_DIR_INDEX] = dest;
+ } else {
+ /*
+ * multi device traces are put into a dest
+ * dir based on the trace name
+ */
+ blktrace_args[DEST_DIR_INDEX] = trace_name;
+ }
+
+ blktrace_args[arg_index] = NULL;
while(*arg) {
fprintf(stderr, " %s", *arg);
diff --git a/tracers.h b/tracers.h
index 7ceeefe..81a2fe7 100644
--- a/tracers.h
+++ b/tracers.h
@@ -19,7 +19,7 @@
#define __IOWATCH_TRACERS
int run_program(char *str);
int stop_blktrace(void);
-int start_blktrace(char *device, char *trace_name, char *dest);
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest);
int start_mpstat(char *trace_name);
int wait_for_tracers(void);
int blktrace_to_dump(char *trace_name);
