Merge branch 'clrkwllms/work' into rt-tests-0.84-devel
Signed-off-by: John Kacur <jkacur@redhat.com>
diff --git a/.gitignore b/.gitignore
index 047925c..1924e73 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,8 @@
 *.o
 *.tar.gz
 *.d
+*.patch
+*.a
 tmp
 patches
 ChangeLog
diff --git a/Makefile b/Makefile
index 3a82407..33e8d60 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,8 @@
 NUMA 	:= 1
 endif
 
-CFLAGS ?= -D_GNU_SOURCE -Wall -Wno-nonnull -Isrc/include
+CFLAGS ?= -Wall -Wno-nonnull
+CPPFLAGS += -D_GNU_SOURCE -Isrc/include
 LDFLAGS ?=
 
 PYLIB  := $(shell python -c 'import distutils.sysconfig;  print distutils.sysconfig.get_python_lib()')
@@ -49,11 +50,11 @@
 VPATH	+= src/hackbench
 
 %.o: %.c
-	$(CC) -D VERSION_STRING=$(VERSION_STRING) -c $< $(CFLAGS)
+	$(CC) -D VERSION_STRING=$(VERSION_STRING) -c $< $(CFLAGS) $(CPPFLAGS)
 
 # Pattern rule to generate dependency files from .c files
 %.d: %.c
-	@$(CC) -MM $(CFLAGS) $< | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' > $@ || rm -f $@
+	@$(CC) -MM $(CFLAGS) $(CPPFLAGS) $< | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' > $@ || rm -f $@
 
 .PHONY: all
 all: $(TARGETS) hwlatdetect
diff --git a/src/cyclictest/cyclictest.c b/src/cyclictest/cyclictest.c
index 523c70e..abf3e8b 100644
--- a/src/cyclictest/cyclictest.c
+++ b/src/cyclictest/cyclictest.c
@@ -150,6 +150,7 @@
 	double avg;
 	long *values;
 	long *hist_array;
+        long *outliers;
 	pthread_t thread;
 	int threadstarted;
 	int tid;
@@ -157,6 +158,7 @@
 	long redmax;
 	long cycleofmax;
 	long hist_overflow;
+        long num_outliers;
 };
 
 static int shutdown;
@@ -174,6 +176,8 @@
 static int refresh_on_max;
 static int force_sched_other;
 static int priospread = 0;
+static int check_clock_resolution;
+static int ct_debug;
 
 static pthread_cond_t refresh_on_max_cond = PTHREAD_COND_INITIALIZER;
 static pthread_mutex_t refresh_on_max_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -200,12 +204,12 @@
 
 /* Latency trick
  * if the file /dev/cpu_dma_latency exists,
- * open it and write a zero into it. This will tell 
- * the power management system not to transition to 
+ * open it and write a zero into it. This will tell
+ * the power management system not to transition to
  * a high cstate (in fact, the system acts like idle=poll)
  * When the fd to /dev/cpu_dma_latency is closed, the behavior
  * goes back to the system default.
- * 
+ *
  * Documentation/power/pm_qos_interface.txt
  */
 static void set_latency_target(void)
@@ -402,7 +406,7 @@
 		switch (kernelversion) {
 		case KV_26_LT18: gettimeofday(0,(struct timezone *)1); break;
 		case KV_26_LT24: prctl(0, 1); break;
-		case KV_26_33: 
+		case KV_26_33:
 		case KV_30:
 			write(trace_fd, "1", 1);
 			break;
@@ -412,7 +416,7 @@
 		switch (kernelversion) {
 		case KV_26_LT18: gettimeofday(0,0); break;
 		case KV_26_LT24: prctl(0, 0); break;
-		case KV_26_33: 
+		case KV_26_33:
 		case KV_30:
 			write(trace_fd, "0", 1);
 			break;
@@ -750,7 +754,7 @@
 
 	memset(&schedp, 0, sizeof(schedp));
 	schedp.sched_priority = par->prio;
-	if (setscheduler(0, par->policy, &schedp)) 
+	if (setscheduler(0, par->policy, &schedp))
 		fatal("timerthread%d: failed to set priority to %d\n", par->cpu, par->prio);
 
 	/* Get current time */
@@ -881,8 +885,11 @@
 
 		/* Update the histogram */
 		if (histogram) {
-			if (diff >= histogram)
+			if (diff >= histogram) {
 				stat->hist_overflow++;
+                                if (stat->num_outliers < histogram)
+					stat->outliers[stat->num_outliers++] = stat->cycles;
+			}
 			else
 				stat->hist_array[diff]++;
 		}
@@ -938,7 +945,7 @@
 		if (kernvar(O_RDONLY, "available_tracers", tracers, sizeof(tracers)))
 			strcpy(tracers, "none");
 	}
-		
+
 	printf("cyclictest V %1.2f\n", VERSION_STRING);
 	printf("Usage:\n"
 	       "cyclictest <options>\n\n"
@@ -954,17 +961,18 @@
 	       "-D       --duration=t      specify a length for the test run\n"
 	       "                           default is in seconds, but 'm', 'h', or 'd' maybe added\n"
 	       "                           to modify value to minutes, hours or days\n"
+	       "-e       --latency=PM_QOS  write PM_QOS to /dev/cpu_dma_latency\n"
 	       "-E       --event           event tracing (used with -b)\n"
 	       "-f       --ftrace          function trace (when -b is active)\n"
 	       "-h       --histogram=US    dump a latency histogram to stdout after the run\n"
-               "                           (with same priority about many threads)\n"
+	       "                           (with same priority about many threads)\n"
 	       "                           US is the max time to be be tracked in microseconds\n"
 	       "-H       --histofall=US    same as -h except with an additional summary column\n"
 	       "-i INTV  --interval=INTV   base interval of thread in us default=1000\n"
 	       "-I       --irqsoff         Irqsoff tracing (used with -b)\n"
 	       "-l LOOPS --loops=LOOPS     number of loops: default=0(endless)\n"
 	       "-m       --mlockall        lock current and future memory allocations\n"
-	       "-M       --refresh_on_max  delay updating the screen until a new max latency is hit\n" 
+	       "-M       --refresh_on_max  delay updating the screen until a new max latency is hit\n"
 	       "-n       --nanosleep       use clock_nanosleep\n"
 	       "-N       --nsecs           print results in ns instead of us (default us)\n"
 	       "-o RED   --oscope=RED      oscilloscope mode, reduce verbose output by RED\n"
@@ -974,7 +982,12 @@
 	       "-q       --quiet           print only a summary on exit\n"
 	       "-Q       --priospread       spread priority levels starting at specified value\n"
 	       "-r       --relative        use relative timer instead of absolute\n"
+	       "-R       --resolution      check clock resolution, calling clock_gettime() many\n"
+	       "                           times.  list of clock_gettime() values will be\n"
+	       "                           reported with -X\n"
 	       "-s       --system          use sys_nanosleep and sys_setitimer\n"
+	       "-S       --smp             Standard SMP testing: options -a -t -n and\n"
+	       "                           same priority of all threads\n"
 	       "-t       --threads         one thread per available processor\n"
 	       "-t [NUM] --threads=NUM     number of threads:\n"
 	       "                           without NUM, threads = max_cpus\n"
@@ -982,16 +995,15 @@
 	       "-T TRACE --tracer=TRACER   set tracing function\n"
 	       "    configured tracers: %s\n"
 	       "-u       --unbuffered      force unbuffered output for live processing\n"
+	       "-U       --numa            Standard NUMA testing (similar to SMP option)\n"
+	       "                           thread data structures allocated from local node\n"
 	       "-v       --verbose         output values on stdout for statistics\n"
 	       "                           format: n:c:v n=tasknum c=count v=value in us\n"
-               "-w       --wakeup          task wakeup tracing (used with -b)\n"
-               "-W       --wakeuprt        rt task wakeup tracing (used with -b)\n"
-               "-y POLI  --policy=POLI     policy of realtime thread, POLI may be fifo(default) or rr\n"
-               "                           format: --policy=fifo(default) or --policy=rr\n"
-	       "-S       --smp             Standard SMP testing: options -a -t -n and\n"
-               "                           same priority of all threads\n"
-	       "-U       --numa            Standard NUMA testing (similar to SMP option)\n"
-               "                           thread data structures allocated from local node\n",
+	       "-w       --wakeup          task wakeup tracing (used with -b)\n"
+	       "-W       --wakeuprt        rt task wakeup tracing (used with -b)\n"
+	       "-X       --dbg_cyclictest  print info useful for debugging cyclictest\n"
+	       "-y POLI  --policy=POLI     policy of realtime thread, POLI may be fifo(default) or rr\n"
+	       "                           format: --policy=fifo(default) or --policy=rr\n",
 	       tracers
 		);
 	if (error)
@@ -1075,48 +1087,53 @@
 
 	for (;;) {
  		int option_index = 0;
-		/** Options for getopt */
+		/*
+		 * Options for getopt
+		 * Ordered alphabetically by single letter name
+		 */
 		static struct option long_options[] = {
-			{"affinity", optional_argument, NULL, 'a'},
-			{"breaktrace", required_argument, NULL, 'b'},
-			{"preemptirqs", no_argument, NULL, 'B'},
-			{"clock", required_argument, NULL, 'c'},
-			{"context", no_argument, NULL, 'C'},
-			{"distance", required_argument, NULL, 'd'},
-			{"event", no_argument, NULL, 'E'},
-			{"ftrace", no_argument, NULL, 'f'},
-			{"histogram", required_argument, NULL, 'h'},
-			{"histofall", required_argument, NULL, 'H'},
-			{"interval", required_argument, NULL, 'i'},
-			{"irqsoff", no_argument, NULL, 'I'},
-			{"loops", required_argument, NULL, 'l'},
-			{"mlockall", no_argument, NULL, 'm' },
-			{"refresh_on_max", no_argument, NULL, 'M' },
-			{"nanosleep", no_argument, NULL, 'n'},
-			{"nsecs", no_argument, NULL, 'N'},
-			{"oscope", required_argument, NULL, 'o'},
-			{"priority", required_argument, NULL, 'p'},
-                        {"policy", required_argument, NULL, 'y'},
-			{"preemptoff", no_argument, NULL, 'P'},
-			{"quiet", no_argument, NULL, 'q'},
-			{"relative", no_argument, NULL, 'r'},
-			{"system", no_argument, NULL, 's'},
-			{"threads", optional_argument, NULL, 't'},
-			{"unbuffered", no_argument, NULL, 'u'},
-			{"verbose", no_argument, NULL, 'v'},
-			{"duration",required_argument, NULL, 'D'},
-                        {"wakeup", no_argument, NULL, 'w'},
-                        {"wakeuprt", no_argument, NULL, 'W'},
-			{"help", no_argument, NULL, '?'},
-			{"tracer", required_argument, NULL, 'T'},
-			{"traceopt", required_argument, NULL, 'O'},
-			{"smp", no_argument, NULL, 'S'},
-			{"numa", no_argument, NULL, 'U'},
-			{"latency", required_argument, NULL, 'e'},
-			{"priospread", no_argument, NULL, 'Q'},
+			{"affinity",         optional_argument, NULL, 'a'},
+			{"breaktrace",       required_argument, NULL, 'b'},
+			{"preemptirqs",      no_argument,       NULL, 'B'},
+			{"clock",            required_argument, NULL, 'c'},
+			{"context",          no_argument,       NULL, 'C'},
+			{"distance",         required_argument, NULL, 'd'},
+			{"duration",         required_argument, NULL, 'D'},
+			{"latency",          required_argument, NULL, 'e'},
+			{"event",            no_argument,       NULL, 'E'},
+			{"ftrace",           no_argument,       NULL, 'f'},
+			{"histogram",        required_argument, NULL, 'h'},
+			{"histofall",        required_argument, NULL, 'H'},
+			{"interval",         required_argument, NULL, 'i'},
+			{"irqsoff",          no_argument,       NULL, 'I'},
+			{"loops",            required_argument, NULL, 'l'},
+			{"mlockall",         no_argument,       NULL, 'm'},
+			{"refresh_on_max",   no_argument,       NULL, 'M'},
+			{"nanosleep",        no_argument,       NULL, 'n'},
+			{"nsecs",            no_argument,       NULL, 'N'},
+			{"oscope",           required_argument, NULL, 'o'},
+			{"traceopt",         required_argument, NULL, 'O'},
+			{"priority",         required_argument, NULL, 'p'},
+			{"preemptoff",       no_argument,       NULL, 'P'},
+			{"quiet",            no_argument,       NULL, 'q'},
+			{"priospread",       no_argument,       NULL, 'Q'},
+			{"relative",         no_argument,       NULL, 'r'},
+			{"resolution",       no_argument,       NULL, 'R'},
+			{"system",           no_argument,       NULL, 's'},
+			{"smp",              no_argument,       NULL, 'S'},
+			{"threads",          optional_argument, NULL, 't'},
+			{"tracer",           required_argument, NULL, 'T'},
+			{"unbuffered",       no_argument,       NULL, 'u'},
+			{"numa",             no_argument,       NULL, 'U'},
+			{"verbose",          no_argument,       NULL, 'v'},
+			{"wakeup",           no_argument,       NULL, 'w'},
+			{"wakeuprt",         no_argument,       NULL, 'W'},
+			{"dbg_cyclictest",   no_argument,       NULL, 'X'},
+			{"policy",           required_argument, NULL, 'y'},
+			{"help",             no_argument,       NULL, '?'},
 			{NULL, 0, NULL, 0}
 		};
-		int c = getopt_long(argc, argv, "a::b:Bc:Cd:Efh:H:i:Il:MnNo:O:p:PmqQrsSt::uUvD:wWT:y:e:",
+		int c = getopt_long(argc, argv, "a::b:Bc:Cd:D:e:Efh:H:i:Il:MnNo:O:p:PmqQrRsSt::uUvD:wWXT:y:",
 				    long_options, &option_index);
 		if (c == -1)
 			break;
@@ -1140,6 +1157,13 @@
 		case 'c': clocksel = atoi(optarg); break;
 		case 'C': tracetype = CTXTSWITCH; break;
 		case 'd': distance = atoi(optarg); break;
+		case 'D': duration = parse_time_string(optarg); break;
+		case 'e': /* power management latency target value */
+			  /* note: default is 0 (zero) */
+			latency_target_value = atoi(optarg);
+			if (latency_target_value < 0)
+				latency_target_value = 0;
+			break;
 		case 'E': enable_events = 1; break;
 		case 'f': tracetype = FUNCTION; ftrace = 1; break;
 		case 'H': histofall = 1; /* fall through */
@@ -1155,12 +1179,14 @@
 			}
 			break;
 		case 'l': max_cycles = atoi(optarg); break;
+		case 'm': lockall = 1; break;
+		case 'M': refresh_on_max = 1; break;
 		case 'n': use_nanosleep = MODE_CLOCK_NANOSLEEP; break;
 		case 'N': use_nsecs = 1; break;
 		case 'o': oscope_reduction = atoi(optarg); break;
 		case 'O': traceopt(optarg); break;
-		case 'p': 
-			priority = atoi(optarg); 
+		case 'p':
+			priority = atoi(optarg);
 			if (policy != SCHED_FIFO && policy != SCHED_RR)
 				policy = SCHED_FIFO;
 			break;
@@ -1176,7 +1202,16 @@
 		case 'q': quiet = 1; break;
 		case 'Q': priospread = 1; break;
 		case 'r': timermode = TIMER_RELTIME; break;
+		case 'R': check_clock_resolution = 1; break;
 		case 's': use_system = MODE_SYS_OFFSET; break;
+		case 'S':  /* SMP testing */
+			if (numa)
+				fatal("numa and smp options are mutually exclusive\n");
+			smp = 1;
+			num_threads = max_cpus;
+			setaffinity = AFFINITY_USEALL;
+			use_nanosleep = MODE_CLOCK_NANOSLEEP;
+			break;
 		case 't':
 			if (smp) {
 				warn("-t ignored due to --smp\n");
@@ -1189,27 +1224,11 @@
 			else
 				num_threads = max_cpus;
 			break;
-		case 'T': 
+		case 'T':
 			tracetype = CUSTOM;
-			strncpy(tracer, optarg, sizeof(tracer)); 
+			strncpy(tracer, optarg, sizeof(tracer));
 			break;
 		case 'u': setvbuf(stdout, NULL, _IONBF, 0); break;
-		case 'v': verbose = 1; break;
-		case 'm': lockall = 1; break;
-		case 'M': refresh_on_max = 1; break;
-		case 'D': duration = parse_time_string(optarg);
-			break;
-                case 'w': tracetype = WAKEUP; break;
-                case 'W': tracetype = WAKEUPRT; break;
-                case 'y': handlepolicy(optarg); break;
-		case 'S':  /* SMP testing */
-			if (numa)
-				fatal("numa and smp options are mutually exclusive\n");
-			smp = 1;
-			num_threads = max_cpus;
-			setaffinity = AFFINITY_USEALL;
-			use_nanosleep = MODE_CLOCK_NANOSLEEP;
-			break;
 		case 'U':  /* NUMA testing */
 			if (smp)
 				fatal("numa and smp options are mutually exclusive\n");
@@ -1225,12 +1244,11 @@
 			warn("ignoring --numa or -U\n");
 #endif
 			break;
-		case 'e': /* power management latency target value */
-			  /* note: default is 0 (zero) */
-			latency_target_value = atoi(optarg);
-			if (latency_target_value < 0)
-				latency_target_value = 0;
-			break;
+		case 'v': verbose = 1; break;
+		case 'w': tracetype = WAKEUP; break;
+		case 'W': tracetype = WAKEUPRT; break;
+		case 'X': ct_debug = 1; break;
+		case 'y': handlepolicy(optarg); break;
 
 		case '?': display_help(0); break;
 		}
@@ -1281,7 +1299,7 @@
 		error = 1;
 
 	if (priospread && priority == 0) {
-		fprintf(stderr, "defaulting realtime priority to %d\n", 
+		fprintf(stderr, "defaulting realtime priority to %d\n",
 			num_threads+1);
 		priority = num_threads+1;
 	}
@@ -1292,7 +1310,7 @@
 	}
 
 	if ((policy == SCHED_FIFO || policy == SCHED_RR) && priority == 0) {
-		fprintf(stderr, "defaulting realtime priority to %d\n", 
+		fprintf(stderr, "defaulting realtime priority to %d\n",
 			num_threads+1);
 		priority = num_threads+1;
 	}
@@ -1334,7 +1352,7 @@
 		kv = KV_30;
 		strcpy(functiontracer, "function");
 		strcpy(traceroptions, "trace_options");
-		
+
 	} else
 		kv = KV_NOT_SUPPORTED;
 
@@ -1416,7 +1434,7 @@
 	printf("# Max Latencies:");
 	maxmax = 0;
 	for (j = 0; j < nthreads; j++) {
- 		printf(" %05lu", par[j]->stats->max);
+		printf(" %05lu", par[j]->stats->max);
 		if (par[j]->stats->max > maxmax)
 			maxmax = par[j]->stats->max;
 	}
@@ -1426,12 +1444,23 @@
 	printf("# Histogram Overflows:");
 	alloverflows = 0;
 	for (j = 0; j < nthreads; j++) {
- 		printf(" %05lu", par[j]->stats->hist_overflow);
+		printf(" %05lu", par[j]->stats->hist_overflow);
 		alloverflows += par[j]->stats->hist_overflow;
 	}
 	if (histofall && nthreads > 1)
 		printf(" %05lu", alloverflows);
 	printf("\n");
+
+	printf("# Histogram Overflow at cycle number:\n");
+	for (i = 0; i < nthreads; i++) {
+		printf("# Thread %d:", i);
+		for (j = 0; j < par[i]->stats->num_outliers; j++)
+			printf(" %05lu", par[i]->stats->outliers[j]);
+		if (par[i]->stats->num_outliers < par[i]->stats->hist_overflow)
+			printf(" # %05lu others", par[i]->stats->hist_overflow - par[i]->stats->num_outliers);
+		printf("\n");
+	}
+	printf("\n");
 }
 
 static void print_stat(struct thread_param *par, int index, int verbose)
@@ -1447,7 +1476,7 @@
 			else
                                 fmt = "T:%2d (%5d) P:%2d I:%ld C:%7lu "
 					"Min:%7ld Act:%5ld Avg:%5ld Max:%8ld\n";
-                        printf(fmt, index, stat->tid, par->prio, 
+                        printf(fmt, index, stat->tid, par->prio,
                                par->interval, stat->cycles, stat->min, stat->act,
 			       stat->cycles ?
 			       (long)(stat->avg/stat->cycles) : 0, stat->max);
@@ -1511,6 +1540,103 @@
 	if (check_timer())
 		warn("High resolution timers not available\n");
 
+	if (check_clock_resolution) {
+		int clock;
+		uint64_t diff;
+		int k;
+		uint64_t min_non_zero_diff = UINT64_MAX;
+		struct timespec now;
+		struct timespec prev;
+		uint64_t reported_resolution = UINT64_MAX;
+		struct timespec res;
+		struct timespec *time;
+		int times;
+
+		clock = clocksources[clocksel];
+
+		if (clock_getres(clock, &res)) {
+			warn("clock_getres failed");
+		} else {
+			reported_resolution = (NSEC_PER_SEC * res.tv_sec) + res.tv_nsec;
+		}
+
+
+		/*
+		 * Calculate how many calls to clock_gettime are needed.
+		 * Then call it that many times.
+		 * Goal is to collect timestamps for ~ 0.001 sec.
+		 * This will reliably capture resolution <= 500 usec.
+		 */
+		times = 1000;
+		clock_gettime(clock, &prev);
+		for (k=0; k < times; k++) {
+			clock_gettime(clock, &now);
+		}
+
+		diff = calcdiff_ns(now, prev);
+		if (diff == 0) {
+			/*
+			 * No clock rollover occurred.
+			 * Use the default value for times.
+			 */
+			times = -1;
+		} else {
+			int call_time;
+			call_time = diff / times;         /* duration 1 call */
+			times = NSEC_PER_SEC / call_time; /* calls per second */
+			times /= 1000;                    /* calls per msec */
+			if (times < 1000)
+				times = 1000;
+		}
+		/* sanity check */
+		if ((times <= 0) || (times > 100000))
+			times = 100000;
+
+		time = calloc(times, sizeof(*time));
+
+		for (k=0; k < times; k++) {
+			clock_gettime(clock, &time[k]);
+		}
+
+		if (ct_debug) {
+			info("For %d consecutive calls to clock_gettime():\n", times);
+			info("time, delta time (nsec)\n");
+		}
+
+		prev = time[0];
+		for (k=1; k < times; k++) {
+
+			diff = calcdiff_ns(time[k], prev);
+			prev = time[k];
+
+			if (diff && (diff < min_non_zero_diff)) {
+				min_non_zero_diff = diff;
+			}
+
+			if (ct_debug)
+				info("%ld.%06ld  %5llu\n",
+				     time[k].tv_sec, time[k].tv_nsec,
+				     (unsigned long long)diff);
+		}
+
+		free(time);
+
+
+		if (verbose ||
+		    (min_non_zero_diff && (min_non_zero_diff > reported_resolution))) {
+			/*
+			 * Measured clock resolution includes the time to call
+			 * clock_gettime(), so it will be slightly larger than
+			 * actual resolution.
+			 */
+			warn("reported clock resolution: %llu nsec\n",
+			     (unsigned long long)reported_resolution);
+			warn("measured clock resolution approximately: %llu nsec\n",
+			     (unsigned long long)min_non_zero_diff);
+		}
+
+	}
+
 	mode = use_nanosleep + use_system;
 
 	sigemptyset(&sigset);
@@ -1580,10 +1706,12 @@
 			int bufsize = histogram * sizeof(long);
 
 			stat->hist_array = threadalloc(bufsize, node);
-			if (stat->hist_array == NULL)
+			stat->outliers = threadalloc(bufsize, node);
+			if (stat->hist_array == NULL || stat->outliers == NULL)
 				fatal("failed to allocate histogram of size %d on node %d\n",
 				      histogram, i);
 			memset(stat->hist_array, 0, bufsize);
+			memset(stat->outliers, 0, bufsize);
 		}
 
 		if (verbose) {
@@ -1699,8 +1827,10 @@
 
 	if (histogram) {
 		print_hist(parameters, num_threads);
-		for (i = 0; i < num_threads; i++)
+		for (i = 0; i < num_threads; i++) {
 			threadfree(statistics[i]->hist_array, histogram*sizeof(long), parameters[i]->node);
+			threadfree(statistics[i]->outliers, histogram*sizeof(long), parameters[i]->node);
+		}
 	}
 
 	if (tracelimit) {
@@ -1710,7 +1840,7 @@
 			printf("# Break value: %llu\n", (unsigned long long)break_thread_value);
 		}
 	}
-	
+
 
 	for (i=0; i < num_threads; i++) {
 		if (!statistics[i])
diff --git a/src/pmqtest/pmqtest.c b/src/pmqtest/pmqtest.c
index b811d95..336a8eb 100644
--- a/src/pmqtest/pmqtest.c
+++ b/src/pmqtest/pmqtest.c
@@ -512,12 +512,12 @@
 					(int) ((receiver[i].sumdiff / receiver[i].samples) + 0.5),
 					receiver[i].maxdiff);
 				if (receiver[i].error[0] != '\0') {
-					printf(receiver[i].error);
+					printf("%s", receiver[i].error);
 					errorlines++;
 					receiver[i].error[0] = '\0';
 				}
 				if (sender[i].error[0] != '\0') {
-					printf(sender[i].error);
+					printf("%s", sender[i].error);
 					errorlines++;
 					receiver[i].error[0] = '\0';
 				}
diff --git a/src/ptsematest/ptsematest.c b/src/ptsematest/ptsematest.c
index 92f0ab5..7558a41 100644
--- a/src/ptsematest/ptsematest.c
+++ b/src/ptsematest/ptsematest.c
@@ -389,12 +389,12 @@
 					(int) ((receiver[i].sumdiff / receiver[i].samples) + 0.5),
 					receiver[i].maxdiff);
 				if (receiver[i].error[0] != '\0') {
-					printf(receiver[i].error);
+					printf("%s", receiver[i].error);
 					errorlines++;
 					receiver[i].error[0] = '\0';
 				}
 				if (sender[i].error[0] != '\0') {
-					printf(sender[i].error);
+					printf("%s", sender[i].error);
 					errorlines++;
 					receiver[i].error[0] = '\0';
 				}
diff --git a/src/sigwaittest/sigwaittest.c b/src/sigwaittest/sigwaittest.c
index 1948d76..428f5ce 100644
--- a/src/sigwaittest/sigwaittest.c
+++ b/src/sigwaittest/sigwaittest.c
@@ -561,12 +561,12 @@
 					    receiver[i].samples) + 0.5),
 					    receiver[i].maxdiff);
 				if (receiver[i].error[0] != '\0') {
-					printf(receiver[i].error);
+					printf("%s", receiver[i].error);
 					receiver[i].error[0] = '\0';
 					errorlines++;
 				}
 				if (sender[i].error[0] != '\0') {
-					printf(sender[i].error);
+					printf("%s", sender[i].error);
 					sender[i].error[0] = '\0';
 					errorlines++;
 				}
diff --git a/src/svsematest/svsematest.c b/src/svsematest/svsematest.c
index 89473a5..c1128cc 100644
--- a/src/svsematest/svsematest.c
+++ b/src/svsematest/svsematest.c
@@ -646,12 +646,12 @@
 					    receiver[i].samples) + 0.5),
 					    receiver[i].maxdiff);
 				if (receiver[i].error[0] != '\0') {
-					printf(receiver[i].error);
+					printf("%s", receiver[i].error);
 					receiver[i].error[0] = '\0';
 					errorlines++;
 				}
 				if (sender[i].error[0] != '\0') {
-					printf(sender[i].error);
+					printf("%s", sender[i].error);
 					sender[i].error[0] = '\0';
 					errorlines++;
 				}