diff --git a/backend.c b/backend.c
index d2b97db..2e6a377 100644
--- a/backend.c
+++ b/backend.c
@@ -62,6 +62,8 @@
 
 int groupid = 0;
 unsigned int thread_number = 0;
+unsigned int nr_segments = 0;
+unsigned int cur_segment = 0;
 unsigned int stat_number = 0;
 int temp_stall_ts;
 unsigned long done_secs = 0;
@@ -75,7 +77,7 @@
 
 static void sig_int(int sig)
 {
-	if (segments[0].threads) {
+	if (nr_segments) {
 		if (is_backend)
 			fio_server_got_signal(sig);
 		else {
diff --git a/fio.h b/fio.h
index 691976a..fffec00 100644
--- a/fio.h
+++ b/fio.h
@@ -470,6 +470,7 @@
 struct thread_segment {
 	struct thread_data *threads;
 	int shm_id;
+	int nr_threads;
 };
 
 /*
@@ -522,6 +523,8 @@
 extern bool exitall_on_terminate;
 extern unsigned int thread_number;
 extern unsigned int stat_number;
+extern unsigned int nr_segments;
+extern unsigned int cur_segment;
 extern int groupid;
 extern int output_format;
 extern int append_terse_output;
@@ -552,6 +555,14 @@
 
 extern struct thread_segment segments[REAL_MAX_SEG];
 
+static inline struct thread_data *tnumber_to_td(unsigned int tnumber)
+{
+	struct thread_segment *seg;
+
+	seg = &segments[tnumber / JOBS_PER_SEG];
+	return &seg->threads[tnumber & (JOBS_PER_SEG - 1)];
+}
+
 static inline bool is_running_backend(void)
 {
 	return is_backend || is_local_backend;
@@ -715,7 +726,7 @@
  * Iterates all threads/processes within all the defined jobs
  */
 #define for_each_td(td, i)	\
-	for ((i) = 0, (td) = &segments[0].threads[0]; (i) < (int) thread_number; (i)++, (td)++)
+	for ((i) = 0, (td) = &segments[0].threads[0]; (i) < (int) thread_number; (i)++, (td) = tnumber_to_td((i)))
 #define for_each_file(td, f, i)	\
 	if ((td)->files_index)						\
 		for ((i) = 0, (f) = (td)->files[0];			\
diff --git a/gettime-thread.c b/gettime-thread.c
index 9b82e53..86c2e2e 100644
--- a/gettime-thread.c
+++ b/gettime-thread.c
@@ -58,7 +58,7 @@
 	 * but I'm not sure what to use outside of a simple CPU nop to relax
 	 * it - we don't want to lose precision.
 	 */
-	while (segments[0].threads) {
+	while (nr_segments) {
 		fio_gtod_update();
 		nop;
 	}
diff --git a/init.c b/init.c
index 36b10ea..c344284 100644
--- a/init.c
+++ b/init.c
@@ -301,25 +301,34 @@
 
 void free_threads_shm(void)
 {
-	if (segments[0].threads) {
-		void *tp = segments[0].threads;
-#ifndef CONFIG_NO_SHM
-		struct shmid_ds sbuf;
+	int i;
 
-		segments[0].threads = NULL;
-		shmdt(tp);
-		shmctl(segments[0].shm_id, IPC_RMID, &sbuf);
-		segments[0].shm_id = -1;
+	for (i = 0; i < nr_segments; i++) {
+		struct thread_segment *seg = &segments[i];
+
+		if (seg->threads) {
+			void *tp = seg->threads;
+#ifndef CONFIG_NO_SHM
+			struct shmid_ds sbuf;
+
+			seg->threads = NULL;
+			shmdt(tp);
+			shmctl(seg->shm_id, IPC_RMID, &sbuf);
+			seg->shm_id = -1;
 #else
-		segments[0].threads = NULL;
-		free(tp);
+			seg->threads = NULL;
+			free(tp);
 #endif
+		}
 	}
+
+	nr_segments = 0;
+	cur_segment = 0;
 }
 
 static void free_shm(void)
 {
-	if (segments[0].threads) {
+	if (nr_segments) {
 		flow_exit();
 		fio_debug_jobp = NULL;
 		fio_warned = NULL;
@@ -337,50 +346,31 @@
 	scleanup();
 }
 
-/*
- * The thread area is shared between the main process and the job
- * threads/processes. So setup a shared memory segment that will hold
- * all the job info. We use the end of the region for keeping track of
- * open files across jobs, for file sharing.
- */
-static int setup_thread_area(void)
+static int add_thread_segment(void)
 {
-	struct thread_segment *seg = &segments[0];
+	struct thread_segment *seg = &segments[nr_segments];
+	size_t size = JOBS_PER_SEG * sizeof(struct thread_data);
 	int i;
 
-	if (seg->threads)
-		return 0;
+	if (nr_segments + 1 >= REAL_MAX_SEG)
+		return -1;
 
-	/*
-	 * 1024 is too much on some machines, scale max_jobs if
-	 * we get a failure that looks like too large a shm segment
-	 */
-	do {
-		size_t size = max_jobs * sizeof(struct thread_data);
-
-		size += 2 * sizeof(unsigned int);
+	size += 2 * sizeof(unsigned int);
 
 #ifndef CONFIG_NO_SHM
-		seg->shm_id = shmget(0, size, IPC_CREAT | 0600);
-		if (seg->shm_id != -1)
-			break;
-		if (errno != EINVAL && errno != ENOMEM && errno != ENOSPC) {
+	seg->shm_id = shmget(0, size, IPC_CREAT | 0600);
+	if (seg->shm_id == -1) {
+		if (errno != EINVAL && errno != ENOMEM && errno != ENOSPC)
 			perror("shmget");
-			break;
-		}
+		return -1;
+	}
 #else
-		seg->threads = malloc(size);
-		if (seg->threads)
-			break;
+	seg->threads = malloc(size);
+	if (!seg->threads)
+		return -1;
 #endif
 
-		max_jobs >>= 1;
-	} while (max_jobs);
-
 #ifndef CONFIG_NO_SHM
-	if (seg->shm_id == -1)
-		return 1;
-
 	seg->threads = shmat(seg->shm_id, NULL, 0);
 	if (seg->threads == (void *) -1) {
 		perror("shmat");
@@ -390,19 +380,43 @@
 		shmctl(seg->shm_id, IPC_RMID, NULL);
 #endif
 
-	memset(seg->threads, 0, max_jobs * sizeof(struct thread_data));
-	for (i = 0; i < max_jobs; i++)
+	nr_segments++;
+
+	memset(seg->threads, 0, JOBS_PER_SEG * sizeof(struct thread_data));
+	for (i = 0; i < JOBS_PER_SEG; i++)
 		DRD_IGNORE_VAR(seg->threads[i]);
-	fio_debug_jobp = (unsigned int *)(seg->threads + max_jobs);
+	seg->nr_threads = 0;
+
+	/* Not first segment, we're done */
+	if (nr_segments != 1) {
+		cur_segment++;
+		return 0;
+	}
+
+	fio_debug_jobp = (unsigned int *)(seg->threads + JOBS_PER_SEG);
 	*fio_debug_jobp = -1;
 	fio_warned = fio_debug_jobp + 1;
 	*fio_warned = 0;
 
 	flow_init();
-
 	return 0;
 }
 
+/*
+ * The thread areas are shared between the main process and the job
+ * threads/processes, and is split into chunks of JOBS_PER_SEG. If the current
+ * segment has no more room, add a new chunk.
+ */
+static int expand_thread_area(void)
+{
+	struct thread_segment *seg = &segments[cur_segment];
+
+	if (nr_segments && seg->nr_threads < JOBS_PER_SEG)
+		return 0;
+
+	return add_thread_segment();
+}
+
 static void dump_print_option(struct print_option *p)
 {
 	const char *delim;
@@ -471,11 +485,12 @@
 static struct thread_data *get_new_job(bool global, struct thread_data *parent,
 				       bool preserve_eo, const char *jobname)
 {
+	struct thread_segment *seg;
 	struct thread_data *td;
 
 	if (global)
 		return &def_thread;
-	if (setup_thread_area()) {
+	if (expand_thread_area()) {
 		log_err("error: failed to setup shm segment\n");
 		return NULL;
 	}
@@ -485,7 +500,9 @@
 		return NULL;
 	}
 
-	td = &segments[0].threads[thread_number++];
+	seg = &segments[cur_segment];
+	td = &seg->threads[seg->nr_threads++];
+	thread_number++;
 	*td = *parent;
 
 	INIT_FLIST_HEAD(&td->opt_list);
@@ -536,6 +553,7 @@
 		free(td->o.name);
 
 	memset(td, 0, sizeof(*td));
+	segments[cur_segment].nr_threads--;
 	thread_number--;
 }
 
diff --git a/libfio.c b/libfio.c
index 7348b16..6144a47 100644
--- a/libfio.c
+++ b/libfio.c
@@ -156,8 +156,13 @@
 
 void reset_fio_state(void)
 {
+	int i;
+
 	groupid = 0;
 	thread_number = 0;
+	cur_segment = 0;
+	for (i = 0; i < nr_segments; i++)
+		segments[i].nr_threads = 0;
 	stat_number = 0;
 	done_secs = 0;
 }
diff --git a/server.c b/server.c
index c9b5c28..1b65297 100644
--- a/server.c
+++ b/server.c
@@ -950,7 +950,7 @@
 		return 0;
 	}
 
-	td = &segments[0].threads[tnumber - 1];
+	td = tnumber_to_td(tnumber);
 	convert_thread_options_to_cpu(&td->o, &pdu->top);
 	send_update_job_reply(cmd->tag, 0);
 	return 0;