kernel/jump_label.c - pub/scm/linux/kernel/git/ash/linux - Git at Google

 /*
  * jump label support
  *
  * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
  *
  */
 #include <linux/jump_label.h>
 #include <linux/memory.h>
 #include <linux/uaccess.h>
 #include <linux/module.h>
 #include <linux/list.h>
 #include <linux/jhash.h>
 #include <linux/slab.h>
 #include <linux/sort.h>
 #include <linux/err.h>

 #ifdef HAVE_JUMP_LABEL

 #define JUMP_LABEL_HASH_BITS 6
 #define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
 static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];

 /* mutex to protect coming/going of the the jump_label table */
 static DEFINE_MUTEX(jump_label_mutex);

 struct jump_label_entry {
 	struct hlist_node hlist;
 	struct jump_entry *table;
 	int nr_entries;
 	/* hang modules off here */
 	struct hlist_head modules;
 	unsigned long key;
 };

 struct jump_label_module_entry {
 	struct hlist_node hlist;
 	struct jump_entry *table;
 	int nr_entries;
 	struct module *mod;
 };

 void jump_label_lock(void)
 {
 	mutex_lock(&jump_label_mutex);
 }

 void jump_label_unlock(void)
 {
 	mutex_unlock(&jump_label_mutex);
 }

 static int jump_label_cmp(const void *a, const void *b)
 {
 	const struct jump_entry *jea = a;
 	const struct jump_entry *jeb = b;

 	if (jea->key < jeb->key)
 		return -1;

 	if (jea->key > jeb->key)
 		return 1;

 	return 0;
 }

 static void
 sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
 {
 	unsigned long size;

 	size = (((unsigned long)stop - (unsigned long)start)
 					/ sizeof(struct jump_entry));
 	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
 }

 static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
 {
 	struct hlist_head *head;
 	struct hlist_node *node;
 	struct jump_label_entry *e;
 	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);

 	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
 	hlist_for_each_entry(e, node, head, hlist) {
 		if (key == e->key)
 			return e;
 	}
 	return NULL;
 }

 static struct jump_label_entry *
 add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
 {
 	struct hlist_head *head;
 	struct jump_label_entry *e;
 	u32 hash;

 	e = get_jump_label_entry(key);
 	if (e)
 		return ERR_PTR(-EEXIST);

 	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
 	if (!e)
 		return ERR_PTR(-ENOMEM);

 	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
 	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
 	e->key = key;
 	e->table = table;
 	e->nr_entries = nr_entries;
 	INIT_HLIST_HEAD(&(e->modules));
 	hlist_add_head(&e->hlist, head);
 	return e;
 }

 static int
 build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
 {
 	struct jump_entry *iter, *iter_begin;
 	struct jump_label_entry *entry;
 	int count;

 	sort_jump_label_entries(start, stop);
 	iter = start;
 	while (iter < stop) {
 		entry = get_jump_label_entry(iter->key);
 		if (!entry) {
 			iter_begin = iter;
 			count = 0;
 			while ((iter < stop) &&
 				(iter->key == iter_begin->key)) {
 				iter++;
 				count++;
 			}
 			entry = add_jump_label_entry(iter_begin->key,
 							count, iter_begin);
 			if (IS_ERR(entry))
 				return PTR_ERR(entry);
 		 } else {
 			WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
 			return -1;
 		}
 	}
 	return 0;
 }

 /***
  * jump_label_update - update jump label text
  * @key -  key value associated with a a jump label
  * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
  *
  * Will enable/disable the jump for jump label @key, depending on the
  * value of @type.
  *
  */

 void jump_label_update(unsigned long key, enum jump_label_type type)
 {
 	struct jump_entry *iter;
 	struct jump_label_entry *entry;
 	struct hlist_node *module_node;
 	struct jump_label_module_entry *e_module;
 	int count;

 	jump_label_lock();
 	entry = get_jump_label_entry((jump_label_t)key);
 	if (entry) {
 		count = entry->nr_entries;
 		iter = entry->table;
 		while (count--) {
 			if (kernel_text_address(iter->code))
 				arch_jump_label_transform(iter, type);
 			iter++;
 		}
 		/* eanble/disable jump labels in modules */
 		hlist_for_each_entry(e_module, module_node, &(entry->modules),
 							hlist) {
 			count = e_module->nr_entries;
 			iter = e_module->table;
 			while (count--) {
 				if (iter->key &&
 						kernel_text_address(iter->code))
 					arch_jump_label_transform(iter, type);
 				iter++;
 			}
 		}
 	}
 	jump_label_unlock();
 }

 static int addr_conflict(struct jump_entry *entry, void *start, void *end)
 {
 	if (entry->code <= (unsigned long)end &&
 		entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
 		return 1;

 	return 0;
 }

 #ifdef CONFIG_MODULES

 static int module_conflict(void *start, void *end)
 {
 	struct hlist_head *head;
 	struct hlist_node *node, *node_next, *module_node, *module_node_next;
 	struct jump_label_entry *e;
 	struct jump_label_module_entry *e_module;
 	struct jump_entry *iter;
 	int i, count;
 	int conflict = 0;

 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
 		head = &jump_label_table[i];
 		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
 			hlist_for_each_entry_safe(e_module, module_node,
 							module_node_next,
 							&(e->modules), hlist) {
 				count = e_module->nr_entries;
 				iter = e_module->table;
 				while (count--) {
 					if (addr_conflict(iter, start, end)) {
 						conflict = 1;
 						goto out;
 					}
 					iter++;
 				}
 			}
 		}
 	}
 out:
 	return conflict;
 }

 #endif

 /***
  * jump_label_text_reserved - check if addr range is reserved
  * @start: start text addr
  * @end: end text addr
  *
  * checks if the text addr located between @start and @end
  * overlaps with any of the jump label patch addresses. Code
  * that wants to modify kernel text should first verify that
  * it does not overlap with any of the jump label addresses.
  * Caller must hold jump_label_mutex.
  *
  * returns 1 if there is an overlap, 0 otherwise
  */
 int jump_label_text_reserved(void *start, void *end)
 {
 	struct jump_entry *iter;
 	struct jump_entry *iter_start = __start___jump_table;
 	struct jump_entry *iter_stop = __start___jump_table;
 	int conflict = 0;

 	iter = iter_start;
 	while (iter < iter_stop) {
 		if (addr_conflict(iter, start, end)) {
 			conflict = 1;
 			goto out;
 		}
 		iter++;
 	}

 	/* now check modules */
 #ifdef CONFIG_MODULES
 	conflict = module_conflict(start, end);
 #endif
 out:
 	return conflict;
 }

 /*
  * Not all archs need this.
  */
 void __weak arch_jump_label_text_poke_early(jump_label_t addr)
 {
 }

 static __init int init_jump_label(void)
 {
 	int ret;
 	struct jump_entry *iter_start = __start___jump_table;
 	struct jump_entry *iter_stop = __stop___jump_table;
 	struct jump_entry *iter;

 	jump_label_lock();
 	ret = build_jump_label_hashtable(__start___jump_table,
 					 __stop___jump_table);
 	iter = iter_start;
 	while (iter < iter_stop) {
 		arch_jump_label_text_poke_early(iter->code);
 		iter++;
 	}
 	jump_label_unlock();
 	return ret;
 }
 early_initcall(init_jump_label);

 #ifdef CONFIG_MODULES

 static struct jump_label_module_entry *
 add_jump_label_module_entry(struct jump_label_entry *entry,
 			    struct jump_entry *iter_begin,
 			    int count, struct module *mod)
 {
 	struct jump_label_module_entry *e;

 	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
 	if (!e)
 		return ERR_PTR(-ENOMEM);
 	e->mod = mod;
 	e->nr_entries = count;
 	e->table = iter_begin;
 	hlist_add_head(&e->hlist, &entry->modules);
 	return e;
 }

 static int add_jump_label_module(struct module *mod)
 {
 	struct jump_entry *iter, *iter_begin;
 	struct jump_label_entry *entry;
 	struct jump_label_module_entry *module_entry;
 	int count;

 	/* if the module doesn't have jump label entries, just return */
 	if (!mod->num_jump_entries)
 		return 0;

 	sort_jump_label_entries(mod->jump_entries,
 				mod->jump_entries + mod->num_jump_entries);
 	iter = mod->jump_entries;
 	while (iter < mod->jump_entries + mod->num_jump_entries) {
 		entry = get_jump_label_entry(iter->key);
 		iter_begin = iter;
 		count = 0;
 		while ((iter < mod->jump_entries + mod->num_jump_entries) &&
 			(iter->key == iter_begin->key)) {
 				iter++;
 				count++;
 		}
 		if (!entry) {
 			entry = add_jump_label_entry(iter_begin->key, 0, NULL);
 			if (IS_ERR(entry))
 				return PTR_ERR(entry);
 		}
 		module_entry = add_jump_label_module_entry(entry, iter_begin,
 							   count, mod);
 		if (IS_ERR(module_entry))
 			return PTR_ERR(module_entry);
 	}
 	return 0;
 }

 static void remove_jump_label_module(struct module *mod)
 {
 	struct hlist_head *head;
 	struct hlist_node *node, *node_next, *module_node, *module_node_next;
 	struct jump_label_entry *e;
 	struct jump_label_module_entry *e_module;
 	int i;

 	/* if the module doesn't have jump label entries, just return */
 	if (!mod->num_jump_entries)
 		return;

 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
 		head = &jump_label_table[i];
 		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
 			hlist_for_each_entry_safe(e_module, module_node,
 						  module_node_next,
 						  &(e->modules), hlist) {
 				if (e_module->mod == mod) {
 					hlist_del(&e_module->hlist);
 					kfree(e_module);
 				}
 			}
 			if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
 				hlist_del(&e->hlist);
 				kfree(e);
 			}
 		}
 	}
 }

 static void remove_jump_label_module_init(struct module *mod)
 {
 	struct hlist_head *head;
 	struct hlist_node *node, *node_next, *module_node, *module_node_next;
 	struct jump_label_entry *e;
 	struct jump_label_module_entry *e_module;
 	struct jump_entry *iter;
 	int i, count;

 	/* if the module doesn't have jump label entries, just return */
 	if (!mod->num_jump_entries)
 		return;

 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
 		head = &jump_label_table[i];
 		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
 			hlist_for_each_entry_safe(e_module, module_node,
 						  module_node_next,
 						  &(e->modules), hlist) {
 				if (e_module->mod != mod)
 					continue;
 				count = e_module->nr_entries;
 				iter = e_module->table;
 				while (count--) {
 					if (within_module_init(iter->code, mod))
 						iter->key = 0;
 					iter++;
 				}
 			}
 		}
 	}
 }

 static int
 jump_label_module_notify(struct notifier_block *self, unsigned long val,
 			 void *data)
 {
 	struct module *mod = data;
 	int ret = 0;

 	switch (val) {
 	case MODULE_STATE_COMING:
 		jump_label_lock();
 		ret = add_jump_label_module(mod);
 		if (ret)
 			remove_jump_label_module(mod);
 		jump_label_unlock();
 		break;
 	case MODULE_STATE_GOING:
 		jump_label_lock();
 		remove_jump_label_module(mod);
 		jump_label_unlock();
 		break;
 	case MODULE_STATE_LIVE:
 		jump_label_lock();
 		remove_jump_label_module_init(mod);
 		jump_label_unlock();
 		break;
 	}
 	return ret;
 }

 /***
  * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
  * @mod: module to patch
  *
  * Allow for run-time selection of the optimal nops. Before the module
  * loads patch these with arch_get_jump_label_nop(), which is specified by
  * the arch specific jump label code.
  */
 void jump_label_apply_nops(struct module *mod)
 {
 	struct jump_entry *iter;

 	/* if the module doesn't have jump label entries, just return */
 	if (!mod->num_jump_entries)
 		return;

 	iter = mod->jump_entries;
 	while (iter < mod->jump_entries + mod->num_jump_entries) {
 		arch_jump_label_text_poke_early(iter->code);
 		iter++;
 	}
 }

 struct notifier_block jump_label_module_nb = {
 	.notifier_call = jump_label_module_notify,
 	.priority = 0,
 };

 static __init int init_jump_label_module(void)
 {
 	return register_module_notifier(&jump_label_module_nb);
 }
 early_initcall(init_jump_label_module);

 #endif /* CONFIG_MODULES */

 #endif
	/*
	* jump label support
	*
	* Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
	*
	*/
	#include <linux/jump_label.h>
	#include <linux/memory.h>
	#include <linux/uaccess.h>
	#include <linux/module.h>
	#include <linux/list.h>
	#include <linux/jhash.h>
	#include <linux/slab.h>
	#include <linux/sort.h>
	#include <linux/err.h>

	#ifdef HAVE_JUMP_LABEL

	#define JUMP_LABEL_HASH_BITS 6
	#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
	static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];

	/* mutex to protect coming/going of the the jump_label table */
	static DEFINE_MUTEX(jump_label_mutex);

	struct jump_label_entry {
	struct hlist_node hlist;
	struct jump_entry *table;
	int nr_entries;
	/* hang modules off here */
	struct hlist_head modules;
	unsigned long key;
	};

	struct jump_label_module_entry {
	struct hlist_node hlist;
	struct jump_entry *table;
	int nr_entries;
	struct module *mod;
	};

	void jump_label_lock(void)
	{
	mutex_lock(&jump_label_mutex);
	}

	void jump_label_unlock(void)
	{
	mutex_unlock(&jump_label_mutex);
	}

	static int jump_label_cmp(const void a, const void b)
	{
	const struct jump_entry *jea = a;
	const struct jump_entry *jeb = b;

	if (jea->key < jeb->key)
	return -1;

	if (jea->key > jeb->key)
	return 1;

	return 0;
	}

	static void
	sort_jump_label_entries(struct jump_entry start, struct jump_entry stop)
	{
	unsigned long size;

	size = (((unsigned long)stop - (unsigned long)start)
	/ sizeof(struct jump_entry));
	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
	}

	static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
	{
	struct hlist_head *head;
	struct hlist_node *node;
	struct jump_label_entry *e;
	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);

	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	hlist_for_each_entry(e, node, head, hlist) {
	if (key == e->key)
	return e;
	}
	return NULL;
	}

	static struct jump_label_entry *
	add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
	{
	struct hlist_head *head;
	struct jump_label_entry *e;
	u32 hash;

	e = get_jump_label_entry(key);
	if (e)
	return ERR_PTR(-EEXIST);

	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
	if (!e)
	return ERR_PTR(-ENOMEM);

	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	e->key = key;
	e->table = table;
	e->nr_entries = nr_entries;
	INIT_HLIST_HEAD(&(e->modules));
	hlist_add_head(&e->hlist, head);
	return e;
	}

	static int
	build_jump_label_hashtable(struct jump_entry start, struct jump_entry stop)
	{
	struct jump_entry iter, iter_begin;
	struct jump_label_entry *entry;
	int count;

	sort_jump_label_entries(start, stop);
	iter = start;
	while (iter < stop) {
	entry = get_jump_label_entry(iter->key);
	if (!entry) {
	iter_begin = iter;
	count = 0;
	while ((iter < stop) &&
	(iter->key == iter_begin->key)) {
	iter++;
	count++;
	}
	entry = add_jump_label_entry(iter_begin->key,
	count, iter_begin);
	if (IS_ERR(entry))
	return PTR_ERR(entry);
	} else {
	WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
	return -1;
	}
	}
	return 0;
	}

	/***
	* jump_label_update - update jump label text
	* @key - key value associated with a a jump label
	* @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
	*
	* Will enable/disable the jump for jump label @key, depending on the
	* value of @type.
	*
	*/

	void jump_label_update(unsigned long key, enum jump_label_type type)
	{
	struct jump_entry *iter;
	struct jump_label_entry *entry;
	struct hlist_node *module_node;
	struct jump_label_module_entry *e_module;
	int count;

	jump_label_lock();
	entry = get_jump_label_entry((jump_label_t)key);
	if (entry) {
	count = entry->nr_entries;
	iter = entry->table;
	while (count--) {
	if (kernel_text_address(iter->code))
	arch_jump_label_transform(iter, type);
	iter++;
	}
	/* eanble/disable jump labels in modules */
	hlist_for_each_entry(e_module, module_node, &(entry->modules),
	hlist) {
	count = e_module->nr_entries;
	iter = e_module->table;
	while (count--) {
	if (iter->key &&
	kernel_text_address(iter->code))
	arch_jump_label_transform(iter, type);
	iter++;
	}
	}
	}
	jump_label_unlock();
	}

	static int addr_conflict(struct jump_entry entry, void start, void *end)
	{
	if (entry->code <= (unsigned long)end &&
	entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
	return 1;

	return 0;
	}

	#ifdef CONFIG_MODULES

	static int module_conflict(void start, void end)
	{
	struct hlist_head *head;
	struct hlist_node node, node_next, module_node, module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	struct jump_entry *iter;
	int i, count;
	int conflict = 0;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	head = &jump_label_table[i];
	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	hlist_for_each_entry_safe(e_module, module_node,
	module_node_next,
	&(e->modules), hlist) {
	count = e_module->nr_entries;
	iter = e_module->table;
	while (count--) {
	if (addr_conflict(iter, start, end)) {
	conflict = 1;
	goto out;
	}
	iter++;
	}
	}
	}
	}
	out:
	return conflict;
	}

	#endif

	/***
	* jump_label_text_reserved - check if addr range is reserved
	* @start: start text addr
	* @end: end text addr
	*
	* checks if the text addr located between @start and @end
	* overlaps with any of the jump label patch addresses. Code
	* that wants to modify kernel text should first verify that
	* it does not overlap with any of the jump label addresses.
	* Caller must hold jump_label_mutex.
	*
	* returns 1 if there is an overlap, 0 otherwise
	*/
	int jump_label_text_reserved(void start, void end)
	{
	struct jump_entry *iter;
	struct jump_entry *iter_start = __start___jump_table;
	struct jump_entry *iter_stop = __start___jump_table;
	int conflict = 0;

	iter = iter_start;
	while (iter < iter_stop) {
	if (addr_conflict(iter, start, end)) {
	conflict = 1;
	goto out;
	}
	iter++;
	}

	/* now check modules */
	#ifdef CONFIG_MODULES
	conflict = module_conflict(start, end);
	#endif
	out:
	return conflict;
	}

	/*
	* Not all archs need this.
	*/
	void __weak arch_jump_label_text_poke_early(jump_label_t addr)
	{
	}

	static __init int init_jump_label(void)
	{
	int ret;
	struct jump_entry *iter_start = __start___jump_table;
	struct jump_entry *iter_stop = __stop___jump_table;
	struct jump_entry *iter;

	jump_label_lock();
	ret = build_jump_label_hashtable(__start___jump_table,
	__stop___jump_table);
	iter = iter_start;
	while (iter < iter_stop) {
	arch_jump_label_text_poke_early(iter->code);
	iter++;
	}
	jump_label_unlock();
	return ret;
	}
	early_initcall(init_jump_label);

	#ifdef CONFIG_MODULES

	static struct jump_label_module_entry *
	add_jump_label_module_entry(struct jump_label_entry *entry,
	struct jump_entry *iter_begin,
	int count, struct module *mod)
	{
	struct jump_label_module_entry *e;

	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
	if (!e)
	return ERR_PTR(-ENOMEM);
	e->mod = mod;
	e->nr_entries = count;
	e->table = iter_begin;
	hlist_add_head(&e->hlist, &entry->modules);
	return e;
	}

	static int add_jump_label_module(struct module *mod)
	{
	struct jump_entry iter, iter_begin;
	struct jump_label_entry *entry;
	struct jump_label_module_entry *module_entry;
	int count;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
	return 0;

	sort_jump_label_entries(mod->jump_entries,
	mod->jump_entries + mod->num_jump_entries);
	iter = mod->jump_entries;
	while (iter < mod->jump_entries + mod->num_jump_entries) {
	entry = get_jump_label_entry(iter->key);
	iter_begin = iter;
	count = 0;
	while ((iter < mod->jump_entries + mod->num_jump_entries) &&
	(iter->key == iter_begin->key)) {
	iter++;
	count++;
	}
	if (!entry) {
	entry = add_jump_label_entry(iter_begin->key, 0, NULL);
	if (IS_ERR(entry))
	return PTR_ERR(entry);
	}
	module_entry = add_jump_label_module_entry(entry, iter_begin,
	count, mod);
	if (IS_ERR(module_entry))
	return PTR_ERR(module_entry);
	}
	return 0;
	}

	static void remove_jump_label_module(struct module *mod)
	{
	struct hlist_head *head;
	struct hlist_node node, node_next, module_node, module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	int i;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
	return;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	head = &jump_label_table[i];
	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	hlist_for_each_entry_safe(e_module, module_node,
	module_node_next,
	&(e->modules), hlist) {
	if (e_module->mod == mod) {
	hlist_del(&e_module->hlist);
	kfree(e_module);
	}
	}
	if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
	hlist_del(&e->hlist);
	kfree(e);
	}
	}
	}
	}

	static void remove_jump_label_module_init(struct module *mod)
	{
	struct hlist_head *head;
	struct hlist_node node, node_next, module_node, module_node_next;
	struct jump_label_entry *e;
	struct jump_label_module_entry *e_module;
	struct jump_entry *iter;
	int i, count;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
	return;

	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	head = &jump_label_table[i];
	hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	hlist_for_each_entry_safe(e_module, module_node,
	module_node_next,
	&(e->modules), hlist) {
	if (e_module->mod != mod)
	continue;
	count = e_module->nr_entries;
	iter = e_module->table;
	while (count--) {
	if (within_module_init(iter->code, mod))
	iter->key = 0;
	iter++;
	}
	}
	}
	}
	}

	static int
	jump_label_module_notify(struct notifier_block *self, unsigned long val,
	void *data)
	{
	struct module *mod = data;
	int ret = 0;

	switch (val) {
	case MODULE_STATE_COMING:
	jump_label_lock();
	ret = add_jump_label_module(mod);
	if (ret)
	remove_jump_label_module(mod);
	jump_label_unlock();
	break;
	case MODULE_STATE_GOING:
	jump_label_lock();
	remove_jump_label_module(mod);
	jump_label_unlock();
	break;
	case MODULE_STATE_LIVE:
	jump_label_lock();
	remove_jump_label_module_init(mod);
	jump_label_unlock();
	break;
	}
	return ret;
	}

	/***
	* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
	* @mod: module to patch
	*
	* Allow for run-time selection of the optimal nops. Before the module
	* loads patch these with arch_get_jump_label_nop(), which is specified by
	* the arch specific jump label code.
	*/
	void jump_label_apply_nops(struct module *mod)
	{
	struct jump_entry *iter;

	/* if the module doesn't have jump label entries, just return */
	if (!mod->num_jump_entries)
	return;

	iter = mod->jump_entries;
	while (iter < mod->jump_entries + mod->num_jump_entries) {
	arch_jump_label_text_poke_early(iter->code);
	iter++;
	}
	}

	struct notifier_block jump_label_module_nb = {
	.notifier_call = jump_label_module_notify,
	.priority = 0,
	};

	static __init int init_jump_label_module(void)
	{
	return register_module_notifier(&jump_label_module_nb);
	}
	early_initcall(init_jump_label_module);

	#endif /* CONFIG_MODULES */

	#endif