drivers/char/fetchop.c

/*
 * SN Platform FetchOp Support
 *
 * This driver exports the SN fetchop facility to user processes.
 * Fetchops are atomic memory operations that are implemented in the
 * memory controller on SGI SN hardware.
 */

/*
 * Copyright (C) 1999,2001-2003 Silicon Graphics, Inc. All rights
 * reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it would be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * Further, this software is distributed without any warranty that it is
 * free of the rightful claim of any third person regarding infringement
 * or the like.  Any license provided herein, whether implied or
 * otherwise, applies only to this software file.  Patent licenses, if
 * any, provided herein do not apply to combinations of this program with
 * other software, or any other product whatsoever.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston MA
 * 02111-1307, USA.
 *
 * Contact information:  Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
 * Mountain View, CA  94043, or:
 *
 * http://www.sgi.com
 *
 * For further information regarding this notice, see:
 *
 * http://oss.sgi.com/projects/GenInfo/NoticeExplan
 */


#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/devfs_fs_kernel.h>
#include <linux/miscdevice.h>
#include <linux/spinlock.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/vmalloc.h>
#include <linux/bitops.h>
#include <linux/efi.h>
#include <asm/system.h>
#include <asm/pgtable.h>
#include <asm/machvec.h>
#include <asm/sn/sgi.h>
#include <asm/sn/addrs.h>
#include <asm/sn/arch.h>
#include <asm/sn/fetchop.h>
#include <asm/sn/sn_cpuid.h>


#define DRIVER_ID_STR	"SGI Fetchop Device Driver"
#define REVISION		"1.03"


#define MSPEC_TO_NID(maddr)	nasid_to_cnodeid(NASID_GET(maddr))


static int fetchop_mmap(struct file *file, struct vm_area_struct *vma);
static void fetchop_open(struct vm_area_struct *vma);
static void fetchop_close(struct vm_area_struct *vma);

static struct file_operations fetchop_fops = {
	owner:		THIS_MODULE,
	mmap:		fetchop_mmap,
};

static struct miscdevice fetchop_miscdev = {
	MISC_DYNAMIC_MINOR,
	"fetchop",
	&fetchop_fops
};

static struct vm_operations_struct fetchop_vm_ops = {
	open:		fetchop_open,
	close:		fetchop_close,
};

/*
 * There is one of these structs per node. It is used to manage the fetchop
 * space that is available on the node. Current assumption is that there is
 * only 1 fetchop block of memory per node.
 */
struct node_fetchops {
	long		maddr;		/* MSPEC address of start of fetchops. */
	int		count;		/* Total number of fetchop pages. */
	atomic_t		free;		/* Number of pages currently free. */
	unsigned long	bits[1];		/* Bitmap for managing pages. */
};


/*
 * One of these structures is allocated when a fetchop region is mmaped. The
 * structure is pointed to by the vma->vm_private_data field in the vma struct. 
 * This structure is used to record the addresses of the fetchop pages.
 */
struct vma_data {
	int		count;		/* Number of pages allocated. */
	atomic_t		refcnt;		/* Number of vmas sharing the data. */
	unsigned long	maddr[1];	/* Array of MSPEC addresses. */
};


/*
 * Fetchop statistics.
 */
struct fetchop_stats {
	unsigned long  map_count;		/* Number of active mmap's */
	unsigned long  pages_in_use;		/* Number of fetchop pages in use */
	unsigned long  pages_total;		/* Total number of fetchop pages */
};

static struct fetchop_stats	fetchop_stats;
static struct node_fetchops	*node_fetchops[MAX_COMPACT_NODES];
static spinlock_t		fetchop_lock = SPIN_LOCK_UNLOCKED;

/*
 * NOTE: This is included here simply because the kernel doesn't have
 * a generally acceptable UC memory allocator.  See PV: 896479 for
 * more details. --cw
 *
 * efi_memmap_walk_uc
 *
 * This function walks the EFI memory map and calls 'callback' once
 * for each EFI memory descriptor that has memory that marked as only
 * EFI_MEMORY_UC.
 */
static void
efi_memmap_walk_uc (efi_freemem_callback_t callback, void *arg)
{
	void *efi_map_start, *efi_map_end, *p;
	efi_memory_desc_t *md;
	u64 efi_desc_size, start, end;

	efi_map_start = __va(ia64_boot_param->efi_memmap);
	efi_map_end   = efi_map_start + ia64_boot_param->efi_memmap_size;
	efi_desc_size = ia64_boot_param->efi_memdesc_size;

	for (p = efi_map_start; p < efi_map_end; p += efi_desc_size) {
		md = p;
		if (md->attribute == EFI_MEMORY_UC) {
			start = PAGE_ALIGN(md->phys_addr);
			end = PAGE_ALIGN((md->phys_addr+(md->num_pages << EFI_PAGE_SHIFT)) & PAGE_MASK);
			if ((*callback)(start, end, arg) < 0)
				return;
		}
	}
}


/*
 * fetchop_initialize_page
 *
 * Initial a page that is about to be used for fetchops. 
 * All fetchop variables in the page are set to 0.
 *
 */
static void
fetchop_initialize_page(unsigned long maddr)
{
	unsigned long	p, pe;

	for (p=FETCHOP_KADDR_TO_MSPEC_ADDR(maddr), pe=p+PAGE_SIZE; p<pe; p+=FETCHOP_VAR_SIZE)
		FETCHOP_STORE_OP(p,FETCHOP_STORE, 0);
}


/*
 * fetchop_alloc_page
 *
 * Allocate 1 fetchop page. Allocates on the requested node. If no
 * fetchops are available on the requested node, roundrobin starting
 * with higher nodes,
 */
static unsigned long
fetchop_alloc_page(int nid)
{
	int i, bit;
	struct node_fetchops *fops;
	unsigned long maddr;

	if (nid < 0 || nid >= numnodes)
		nid = numa_node_id();
	for (i=0; i<numnodes; i++) {
		fops = node_fetchops[nid];
		while (fops && (bit = find_first_zero_bit(fops->bits, fops->count)) < fops->count) {
			if (test_and_set_bit(bit, fops->bits) == 0) {
				atomic_dec(&node_fetchops[nid]->free);
				maddr = fops->maddr + (bit<<PAGE_SHIFT);
				fetchop_initialize_page(maddr);
				return maddr;
			}
		}
		nid = (nid+1 < numnodes) ? nid+1 : 0;
	}
	return 0;

}


/*
 * fetchop_free_pages
 *
 * Free all fetchop pages that are linked to a vma struct.
 */
static void
fetchop_free_page(unsigned long maddr)
{
	int nid, bit;

	nid = MSPEC_TO_NID(maddr);
	bit = (maddr - node_fetchops[nid]->maddr) >> PAGE_SHIFT;
	clear_bit(bit, node_fetchops[nid]->bits);
	atomic_inc(&node_fetchops[nid]->free);
}

static void
fetchop_free_pages(struct vma_data *vdata)
{
	int i;

	for (i=0; i<vdata->count; i++)
		fetchop_free_page(vdata->maddr[i]);
}


/*
 * fetchop_update_stats
 *
 * Update statistics of the number of fetchop mappings & pages.
 * If creating a new mapping, ensure that we don't exceed the maximum allowed
 * number of fetchop pages.
 */
static int
fetchop_update_stats(int mmap, long count)
{
	int	ret = 0;

	spin_lock(&fetchop_lock);
	if (count > 0 && fetchop_stats.pages_in_use + count > fetchop_stats.pages_total)  {
		ret = -1;
	} else {
		fetchop_stats.map_count += mmap;
		fetchop_stats.pages_in_use += count;
	}
	spin_unlock(&fetchop_lock);

	return ret;
}


/*
 * fetchop_mmap
 *
 * Called when mmaping the device. Creates fetchop pages and map them
 * to user space.
 */
static int
fetchop_mmap(struct file *file, struct vm_area_struct *vma)
{
	unsigned long vm_start;
	unsigned long maddr;
	int pages;
	struct vma_data *vdata;

	if (vma->vm_pgoff != 0)
		return -EINVAL;

	if ((vma->vm_flags&VM_WRITE) == 0)
		return -EPERM;

	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
	if (fetchop_update_stats(1, pages) < 0)
		return -ENOSPC;

	if (!(vdata=vmalloc(sizeof(struct vma_data)+(pages-1)*sizeof(long)))) {
		fetchop_update_stats(-1, -pages);
		return -ENOMEM;
	}

	vdata->count = 0;
	vdata->refcnt = ATOMIC_INIT(1);
	vma->vm_private_data = vdata;

	vma->vm_flags |= (VM_IO | VM_SHM | VM_LOCKED | VM_NONCACHED);
	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	vma->vm_ops = &fetchop_vm_ops;
	vm_start = vma->vm_start;

	while (vm_start < vma->vm_end) {
		maddr = fetchop_alloc_page(numa_node_id());
		if (maddr == 0)
			BUG();
		vdata->maddr[vdata->count++] = maddr;


		if (remap_page_range(vm_start, __pa(maddr), PAGE_SIZE, vma->vm_page_prot)) {
			fetchop_free_pages(vma->vm_private_data);
			vfree(vdata);
			fetchop_update_stats(-1, -pages);
			return -EAGAIN;
		}
		vm_start += PAGE_SIZE;
	}

	return 0;
}

/*
 * fetchop_open
 *
 * Called when a device mapping is created by a means other than mmap
 * (via fork, etc.).  Increments the reference count on the underlying
 * fetchop data so it is not freed prematurely.
 */
static void
fetchop_open(struct vm_area_struct *vma)
{
	struct vma_data *vdata;

	vdata = vma->vm_private_data;
	if (vdata && vdata->count) {
		atomic_inc(&vdata->refcnt);
	}
}

/*
 * fetchop_close
 *
 * Called when unmapping a device mapping. Frees all fetchop pages
 * belonging to the vma.
 */
static void
fetchop_close(struct vm_area_struct *vma)
{
	struct vma_data *vdata;

	vdata = vma->vm_private_data;
	if (vdata && vdata->count && !atomic_dec(&vdata->refcnt)) {
		fetchop_free_pages(vdata);
		fetchop_update_stats(-1, -vdata->count);
		vfree(vdata);
	}
}

#ifdef CONFIG_PROC_FS

static struct proc_dir_entry   *proc_fetchop;

/*
 * fetchop_read_proc
 *
 * Implements /proc/fetchop. Return statistics about fetchops.
 */
static int
fetchop_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data)
{
	struct node_fetchops *fops;
	int len = 0, nid;

	len += sprintf(page + len, "mappings               : %lu\n", fetchop_stats.map_count);
	len += sprintf(page + len, "current fetchop pages  : %lu\n", fetchop_stats.pages_in_use);
	len += sprintf(page + len, "maximum fetchop pages  : %lu\n", fetchop_stats.pages_total);

	len += sprintf(page + len, "%4s %7s %7s\n", "node", "total", "free");
	for (nid = 0; nid < numnodes; nid++) {
		fops = node_fetchops[nid];
		len += sprintf(page + len, "%4d %7d %7d\n", nid, fops ? fops->count : 0, fops ? atomic_read(&fops->free) : 0);
	}

	if (len <= off+count) *eof = 1;
	*start = page + off;
	len   -= off;
	if (len>count) len = count;
	if (len<0) len = 0;
	return len;
}

static int
fetchop_write_proc (struct file *file, const char *userbuf, unsigned long count, void *data)
{
    char buf[80];

    if (copy_from_user(buf, userbuf, count < sizeof(buf) ? count : sizeof(buf)))
        return -EFAULT;

    return count;
}
#endif /* CONFIG_PROC_FS */

/*
 * fetchop_build_memmap,
 *
 * Called at boot time to build a map of pages that can be used for
 * fetchops.
 */
static int __init
fetchop_build_memmap(unsigned long start, unsigned long end, void *arg)
{
	struct node_fetchops *fops;
	long count, bytes;

	count = (end - start) >> PAGE_SHIFT;
	bytes = sizeof(struct node_fetchops) + count/8;
	fops = vmalloc(bytes);
	memset(fops, 0, bytes);
	fops->maddr = FETCHOP_KADDR_TO_MSPEC_ADDR(start);
	fops->count = count;
	atomic_add(count, &fops->free);
	fetchop_stats.pages_total += count;
	node_fetchops[MSPEC_TO_NID(start)] = fops;

	sn_flush_all_caches((long)__va(start), end - start);

	return 0;
}



/*
 * fetchop_init
 *
 * Called at boot time to initialize the fetchop facility.
 */
static int __init
fetchop_init(void)
{
	int ret;
	devfs_handle_t  hnd;

	if (!ia64_platform_is("sn2"))
		return -ENODEV;

#ifdef CONFIG_DEVFS_FS
	if (!devfs_register(NULL, FETCHOP_BASENAME, DEVFS_FL_AUTO_DEVNUM,
			    0, 0, S_IFCHR | S_IRUGO | S_IWUGO, &fetchop_fops, NULL)) {
		printk(KERN_ERR "%s:  failed to register device\n", DRIVER_ID_STR);
		return -ENODEV;
	}
#endif

	if ((ret = misc_register(&fetchop_miscdev))) {
		printk(KERN_ERR "%s: failed to register device\n", DRIVER_ID_STR);
		return ret;
	}
	printk(KERN_DEBUG "%s:  registered misc-device with minor %d\n", DRIVER_ID_STR, fetchop_miscdev.minor);

	if ((proc_fetchop = create_proc_entry(FETCHOP_BASENAME, 0644, NULL)) == NULL) {
		printk(KERN_ERR "%s: unable to create proc entry", DRIVER_ID_STR);
		devfs_unregister(hnd);
		return -EINVAL;
	}

#ifdef CONFIG_PROC_FS
	if ((proc_fetchop = create_proc_entry(FETCHOP_BASENAME, 0644, NULL)) == NULL) {
		printk(KERN_ERR "%s: unable to create proc entry", DRIVER_ID_STR);
			devfs_unregister(hnd);
		return -EINVAL;
	}
	proc_fetchop->read_proc = fetchop_read_proc;
	proc_fetchop->write_proc = fetchop_write_proc;
#endif /* CONFIG_PROC_FS */

	efi_memmap_walk_uc(fetchop_build_memmap, 0);
	printk(KERN_INFO "%s: v%s\n", DRIVER_ID_STR, REVISION);

	return 0;
}



/*-----------------------------------------------------------------------------
 * KERNEL APIs
 * 	Note: right now, these APIs return a full page of fetchops.  If these
 *	interfaces are used often for tasks which do not require a full page of
 *	fetchops, new APIs should be added to suballocate out of a single page.
 */

unsigned long
fetchop_kalloc_page(int nid)
{
	if (fetchop_update_stats(1, 1) < 0)
		return 0;
	return fetchop_alloc_page(nid);
}
EXPORT_SYMBOL(fetchop_kalloc_page);


void
fetchop_kfree_page(unsigned long maddr)
{
	fetchop_free_page(maddr);
	fetchop_update_stats(-1, -1);
}
EXPORT_SYMBOL(fetchop_kfree_page);

module_init(fetchop_init);

MODULE_AUTHOR("Silicon Graphics Inc.");
MODULE_DESCRIPTION("Driver for SGI SN 'fetchop' atomic memory operations");
MODULE_LICENSE("GPL");