mm/balloon.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common interface for implementing a memory balloon, including support
 * for migration of pages inflated in a memory balloon.
 *
 * Copyright (C) 2012, Red Hat, Inc.  Rafael Aquini <aquini@redhat.com>
 */
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/balloon.h>

/*
 * Lock protecting the balloon_dev_info of all devices. We don't really
 * expect more than one device.
 */
static DEFINE_SPINLOCK(balloon_pages_lock);

/**
 * balloon_page_insert - insert a page into the balloon's page list and make
 *			 the page->private assignment accordingly.
 * @balloon : pointer to balloon device
 * @page    : page to be assigned as a 'balloon page'
 *
 * Caller must ensure the balloon_pages_lock is held.
 */
static void balloon_page_insert(struct balloon_dev_info *balloon,
				       struct page *page)
{
	lockdep_assert_held(&balloon_pages_lock);
	__SetPageOffline(page);
	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) {
		SetPageMovableOps(page);
		set_page_private(page, (unsigned long)balloon);
	}
	list_add(&page->lru, &balloon->pages);
}

/**
 * balloon_page_finalize - prepare a balloon page that was removed from the
 *			   balloon list for release to the page allocator
 * @page: page to be released to the page allocator
 *
 * Caller must ensure the balloon_pages_lock is held.
 */
static void balloon_page_finalize(struct page *page)
{
	lockdep_assert_held(&balloon_pages_lock);
	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
		set_page_private(page, 0);
	/* PageOffline is sticky until the page is freed to the buddy. */
}

static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
				     struct page *page)
{
	balloon_page_insert(b_dev_info, page);
	if (b_dev_info->adjust_managed_page_count)
		adjust_managed_page_count(page, -1);
	__count_vm_event(BALLOON_INFLATE);
	inc_node_page_state(page, NR_BALLOON_PAGES);
}

/**
 * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
 *				 list.
 * @b_dev_info: balloon device descriptor where we will insert a new page to
 * @pages: pages to enqueue - allocated using balloon_page_alloc.
 *
 * Driver must call this function to properly enqueue balloon pages before
 * definitively removing them from the guest system.
 *
 * Return: number of pages that were enqueued.
 */
size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
				 struct list_head *pages)
{
	struct page *page, *tmp;
	unsigned long flags;
	size_t n_pages = 0;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	list_for_each_entry_safe(page, tmp, pages, lru) {
		list_del(&page->lru);
		balloon_page_enqueue_one(b_dev_info, page);
		n_pages++;
	}
	spin_unlock_irqrestore(&balloon_pages_lock, flags);
	return n_pages;
}
EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);

/**
 * balloon_page_list_dequeue() - removes pages from balloon's page list and
 *				 returns a list of the pages.
 * @b_dev_info: balloon device descriptor where we will grab a page from.
 * @pages: pointer to the list of pages that would be returned to the caller.
 * @n_req_pages: number of requested pages.
 *
 * Driver must call this function to properly de-allocate a previous enlisted
 * balloon pages before definitively releasing it back to the guest system.
 * This function tries to remove @n_req_pages from the ballooned pages and
 * return them to the caller in the @pages list.
 *
 * Note that this function may fail to dequeue some pages even if the balloon
 * isn't empty - since the page list can be temporarily empty due to compaction
 * of isolated pages.
 *
 * Return: number of pages that were added to the @pages list.
 */
size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
				 struct list_head *pages, size_t n_req_pages)
{
	struct page *page, *tmp;
	unsigned long flags;
	size_t n_pages = 0;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
		if (n_pages == n_req_pages)
			break;
		list_del(&page->lru);
		if (b_dev_info->adjust_managed_page_count)
			adjust_managed_page_count(page, 1);
		balloon_page_finalize(page);
		__count_vm_event(BALLOON_DEFLATE);
		list_add(&page->lru, pages);
		dec_node_page_state(page, NR_BALLOON_PAGES);
		n_pages++;
	}
	spin_unlock_irqrestore(&balloon_pages_lock, flags);

	return n_pages;
}
EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);

/**
 * balloon_page_alloc - allocates a new page for insertion into the balloon
 *			page list.
 *
 * Driver must call this function to properly allocate a new balloon page.
 * Driver must call balloon_page_enqueue before definitively removing the page
 * from the guest system.
 *
 * Return: struct page for the allocated page or NULL on allocation failure.
 */
struct page *balloon_page_alloc(void)
{
	gfp_t gfp_flags = __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;

	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
		gfp_flags |= GFP_HIGHUSER_MOVABLE;
	else
		gfp_flags |= GFP_HIGHUSER;

	return alloc_page(gfp_flags);
}
EXPORT_SYMBOL_GPL(balloon_page_alloc);

/**
 * balloon_page_enqueue - inserts a new page into the balloon page list.
 *
 * @b_dev_info: balloon device descriptor where we will insert a new page
 * @page: new page to enqueue - allocated using balloon_page_alloc.
 *
 * Drivers must call this function to properly enqueue a new allocated balloon
 * page before definitively removing the page from the guest system.
 *
 * Drivers must not enqueue pages while page->lru is still in
 * use, and must not use page->lru until a page was unqueued again.
 */
void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
			  struct page *page)
{
	unsigned long flags;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	balloon_page_enqueue_one(b_dev_info, page);
	spin_unlock_irqrestore(&balloon_pages_lock, flags);
}
EXPORT_SYMBOL_GPL(balloon_page_enqueue);

/**
 * balloon_page_dequeue - removes a page from balloon's page list and returns
 *			  its address to allow the driver to release the page.
 * @b_dev_info: balloon device descriptor where we will grab a page from.
 *
 * Driver must call this function to properly dequeue a previously enqueued page
 * before definitively releasing it back to the guest system.
 *
 * Caller must perform its own accounting to ensure that this
 * function is called only if some pages are actually enqueued.
 *
 * Note that this function may fail to dequeue some pages even if there are
 * some enqueued pages - since the page list can be temporarily empty due to
 * the compaction of isolated pages.
 *
 * TODO: remove the caller accounting requirements, and allow caller to wait
 * until all pages can be dequeued.
 *
 * Return: struct page for the dequeued page, or NULL if no page was dequeued.
 */
struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
{
	unsigned long flags;
	LIST_HEAD(pages);
	int n_pages;

	n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);

	if (n_pages != 1) {
		/*
		 * If we are unable to dequeue a balloon page because the page
		 * list is empty and there are no isolated pages, then something
		 * went out of track and some balloon pages are lost.
		 * BUG() here, otherwise the balloon driver may get stuck in
		 * an infinite loop while attempting to release all its pages.
		 */
		spin_lock_irqsave(&balloon_pages_lock, flags);
		if (unlikely(list_empty(&b_dev_info->pages) &&
			     !b_dev_info->isolated_pages))
			BUG();
		spin_unlock_irqrestore(&balloon_pages_lock, flags);
		return NULL;
	}
	return list_first_entry(&pages, struct page, lru);
}
EXPORT_SYMBOL_GPL(balloon_page_dequeue);

#ifdef CONFIG_BALLOON_MIGRATION
static struct balloon_dev_info *balloon_page_device(struct page *page)
{
	return (struct balloon_dev_info *)page_private(page);
}

static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)

{
	struct balloon_dev_info *b_dev_info;
	unsigned long flags;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	b_dev_info = balloon_page_device(page);
	if (!b_dev_info) {
		/*
		 * The page already got deflated and removed from the
		 * balloon list.
		 */
		spin_unlock_irqrestore(&balloon_pages_lock, flags);
		return false;
	}
	list_del(&page->lru);
	b_dev_info->isolated_pages++;
	spin_unlock_irqrestore(&balloon_pages_lock, flags);

	return true;
}

static void balloon_page_putback(struct page *page)
{
	struct balloon_dev_info *b_dev_info = balloon_page_device(page);
	unsigned long flags;

	/*
	 * When we isolated the page, the page was still inflated in a balloon
	 * device. As isolated balloon pages cannot get deflated, we still have
	 * a balloon device here.
	 */
	if (WARN_ON_ONCE(!b_dev_info))
		return;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	list_add(&page->lru, &b_dev_info->pages);
	b_dev_info->isolated_pages--;
	spin_unlock_irqrestore(&balloon_pages_lock, flags);
}

static int balloon_page_migrate(struct page *newpage, struct page *page,
		enum migrate_mode mode)
{
	struct balloon_dev_info *b_dev_info = balloon_page_device(page);
	unsigned long flags;
	int rc;

	/*
	 * When we isolated the page, the page was still inflated in a balloon
	 * device. As isolated balloon pages cannot get deflated, we still have
	 * a balloon device here.
	 */
	if (WARN_ON_ONCE(!b_dev_info))
		return -EAGAIN;

	rc = b_dev_info->migratepage(b_dev_info, newpage, page, mode);
	if (rc < 0 && rc != -ENOENT)
		return rc;

	spin_lock_irqsave(&balloon_pages_lock, flags);
	if (!rc) {
		/* Insert the new page into the balloon list. */
		get_page(newpage);
		balloon_page_insert(b_dev_info, newpage);
		__count_vm_event(BALLOON_MIGRATE);

		if (b_dev_info->adjust_managed_page_count &&
		    page_zone(page) != page_zone(newpage)) {
			/*
			 * When we migrate a page to a different zone we
			 * have to fixup the count of both involved zones.
			 */
			adjust_managed_page_count(page, 1);
			adjust_managed_page_count(newpage, -1);
		}
	} else {
		/* Old page was deflated but new page not inflated. */
		__count_vm_event(BALLOON_DEFLATE);

		if (b_dev_info->adjust_managed_page_count)
			adjust_managed_page_count(page, 1);
	}

	b_dev_info->isolated_pages--;

	/* Free the now-deflated page we isolated in balloon_page_isolate(). */
	balloon_page_finalize(page);
	spin_unlock_irqrestore(&balloon_pages_lock, flags);

	put_page(page);

	return 0;
}

static const struct movable_operations balloon_mops = {
	.migrate_page = balloon_page_migrate,
	.isolate_page = balloon_page_isolate,
	.putback_page = balloon_page_putback,
};

static int __init balloon_init(void)
{
	return set_movable_ops(&balloon_mops, PGTY_offline);
}
core_initcall(balloon_init);

#endif /* CONFIG_BALLOON_MIGRATION */