forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
560 lines
14 KiB
560 lines
14 KiB
// SPDX-License-Identifier: GPL-2.0 OR MIT |
|
|
|
/* |
|
* Xen frontend/backend page directory based shared buffer |
|
* helper module. |
|
* |
|
* Copyright (C) 2018 EPAM Systems Inc. |
|
* |
|
* Author: Oleksandr Andrushchenko <[email protected]> |
|
*/ |
|
|
|
#include <linux/module.h> |
|
#include <linux/errno.h> |
|
#include <linux/mm.h> |
|
|
|
#include <asm/xen/hypervisor.h> |
|
#include <xen/balloon.h> |
|
#include <xen/xen.h> |
|
#include <xen/xenbus.h> |
|
#include <xen/interface/io/ring.h> |
|
|
|
#include <xen/xen-front-pgdir-shbuf.h> |
|
|
|
#ifndef GRANT_INVALID_REF |
|
/* |
|
* FIXME: usage of grant reference 0 as invalid grant reference: |
|
* grant reference 0 is valid, but never exposed to a PV driver, |
|
* because of the fact it is already in use/reserved by the PV console. |
|
*/ |
|
#define GRANT_INVALID_REF 0 |
|
#endif |
|
|
|
/** |
|
* This structure represents the structure of a shared page |
|
* that contains grant references to the pages of the shared |
|
* buffer. This structure is common to many Xen para-virtualized |
|
* protocols at include/xen/interface/io/ |
|
*/ |
|
struct xen_page_directory { |
|
grant_ref_t gref_dir_next_page; |
|
grant_ref_t gref[1]; /* Variable length */ |
|
}; |
|
|
|
/** |
|
* Shared buffer ops which are differently implemented |
|
* depending on the allocation mode, e.g. if the buffer |
|
* is allocated by the corresponding backend or frontend. |
|
* Some of the operations. |
|
*/ |
|
struct xen_front_pgdir_shbuf_ops { |
|
/* |
|
* Calculate number of grefs required to handle this buffer, |
|
* e.g. if grefs are required for page directory only or the buffer |
|
* pages as well. |
|
*/ |
|
void (*calc_num_grefs)(struct xen_front_pgdir_shbuf *buf); |
|
|
|
/* Fill page directory according to para-virtual display protocol. */ |
|
void (*fill_page_dir)(struct xen_front_pgdir_shbuf *buf); |
|
|
|
/* Claim grant references for the pages of the buffer. */ |
|
int (*grant_refs_for_buffer)(struct xen_front_pgdir_shbuf *buf, |
|
grant_ref_t *priv_gref_head, int gref_idx); |
|
|
|
/* Map grant references of the buffer. */ |
|
int (*map)(struct xen_front_pgdir_shbuf *buf); |
|
|
|
/* Unmap grant references of the buffer. */ |
|
int (*unmap)(struct xen_front_pgdir_shbuf *buf); |
|
}; |
|
|
|
/** |
|
* Get granted reference to the very first page of the |
|
* page directory. Usually this is passed to the backend, |
|
* so it can find/fill the grant references to the buffer's |
|
* pages. |
|
* |
|
* \param buf shared buffer which page directory is of interest. |
|
* \return granted reference to the very first page of the |
|
* page directory. |
|
*/ |
|
grant_ref_t |
|
xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
if (!buf->grefs) |
|
return GRANT_INVALID_REF; |
|
|
|
return buf->grefs[0]; |
|
} |
|
EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start); |
|
|
|
/** |
|
* Map granted references of the shared buffer. |
|
* |
|
* Depending on the shared buffer mode of allocation |
|
* (be_alloc flag) this can either do nothing (for buffers |
|
* shared by the frontend itself) or map the provided granted |
|
* references onto the backing storage (buf->pages). |
|
* |
|
* \param buf shared buffer which grants to be maped. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
if (buf->ops && buf->ops->map) |
|
return buf->ops->map(buf); |
|
|
|
/* No need to map own grant references. */ |
|
return 0; |
|
} |
|
EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map); |
|
|
|
/** |
|
* Unmap granted references of the shared buffer. |
|
* |
|
* Depending on the shared buffer mode of allocation |
|
* (be_alloc flag) this can either do nothing (for buffers |
|
* shared by the frontend itself) or unmap the provided granted |
|
* references. |
|
* |
|
* \param buf shared buffer which grants to be unmaped. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
if (buf->ops && buf->ops->unmap) |
|
return buf->ops->unmap(buf); |
|
|
|
/* No need to unmap own grant references. */ |
|
return 0; |
|
} |
|
EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap); |
|
|
|
/** |
|
* Free all the resources of the shared buffer. |
|
* |
|
* \param buf shared buffer which resources to be freed. |
|
*/ |
|
void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
if (buf->grefs) { |
|
int i; |
|
|
|
for (i = 0; i < buf->num_grefs; i++) |
|
if (buf->grefs[i] != GRANT_INVALID_REF) |
|
gnttab_end_foreign_access(buf->grefs[i], |
|
0, 0UL); |
|
} |
|
kfree(buf->grefs); |
|
kfree(buf->directory); |
|
} |
|
EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free); |
|
|
|
/* |
|
* Number of grefs a page can hold with respect to the |
|
* struct xen_page_directory header. |
|
*/ |
|
#define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \ |
|
offsetof(struct xen_page_directory, \ |
|
gref)) / sizeof(grant_ref_t)) |
|
|
|
/** |
|
* Get the number of pages the page directory consumes itself. |
|
* |
|
* \param buf shared buffer. |
|
*/ |
|
static int get_num_pages_dir(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
return DIV_ROUND_UP(buf->num_pages, XEN_NUM_GREFS_PER_PAGE); |
|
} |
|
|
|
/** |
|
* Calculate the number of grant references needed to share the buffer |
|
* and its pages when backend allocates the buffer. |
|
* |
|
* \param buf shared buffer. |
|
*/ |
|
static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
/* Only for pages the page directory consumes itself. */ |
|
buf->num_grefs = get_num_pages_dir(buf); |
|
} |
|
|
|
/** |
|
* Calculate the number of grant references needed to share the buffer |
|
* and its pages when frontend allocates the buffer. |
|
* |
|
* \param buf shared buffer. |
|
*/ |
|
static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
/* |
|
* Number of pages the page directory consumes itself |
|
* plus grefs for the buffer pages. |
|
*/ |
|
buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages; |
|
} |
|
|
|
#define xen_page_to_vaddr(page) \ |
|
((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page))) |
|
|
|
/** |
|
* Unmap the buffer previously mapped with grant references |
|
* provided by the backend. |
|
* |
|
* \param buf shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
static int backend_unmap(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
struct gnttab_unmap_grant_ref *unmap_ops; |
|
int i, ret; |
|
|
|
if (!buf->pages || !buf->backend_map_handles || !buf->grefs) |
|
return 0; |
|
|
|
unmap_ops = kcalloc(buf->num_pages, sizeof(*unmap_ops), |
|
GFP_KERNEL); |
|
if (!unmap_ops) |
|
return -ENOMEM; |
|
|
|
for (i = 0; i < buf->num_pages; i++) { |
|
phys_addr_t addr; |
|
|
|
addr = xen_page_to_vaddr(buf->pages[i]); |
|
gnttab_set_unmap_op(&unmap_ops[i], addr, GNTMAP_host_map, |
|
buf->backend_map_handles[i]); |
|
} |
|
|
|
ret = gnttab_unmap_refs(unmap_ops, NULL, buf->pages, |
|
buf->num_pages); |
|
|
|
for (i = 0; i < buf->num_pages; i++) { |
|
if (unlikely(unmap_ops[i].status != GNTST_okay)) |
|
dev_err(&buf->xb_dev->dev, |
|
"Failed to unmap page %d: %d\n", |
|
i, unmap_ops[i].status); |
|
} |
|
|
|
if (ret) |
|
dev_err(&buf->xb_dev->dev, |
|
"Failed to unmap grant references, ret %d", ret); |
|
|
|
kfree(unmap_ops); |
|
kfree(buf->backend_map_handles); |
|
buf->backend_map_handles = NULL; |
|
return ret; |
|
} |
|
|
|
/** |
|
* Map the buffer with grant references provided by the backend. |
|
* |
|
* \param buf shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
static int backend_map(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
struct gnttab_map_grant_ref *map_ops = NULL; |
|
unsigned char *ptr; |
|
int ret, cur_gref, cur_dir_page, cur_page, grefs_left; |
|
|
|
map_ops = kcalloc(buf->num_pages, sizeof(*map_ops), GFP_KERNEL); |
|
if (!map_ops) |
|
return -ENOMEM; |
|
|
|
buf->backend_map_handles = kcalloc(buf->num_pages, |
|
sizeof(*buf->backend_map_handles), |
|
GFP_KERNEL); |
|
if (!buf->backend_map_handles) { |
|
kfree(map_ops); |
|
return -ENOMEM; |
|
} |
|
|
|
/* |
|
* Read page directory to get grefs from the backend: for external |
|
* buffer we only allocate buf->grefs for the page directory, |
|
* so buf->num_grefs has number of pages in the page directory itself. |
|
*/ |
|
ptr = buf->directory; |
|
grefs_left = buf->num_pages; |
|
cur_page = 0; |
|
for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) { |
|
struct xen_page_directory *page_dir = |
|
(struct xen_page_directory *)ptr; |
|
int to_copy = XEN_NUM_GREFS_PER_PAGE; |
|
|
|
if (to_copy > grefs_left) |
|
to_copy = grefs_left; |
|
|
|
for (cur_gref = 0; cur_gref < to_copy; cur_gref++) { |
|
phys_addr_t addr; |
|
|
|
addr = xen_page_to_vaddr(buf->pages[cur_page]); |
|
gnttab_set_map_op(&map_ops[cur_page], addr, |
|
GNTMAP_host_map, |
|
page_dir->gref[cur_gref], |
|
buf->xb_dev->otherend_id); |
|
cur_page++; |
|
} |
|
|
|
grefs_left -= to_copy; |
|
ptr += PAGE_SIZE; |
|
} |
|
ret = gnttab_map_refs(map_ops, NULL, buf->pages, buf->num_pages); |
|
|
|
/* Save handles even if error, so we can unmap. */ |
|
for (cur_page = 0; cur_page < buf->num_pages; cur_page++) { |
|
if (likely(map_ops[cur_page].status == GNTST_okay)) { |
|
buf->backend_map_handles[cur_page] = |
|
map_ops[cur_page].handle; |
|
} else { |
|
buf->backend_map_handles[cur_page] = |
|
INVALID_GRANT_HANDLE; |
|
if (!ret) |
|
ret = -ENXIO; |
|
dev_err(&buf->xb_dev->dev, |
|
"Failed to map page %d: %d\n", |
|
cur_page, map_ops[cur_page].status); |
|
} |
|
} |
|
|
|
if (ret) { |
|
dev_err(&buf->xb_dev->dev, |
|
"Failed to map grant references, ret %d", ret); |
|
backend_unmap(buf); |
|
} |
|
|
|
kfree(map_ops); |
|
return ret; |
|
} |
|
|
|
/** |
|
* Fill page directory with grant references to the pages of the |
|
* page directory itself. |
|
* |
|
* The grant references to the buffer pages are provided by the |
|
* backend in this case. |
|
* |
|
* \param buf shared buffer. |
|
*/ |
|
static void backend_fill_page_dir(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
struct xen_page_directory *page_dir; |
|
unsigned char *ptr; |
|
int i, num_pages_dir; |
|
|
|
ptr = buf->directory; |
|
num_pages_dir = get_num_pages_dir(buf); |
|
|
|
/* Fill only grefs for the page directory itself. */ |
|
for (i = 0; i < num_pages_dir - 1; i++) { |
|
page_dir = (struct xen_page_directory *)ptr; |
|
|
|
page_dir->gref_dir_next_page = buf->grefs[i + 1]; |
|
ptr += PAGE_SIZE; |
|
} |
|
/* Last page must say there is no more pages. */ |
|
page_dir = (struct xen_page_directory *)ptr; |
|
page_dir->gref_dir_next_page = GRANT_INVALID_REF; |
|
} |
|
|
|
/** |
|
* Fill page directory with grant references to the pages of the |
|
* page directory and the buffer we share with the backend. |
|
* |
|
* \param buf shared buffer. |
|
*/ |
|
static void guest_fill_page_dir(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
unsigned char *ptr; |
|
int cur_gref, grefs_left, to_copy, i, num_pages_dir; |
|
|
|
ptr = buf->directory; |
|
num_pages_dir = get_num_pages_dir(buf); |
|
|
|
/* |
|
* While copying, skip grefs at start, they are for pages |
|
* granted for the page directory itself. |
|
*/ |
|
cur_gref = num_pages_dir; |
|
grefs_left = buf->num_pages; |
|
for (i = 0; i < num_pages_dir; i++) { |
|
struct xen_page_directory *page_dir = |
|
(struct xen_page_directory *)ptr; |
|
|
|
if (grefs_left <= XEN_NUM_GREFS_PER_PAGE) { |
|
to_copy = grefs_left; |
|
page_dir->gref_dir_next_page = GRANT_INVALID_REF; |
|
} else { |
|
to_copy = XEN_NUM_GREFS_PER_PAGE; |
|
page_dir->gref_dir_next_page = buf->grefs[i + 1]; |
|
} |
|
memcpy(&page_dir->gref, &buf->grefs[cur_gref], |
|
to_copy * sizeof(grant_ref_t)); |
|
ptr += PAGE_SIZE; |
|
grefs_left -= to_copy; |
|
cur_gref += to_copy; |
|
} |
|
} |
|
|
|
/** |
|
* Grant references to the frontend's buffer pages. |
|
* |
|
* These will be shared with the backend, so it can |
|
* access the buffer's data. |
|
* |
|
* \param buf shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf *buf, |
|
grant_ref_t *priv_gref_head, |
|
int gref_idx) |
|
{ |
|
int i, cur_ref, otherend_id; |
|
|
|
otherend_id = buf->xb_dev->otherend_id; |
|
for (i = 0; i < buf->num_pages; i++) { |
|
cur_ref = gnttab_claim_grant_reference(priv_gref_head); |
|
if (cur_ref < 0) |
|
return cur_ref; |
|
|
|
gnttab_grant_foreign_access_ref(cur_ref, otherend_id, |
|
xen_page_to_gfn(buf->pages[i]), |
|
0); |
|
buf->grefs[gref_idx++] = cur_ref; |
|
} |
|
return 0; |
|
} |
|
|
|
/** |
|
* Grant all the references needed to share the buffer. |
|
* |
|
* Grant references to the page directory pages and, if |
|
* needed, also to the pages of the shared buffer data. |
|
* |
|
* \param buf shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
static int grant_references(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
grant_ref_t priv_gref_head; |
|
int ret, i, j, cur_ref; |
|
int otherend_id, num_pages_dir; |
|
|
|
ret = gnttab_alloc_grant_references(buf->num_grefs, &priv_gref_head); |
|
if (ret < 0) { |
|
dev_err(&buf->xb_dev->dev, |
|
"Cannot allocate grant references\n"); |
|
return ret; |
|
} |
|
|
|
otherend_id = buf->xb_dev->otherend_id; |
|
j = 0; |
|
num_pages_dir = get_num_pages_dir(buf); |
|
for (i = 0; i < num_pages_dir; i++) { |
|
unsigned long frame; |
|
|
|
cur_ref = gnttab_claim_grant_reference(&priv_gref_head); |
|
if (cur_ref < 0) |
|
return cur_ref; |
|
|
|
frame = xen_page_to_gfn(virt_to_page(buf->directory + |
|
PAGE_SIZE * i)); |
|
gnttab_grant_foreign_access_ref(cur_ref, otherend_id, frame, 0); |
|
buf->grefs[j++] = cur_ref; |
|
} |
|
|
|
if (buf->ops->grant_refs_for_buffer) { |
|
ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j); |
|
if (ret) |
|
return ret; |
|
} |
|
|
|
gnttab_free_grant_references(priv_gref_head); |
|
return 0; |
|
} |
|
|
|
/** |
|
* Allocate all required structures to mange shared buffer. |
|
* |
|
* \param buf shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
static int alloc_storage(struct xen_front_pgdir_shbuf *buf) |
|
{ |
|
buf->grefs = kcalloc(buf->num_grefs, sizeof(*buf->grefs), GFP_KERNEL); |
|
if (!buf->grefs) |
|
return -ENOMEM; |
|
|
|
buf->directory = kcalloc(get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL); |
|
if (!buf->directory) |
|
return -ENOMEM; |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* For backend allocated buffers we don't need grant_refs_for_buffer |
|
* as those grant references are allocated at backend side. |
|
*/ |
|
static const struct xen_front_pgdir_shbuf_ops backend_ops = { |
|
.calc_num_grefs = backend_calc_num_grefs, |
|
.fill_page_dir = backend_fill_page_dir, |
|
.map = backend_map, |
|
.unmap = backend_unmap |
|
}; |
|
|
|
/* |
|
* For locally granted references we do not need to map/unmap |
|
* the references. |
|
*/ |
|
static const struct xen_front_pgdir_shbuf_ops local_ops = { |
|
.calc_num_grefs = guest_calc_num_grefs, |
|
.fill_page_dir = guest_fill_page_dir, |
|
.grant_refs_for_buffer = guest_grant_refs_for_buffer, |
|
}; |
|
|
|
/** |
|
* Allocate a new instance of a shared buffer. |
|
* |
|
* \param cfg configuration to be used while allocating a new shared buffer. |
|
* \return zero on success or a negative number on failure. |
|
*/ |
|
int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg *cfg) |
|
{ |
|
struct xen_front_pgdir_shbuf *buf = cfg->pgdir; |
|
int ret; |
|
|
|
if (cfg->be_alloc) |
|
buf->ops = &backend_ops; |
|
else |
|
buf->ops = &local_ops; |
|
buf->xb_dev = cfg->xb_dev; |
|
buf->num_pages = cfg->num_pages; |
|
buf->pages = cfg->pages; |
|
|
|
buf->ops->calc_num_grefs(buf); |
|
|
|
ret = alloc_storage(buf); |
|
if (ret) |
|
goto fail; |
|
|
|
ret = grant_references(buf); |
|
if (ret) |
|
goto fail; |
|
|
|
buf->ops->fill_page_dir(buf); |
|
|
|
return 0; |
|
|
|
fail: |
|
xen_front_pgdir_shbuf_free(buf); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc); |
|
|
|
MODULE_DESCRIPTION("Xen frontend/backend page directory based " |
|
"shared buffer handling"); |
|
MODULE_AUTHOR("Oleksandr Andrushchenko"); |
|
MODULE_LICENSE("GPL");
|
|
|