forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
156 lines
3.5 KiB
156 lines
3.5 KiB
#include <sys/types.h> |
|
#include <sys/stat.h> |
|
#include <sys/mman.h> |
|
#include <unistd.h> |
|
#include <errno.h> |
|
#include <string.h> |
|
|
|
#include "liburing.h" |
|
#include "barrier.h" |
|
|
|
static int __io_uring_get_cqe(struct io_uring *ring, |
|
struct io_uring_cqe **cqe_ptr, int wait) |
|
{ |
|
struct io_uring_cq *cq = &ring->cq; |
|
const unsigned mask = *cq->kring_mask; |
|
unsigned head; |
|
int ret; |
|
|
|
*cqe_ptr = NULL; |
|
head = *cq->khead; |
|
do { |
|
/* |
|
* It's necessary to use a read_barrier() before reading |
|
* the CQ tail, since the kernel updates it locklessly. The |
|
* kernel has the matching store barrier for the update. The |
|
* kernel also ensures that previous stores to CQEs are ordered |
|
* with the tail update. |
|
*/ |
|
read_barrier(); |
|
if (head != *cq->ktail) { |
|
*cqe_ptr = &cq->cqes[head & mask]; |
|
break; |
|
} |
|
if (!wait) |
|
break; |
|
ret = io_uring_enter(ring->ring_fd, 0, 1, |
|
IORING_ENTER_GETEVENTS, NULL); |
|
if (ret < 0) |
|
return -errno; |
|
} while (1); |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* Return an IO completion, if one is readily available. Returns 0 with |
|
* cqe_ptr filled in on success, -errno on failure. |
|
*/ |
|
int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
|
{ |
|
return __io_uring_get_cqe(ring, cqe_ptr, 0); |
|
} |
|
|
|
/* |
|
* Return an IO completion, waiting for it if necessary. Returns 0 with |
|
* cqe_ptr filled in on success, -errno on failure. |
|
*/ |
|
int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr) |
|
{ |
|
return __io_uring_get_cqe(ring, cqe_ptr, 1); |
|
} |
|
|
|
/* |
|
* Submit sqes acquired from io_uring_get_sqe() to the kernel. |
|
* |
|
* Returns number of sqes submitted |
|
*/ |
|
int io_uring_submit(struct io_uring *ring) |
|
{ |
|
struct io_uring_sq *sq = &ring->sq; |
|
const unsigned mask = *sq->kring_mask; |
|
unsigned ktail, ktail_next, submitted, to_submit; |
|
int ret; |
|
|
|
/* |
|
* If we have pending IO in the kring, submit it first. We need a |
|
* read barrier here to match the kernels store barrier when updating |
|
* the SQ head. |
|
*/ |
|
read_barrier(); |
|
if (*sq->khead != *sq->ktail) { |
|
submitted = *sq->kring_entries; |
|
goto submit; |
|
} |
|
|
|
if (sq->sqe_head == sq->sqe_tail) |
|
return 0; |
|
|
|
/* |
|
* Fill in sqes that we have queued up, adding them to the kernel ring |
|
*/ |
|
submitted = 0; |
|
ktail = ktail_next = *sq->ktail; |
|
to_submit = sq->sqe_tail - sq->sqe_head; |
|
while (to_submit--) { |
|
ktail_next++; |
|
read_barrier(); |
|
|
|
sq->array[ktail & mask] = sq->sqe_head & mask; |
|
ktail = ktail_next; |
|
|
|
sq->sqe_head++; |
|
submitted++; |
|
} |
|
|
|
if (!submitted) |
|
return 0; |
|
|
|
if (*sq->ktail != ktail) { |
|
/* |
|
* First write barrier ensures that the SQE stores are updated |
|
* with the tail update. This is needed so that the kernel |
|
* will never see a tail update without the preceeding sQE |
|
* stores being done. |
|
*/ |
|
write_barrier(); |
|
*sq->ktail = ktail; |
|
/* |
|
* The kernel has the matching read barrier for reading the |
|
* SQ tail. |
|
*/ |
|
write_barrier(); |
|
} |
|
|
|
submit: |
|
ret = io_uring_enter(ring->ring_fd, submitted, 0, |
|
IORING_ENTER_GETEVENTS, NULL); |
|
if (ret < 0) |
|
return -errno; |
|
|
|
return ret; |
|
} |
|
|
|
/* |
|
* Return an sqe to fill. Application must later call io_uring_submit() |
|
* when it's ready to tell the kernel about it. The caller may call this |
|
* function multiple times before calling io_uring_submit(). |
|
* |
|
* Returns a vacant sqe, or NULL if we're full. |
|
*/ |
|
struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) |
|
{ |
|
struct io_uring_sq *sq = &ring->sq; |
|
unsigned next = sq->sqe_tail + 1; |
|
struct io_uring_sqe *sqe; |
|
|
|
/* |
|
* All sqes are used |
|
*/ |
|
if (next - sq->sqe_head > *sq->kring_entries) |
|
return NULL; |
|
|
|
sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask]; |
|
sq->sqe_tail = next; |
|
return sqe; |
|
}
|
|
|