forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
334 lines
8.1 KiB
334 lines
8.1 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Access to PCI I/O memory from user space programs. |
|
* |
|
* Copyright IBM Corp. 2014 |
|
* Author(s): Alexey Ishchuk <[email protected]> |
|
*/ |
|
#include <linux/kernel.h> |
|
#include <linux/syscalls.h> |
|
#include <linux/init.h> |
|
#include <linux/mm.h> |
|
#include <linux/errno.h> |
|
#include <linux/pci.h> |
|
#include <asm/pci_io.h> |
|
#include <asm/pci_debug.h> |
|
|
|
static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) |
|
{ |
|
struct { |
|
u64 offset; |
|
u8 cc; |
|
u8 status; |
|
} data = {offset, cc, status}; |
|
|
|
zpci_err_hex(&data, sizeof(data)); |
|
} |
|
|
|
static inline int __pcistb_mio_inuser( |
|
void __iomem *ioaddr, const void __user *src, |
|
u64 len, u8 *status) |
|
{ |
|
int cc = -ENXIO; |
|
|
|
asm volatile ( |
|
" sacf 256\n" |
|
"0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" |
|
"1: ipm %[cc]\n" |
|
" srl %[cc],28\n" |
|
"2: sacf 768\n" |
|
EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) |
|
: [cc] "+d" (cc), [len] "+d" (len) |
|
: [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) |
|
: "cc", "memory"); |
|
*status = len >> 24 & 0xff; |
|
return cc; |
|
} |
|
|
|
static inline int __pcistg_mio_inuser( |
|
void __iomem *ioaddr, const void __user *src, |
|
u64 ulen, u8 *status) |
|
{ |
|
union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
|
int cc = -ENXIO; |
|
u64 val = 0; |
|
u64 cnt = ulen; |
|
u8 tmp; |
|
|
|
/* |
|
* copy 0 < @len <= 8 bytes from @src into the right most bytes of |
|
* a register, then store it to PCI at @ioaddr while in secondary |
|
* address space. pcistg then uses the user mappings. |
|
*/ |
|
asm volatile ( |
|
" sacf 256\n" |
|
"0: llgc %[tmp],0(%[src])\n" |
|
" sllg %[val],%[val],8\n" |
|
" aghi %[src],1\n" |
|
" ogr %[val],%[tmp]\n" |
|
" brctg %[cnt],0b\n" |
|
"1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" |
|
"2: ipm %[cc]\n" |
|
" srl %[cc],28\n" |
|
"3: sacf 768\n" |
|
EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) |
|
: |
|
[src] "+a" (src), [cnt] "+d" (cnt), |
|
[val] "+d" (val), [tmp] "=d" (tmp), |
|
[cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) |
|
:: "cc", "memory"); |
|
*status = ioaddr_len.odd >> 24 & 0xff; |
|
|
|
/* did we read everything from user memory? */ |
|
if (!cc && cnt != 0) |
|
cc = -EFAULT; |
|
|
|
return cc; |
|
} |
|
|
|
static inline int __memcpy_toio_inuser(void __iomem *dst, |
|
const void __user *src, size_t n) |
|
{ |
|
int size, rc = 0; |
|
u8 status = 0; |
|
|
|
if (!src) |
|
return -EINVAL; |
|
|
|
while (n > 0) { |
|
size = zpci_get_max_write_size((u64 __force) dst, |
|
(u64 __force) src, n, |
|
ZPCI_MAX_WRITE_SIZE); |
|
if (size > 8) /* main path */ |
|
rc = __pcistb_mio_inuser(dst, src, size, &status); |
|
else |
|
rc = __pcistg_mio_inuser(dst, src, size, &status); |
|
if (rc) |
|
break; |
|
src += size; |
|
dst += size; |
|
n -= size; |
|
} |
|
if (rc) |
|
zpci_err_mmio(rc, status, (__force u64) dst); |
|
return rc; |
|
} |
|
|
|
SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, |
|
const void __user *, user_buffer, size_t, length) |
|
{ |
|
u8 local_buf[64]; |
|
void __iomem *io_addr; |
|
void *buf; |
|
struct vm_area_struct *vma; |
|
pte_t *ptep; |
|
spinlock_t *ptl; |
|
long ret; |
|
|
|
if (!zpci_is_enabled()) |
|
return -ENODEV; |
|
|
|
if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) |
|
return -EINVAL; |
|
|
|
/* |
|
* We only support write access to MIO capable devices if we are on |
|
* a MIO enabled system. Otherwise we would have to check for every |
|
* address if it is a special ZPCI_ADDR and would have to do |
|
* a pfn lookup which we don't need for MIO capable devices. Currently |
|
* ISM devices are the only devices without MIO support and there is no |
|
* known need for accessing these from userspace. |
|
*/ |
|
if (static_branch_likely(&have_mio)) { |
|
ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, |
|
user_buffer, |
|
length); |
|
return ret; |
|
} |
|
|
|
if (length > 64) { |
|
buf = kmalloc(length, GFP_KERNEL); |
|
if (!buf) |
|
return -ENOMEM; |
|
} else |
|
buf = local_buf; |
|
|
|
ret = -EFAULT; |
|
if (copy_from_user(buf, user_buffer, length)) |
|
goto out_free; |
|
|
|
mmap_read_lock(current->mm); |
|
ret = -EINVAL; |
|
vma = vma_lookup(current->mm, mmio_addr); |
|
if (!vma) |
|
goto out_unlock_mmap; |
|
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) |
|
goto out_unlock_mmap; |
|
ret = -EACCES; |
|
if (!(vma->vm_flags & VM_WRITE)) |
|
goto out_unlock_mmap; |
|
|
|
ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); |
|
if (ret) |
|
goto out_unlock_mmap; |
|
|
|
io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | |
|
(mmio_addr & ~PAGE_MASK)); |
|
|
|
if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) |
|
goto out_unlock_pt; |
|
|
|
ret = zpci_memcpy_toio(io_addr, buf, length); |
|
out_unlock_pt: |
|
pte_unmap_unlock(ptep, ptl); |
|
out_unlock_mmap: |
|
mmap_read_unlock(current->mm); |
|
out_free: |
|
if (buf != local_buf) |
|
kfree(buf); |
|
return ret; |
|
} |
|
|
|
static inline int __pcilg_mio_inuser( |
|
void __user *dst, const void __iomem *ioaddr, |
|
u64 ulen, u8 *status) |
|
{ |
|
union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
|
u64 cnt = ulen; |
|
int shift = ulen * 8; |
|
int cc = -ENXIO; |
|
u64 val, tmp; |
|
|
|
/* |
|
* read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in |
|
* user space) into a register using pcilg then store these bytes at |
|
* user address @dst |
|
*/ |
|
asm volatile ( |
|
" sacf 256\n" |
|
"0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" |
|
"1: ipm %[cc]\n" |
|
" srl %[cc],28\n" |
|
" ltr %[cc],%[cc]\n" |
|
" jne 4f\n" |
|
"2: ahi %[shift],-8\n" |
|
" srlg %[tmp],%[val],0(%[shift])\n" |
|
"3: stc %[tmp],0(%[dst])\n" |
|
" aghi %[dst],1\n" |
|
" brctg %[cnt],2b\n" |
|
"4: sacf 768\n" |
|
EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) |
|
: |
|
[ioaddr_len] "+&d" (ioaddr_len.pair), |
|
[cc] "+d" (cc), [val] "=d" (val), |
|
[dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), |
|
[shift] "+d" (shift) |
|
:: "cc", "memory"); |
|
|
|
/* did we write everything to the user space buffer? */ |
|
if (!cc && cnt != 0) |
|
cc = -EFAULT; |
|
|
|
*status = ioaddr_len.odd >> 24 & 0xff; |
|
return cc; |
|
} |
|
|
|
static inline int __memcpy_fromio_inuser(void __user *dst, |
|
const void __iomem *src, |
|
unsigned long n) |
|
{ |
|
int size, rc = 0; |
|
u8 status; |
|
|
|
while (n > 0) { |
|
size = zpci_get_max_write_size((u64 __force) src, |
|
(u64 __force) dst, n, |
|
ZPCI_MAX_READ_SIZE); |
|
rc = __pcilg_mio_inuser(dst, src, size, &status); |
|
if (rc) |
|
break; |
|
src += size; |
|
dst += size; |
|
n -= size; |
|
} |
|
if (rc) |
|
zpci_err_mmio(rc, status, (__force u64) dst); |
|
return rc; |
|
} |
|
|
|
SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, |
|
void __user *, user_buffer, size_t, length) |
|
{ |
|
u8 local_buf[64]; |
|
void __iomem *io_addr; |
|
void *buf; |
|
struct vm_area_struct *vma; |
|
pte_t *ptep; |
|
spinlock_t *ptl; |
|
long ret; |
|
|
|
if (!zpci_is_enabled()) |
|
return -ENODEV; |
|
|
|
if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) |
|
return -EINVAL; |
|
|
|
/* |
|
* We only support read access to MIO capable devices if we are on |
|
* a MIO enabled system. Otherwise we would have to check for every |
|
* address if it is a special ZPCI_ADDR and would have to do |
|
* a pfn lookup which we don't need for MIO capable devices. Currently |
|
* ISM devices are the only devices without MIO support and there is no |
|
* known need for accessing these from userspace. |
|
*/ |
|
if (static_branch_likely(&have_mio)) { |
|
ret = __memcpy_fromio_inuser( |
|
user_buffer, (const void __iomem *)mmio_addr, |
|
length); |
|
return ret; |
|
} |
|
|
|
if (length > 64) { |
|
buf = kmalloc(length, GFP_KERNEL); |
|
if (!buf) |
|
return -ENOMEM; |
|
} else { |
|
buf = local_buf; |
|
} |
|
|
|
mmap_read_lock(current->mm); |
|
ret = -EINVAL; |
|
vma = vma_lookup(current->mm, mmio_addr); |
|
if (!vma) |
|
goto out_unlock_mmap; |
|
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) |
|
goto out_unlock_mmap; |
|
ret = -EACCES; |
|
if (!(vma->vm_flags & VM_WRITE)) |
|
goto out_unlock_mmap; |
|
|
|
ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); |
|
if (ret) |
|
goto out_unlock_mmap; |
|
|
|
io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | |
|
(mmio_addr & ~PAGE_MASK)); |
|
|
|
if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { |
|
ret = -EFAULT; |
|
goto out_unlock_pt; |
|
} |
|
ret = zpci_memcpy_fromio(buf, io_addr, length); |
|
|
|
out_unlock_pt: |
|
pte_unmap_unlock(ptep, ptl); |
|
out_unlock_mmap: |
|
mmap_read_unlock(current->mm); |
|
|
|
if (!ret && copy_to_user(user_buffer, buf, length)) |
|
ret = -EFAULT; |
|
|
|
if (buf != local_buf) |
|
kfree(buf); |
|
return ret; |
|
}
|
|
|