|
|
// SPDX-License-Identifier: GPL-2.0 |
|
|
/* |
|
|
* linux/arch/alpha/kernel/pci_iommu.c |
|
|
*/ |
|
|
|
|
|
#include <linux/kernel.h> |
|
|
#include <linux/mm.h> |
|
|
#include <linux/pci.h> |
|
|
#include <linux/gfp.h> |
|
|
#include <linux/memblock.h> |
|
|
#include <linux/export.h> |
|
|
#include <linux/scatterlist.h> |
|
|
#include <linux/log2.h> |
|
|
#include <linux/dma-map-ops.h> |
|
|
#include <linux/iommu-helper.h> |
|
|
|
|
|
#include <asm/io.h> |
|
|
#include <asm/hwrpb.h> |
|
|
|
|
|
#include "proto.h" |
|
|
#include "pci_impl.h" |
|
|
|
|
|
|
|
|
#define DEBUG_ALLOC 0 |
|
|
#if DEBUG_ALLOC > 0 |
|
|
# define DBGA(args...) printk(KERN_DEBUG args) |
|
|
#else |
|
|
# define DBGA(args...) |
|
|
#endif |
|
|
#if DEBUG_ALLOC > 1 |
|
|
# define DBGA2(args...) printk(KERN_DEBUG args) |
|
|
#else |
|
|
# define DBGA2(args...) |
|
|
#endif |
|
|
|
|
|
#define DEBUG_NODIRECT 0 |
|
|
|
|
|
#define ISA_DMA_MASK 0x00ffffff |
|
|
|
|
|
static inline unsigned long |
|
|
mk_iommu_pte(unsigned long paddr) |
|
|
{ |
|
|
return (paddr >> (PAGE_SHIFT-1)) | 1; |
|
|
} |
|
|
|
|
|
/* Return the minimum of MAX or the first power of two larger |
|
|
than main memory. */ |
|
|
|
|
|
unsigned long |
|
|
size_for_memory(unsigned long max) |
|
|
{ |
|
|
unsigned long mem = max_low_pfn << PAGE_SHIFT; |
|
|
if (mem < max) |
|
|
max = roundup_pow_of_two(mem); |
|
|
return max; |
|
|
} |
|
|
|
|
|
struct pci_iommu_arena * __init |
|
|
iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, |
|
|
unsigned long window_size, unsigned long align) |
|
|
{ |
|
|
unsigned long mem_size; |
|
|
struct pci_iommu_arena *arena; |
|
|
|
|
|
mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long)); |
|
|
|
|
|
/* Note that the TLB lookup logic uses bitwise concatenation, |
|
|
not addition, so the required arena alignment is based on |
|
|
the size of the window. Retain the align parameter so that |
|
|
particular systems can over-align the arena. */ |
|
|
if (align < mem_size) |
|
|
align = mem_size; |
|
|
|
|
|
|
|
|
#ifdef CONFIG_DISCONTIGMEM |
|
|
|
|
|
arena = memblock_alloc_node(sizeof(*arena), align, nid); |
|
|
if (!NODE_DATA(nid) || !arena) { |
|
|
printk("%s: couldn't allocate arena from node %d\n" |
|
|
" falling back to system-wide allocation\n", |
|
|
__func__, nid); |
|
|
arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); |
|
|
if (!arena) |
|
|
panic("%s: Failed to allocate %zu bytes\n", __func__, |
|
|
sizeof(*arena)); |
|
|
} |
|
|
|
|
|
arena->ptes = memblock_alloc_node(sizeof(*arena), align, nid); |
|
|
if (!NODE_DATA(nid) || !arena->ptes) { |
|
|
printk("%s: couldn't allocate arena ptes from node %d\n" |
|
|
" falling back to system-wide allocation\n", |
|
|
__func__, nid); |
|
|
arena->ptes = memblock_alloc(mem_size, align); |
|
|
if (!arena->ptes) |
|
|
panic("%s: Failed to allocate %lu bytes align=0x%lx\n", |
|
|
__func__, mem_size, align); |
|
|
} |
|
|
|
|
|
#else /* CONFIG_DISCONTIGMEM */ |
|
|
|
|
|
arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); |
|
|
if (!arena) |
|
|
panic("%s: Failed to allocate %zu bytes\n", __func__, |
|
|
sizeof(*arena)); |
|
|
arena->ptes = memblock_alloc(mem_size, align); |
|
|
if (!arena->ptes) |
|
|
panic("%s: Failed to allocate %lu bytes align=0x%lx\n", |
|
|
__func__, mem_size, align); |
|
|
|
|
|
#endif /* CONFIG_DISCONTIGMEM */ |
|
|
|
|
|
spin_lock_init(&arena->lock); |
|
|
arena->hose = hose; |
|
|
arena->dma_base = base; |
|
|
arena->size = window_size; |
|
|
arena->next_entry = 0; |
|
|
|
|
|
/* Align allocations to a multiple of a page size. Not needed |
|
|
unless there are chip bugs. */ |
|
|
arena->align_entry = 1; |
|
|
|
|
|
return arena; |
|
|
} |
|
|
|
|
|
struct pci_iommu_arena * __init |
|
|
iommu_arena_new(struct pci_controller *hose, dma_addr_t base, |
|
|
unsigned long window_size, unsigned long align) |
|
|
{ |
|
|
return iommu_arena_new_node(0, hose, base, window_size, align); |
|
|
} |
|
|
|
|
|
/* Must be called with the arena lock held */ |
|
|
static long |
|
|
iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena, |
|
|
long n, long mask) |
|
|
{ |
|
|
unsigned long *ptes; |
|
|
long i, p, nent; |
|
|
int pass = 0; |
|
|
unsigned long base; |
|
|
unsigned long boundary_size; |
|
|
|
|
|
base = arena->dma_base >> PAGE_SHIFT; |
|
|
boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT); |
|
|
|
|
|
/* Search forward for the first mask-aligned sequence of N free ptes */ |
|
|
ptes = arena->ptes; |
|
|
nent = arena->size >> PAGE_SHIFT; |
|
|
p = ALIGN(arena->next_entry, mask + 1); |
|
|
i = 0; |
|
|
|
|
|
again: |
|
|
while (i < n && p+i < nent) { |
|
|
if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) { |
|
|
p = ALIGN(p + 1, mask + 1); |
|
|
goto again; |
|
|
} |
|
|
|
|
|
if (ptes[p+i]) |
|
|
p = ALIGN(p + i + 1, mask + 1), i = 0; |
|
|
else |
|
|
i = i + 1; |
|
|
} |
|
|
|
|
|
if (i < n) { |
|
|
if (pass < 1) { |
|
|
/* |
|
|
* Reached the end. Flush the TLB and restart |
|
|
* the search from the beginning. |
|
|
*/ |
|
|
alpha_mv.mv_pci_tbi(arena->hose, 0, -1); |
|
|
|
|
|
pass++; |
|
|
p = 0; |
|
|
i = 0; |
|
|
goto again; |
|
|
} else |
|
|
return -1; |
|
|
} |
|
|
|
|
|
/* Success. It's the responsibility of the caller to mark them |
|
|
in use before releasing the lock */ |
|
|
return p; |
|
|
} |
|
|
|
|
|
static long |
|
|
iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n, |
|
|
unsigned int align) |
|
|
{ |
|
|
unsigned long flags; |
|
|
unsigned long *ptes; |
|
|
long i, p, mask; |
|
|
|
|
|
spin_lock_irqsave(&arena->lock, flags); |
|
|
|
|
|
/* Search for N empty ptes */ |
|
|
ptes = arena->ptes; |
|
|
mask = max(align, arena->align_entry) - 1; |
|
|
p = iommu_arena_find_pages(dev, arena, n, mask); |
|
|
if (p < 0) { |
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
return -1; |
|
|
} |
|
|
|
|
|
/* Success. Mark them all in use, ie not zero and invalid |
|
|
for the iommu tlb that could load them from under us. |
|
|
The chip specific bits will fill this in with something |
|
|
kosher when we return. */ |
|
|
for (i = 0; i < n; ++i) |
|
|
ptes[p+i] = IOMMU_INVALID_PTE; |
|
|
|
|
|
arena->next_entry = p + n; |
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
|
|
|
return p; |
|
|
} |
|
|
|
|
|
static void |
|
|
iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n) |
|
|
{ |
|
|
unsigned long *p; |
|
|
long i; |
|
|
|
|
|
p = arena->ptes + ofs; |
|
|
for (i = 0; i < n; ++i) |
|
|
p[i] = 0; |
|
|
} |
|
|
|
|
|
/* |
|
|
* True if the machine supports DAC addressing, and DEV can |
|
|
* make use of it given MASK. |
|
|
*/ |
|
|
static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask) |
|
|
{ |
|
|
dma_addr_t dac_offset = alpha_mv.pci_dac_offset; |
|
|
int ok = 1; |
|
|
|
|
|
/* If this is not set, the machine doesn't support DAC at all. */ |
|
|
if (dac_offset == 0) |
|
|
ok = 0; |
|
|
|
|
|
/* The device has to be able to address our DAC bit. */ |
|
|
if ((dac_offset & dev->dma_mask) != dac_offset) |
|
|
ok = 0; |
|
|
|
|
|
/* If both conditions above are met, we are fine. */ |
|
|
DBGA("pci_dac_dma_supported %s from %ps\n", |
|
|
ok ? "yes" : "no", __builtin_return_address(0)); |
|
|
|
|
|
return ok; |
|
|
} |
|
|
|
|
|
/* Map a single buffer of the indicated size for PCI DMA in streaming |
|
|
mode. The 32-bit PCI bus mastering address to use is returned. |
|
|
Once the device is given the dma address, the device owns this memory |
|
|
until either pci_unmap_single or pci_dma_sync_single is performed. */ |
|
|
|
|
|
static dma_addr_t |
|
|
pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, |
|
|
int dac_allowed) |
|
|
{ |
|
|
struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; |
|
|
dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
|
|
struct pci_iommu_arena *arena; |
|
|
long npages, dma_ofs, i; |
|
|
unsigned long paddr; |
|
|
dma_addr_t ret; |
|
|
unsigned int align = 0; |
|
|
struct device *dev = pdev ? &pdev->dev : NULL; |
|
|
|
|
|
paddr = __pa(cpu_addr); |
|
|
|
|
|
#if !DEBUG_NODIRECT |
|
|
/* First check to see if we can use the direct map window. */ |
|
|
if (paddr + size + __direct_map_base - 1 <= max_dma |
|
|
&& paddr + size <= __direct_map_size) { |
|
|
ret = paddr + __direct_map_base; |
|
|
|
|
|
DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %ps\n", |
|
|
cpu_addr, size, ret, __builtin_return_address(0)); |
|
|
|
|
|
return ret; |
|
|
} |
|
|
#endif |
|
|
|
|
|
/* Next, use DAC if selected earlier. */ |
|
|
if (dac_allowed) { |
|
|
ret = paddr + alpha_mv.pci_dac_offset; |
|
|
|
|
|
DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %ps\n", |
|
|
cpu_addr, size, ret, __builtin_return_address(0)); |
|
|
|
|
|
return ret; |
|
|
} |
|
|
|
|
|
/* If the machine doesn't define a pci_tbi routine, we have to |
|
|
assume it doesn't support sg mapping, and, since we tried to |
|
|
use direct_map above, it now must be considered an error. */ |
|
|
if (! alpha_mv.mv_pci_tbi) { |
|
|
printk_once(KERN_WARNING "pci_map_single: no HW sg\n"); |
|
|
return DMA_MAPPING_ERROR; |
|
|
} |
|
|
|
|
|
arena = hose->sg_pci; |
|
|
if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
|
|
arena = hose->sg_isa; |
|
|
|
|
|
npages = iommu_num_pages(paddr, size, PAGE_SIZE); |
|
|
|
|
|
/* Force allocation to 64KB boundary for ISA bridges. */ |
|
|
if (pdev && pdev == isa_bridge) |
|
|
align = 8; |
|
|
dma_ofs = iommu_arena_alloc(dev, arena, npages, align); |
|
|
if (dma_ofs < 0) { |
|
|
printk(KERN_WARNING "pci_map_single failed: " |
|
|
"could not allocate dma page tables\n"); |
|
|
return DMA_MAPPING_ERROR; |
|
|
} |
|
|
|
|
|
paddr &= PAGE_MASK; |
|
|
for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) |
|
|
arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr); |
|
|
|
|
|
ret = arena->dma_base + dma_ofs * PAGE_SIZE; |
|
|
ret += (unsigned long)cpu_addr & ~PAGE_MASK; |
|
|
|
|
|
DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %ps\n", |
|
|
cpu_addr, size, npages, ret, __builtin_return_address(0)); |
|
|
|
|
|
return ret; |
|
|
} |
|
|
|
|
|
/* Helper for generic DMA-mapping functions. */ |
|
|
static struct pci_dev *alpha_gendev_to_pci(struct device *dev) |
|
|
{ |
|
|
if (dev && dev_is_pci(dev)) |
|
|
return to_pci_dev(dev); |
|
|
|
|
|
/* Assume that non-PCI devices asking for DMA are either ISA or EISA, |
|
|
BUG() otherwise. */ |
|
|
BUG_ON(!isa_bridge); |
|
|
|
|
|
/* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA |
|
|
bridge is bus master then). */ |
|
|
if (!dev || !dev->dma_mask || !*dev->dma_mask) |
|
|
return isa_bridge; |
|
|
|
|
|
/* For EISA bus masters, return isa_bridge (it might have smaller |
|
|
dma_mask due to wiring limitations). */ |
|
|
if (*dev->dma_mask >= isa_bridge->dma_mask) |
|
|
return isa_bridge; |
|
|
|
|
|
/* This assumes ISA bus master with dma_mask 0xffffff. */ |
|
|
return NULL; |
|
|
} |
|
|
|
|
|
static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, |
|
|
unsigned long offset, size_t size, |
|
|
enum dma_data_direction dir, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
int dac_allowed; |
|
|
|
|
|
BUG_ON(dir == PCI_DMA_NONE); |
|
|
|
|
|
dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; |
|
|
return pci_map_single_1(pdev, (char *)page_address(page) + offset, |
|
|
size, dac_allowed); |
|
|
} |
|
|
|
|
|
/* Unmap a single streaming mode DMA translation. The DMA_ADDR and |
|
|
SIZE must match what was provided for in a previous pci_map_single |
|
|
call. All other usages are undefined. After this call, reads by |
|
|
the cpu to the buffer are guaranteed to see whatever the device |
|
|
wrote there. */ |
|
|
|
|
|
static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, |
|
|
size_t size, enum dma_data_direction dir, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
unsigned long flags; |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; |
|
|
struct pci_iommu_arena *arena; |
|
|
long dma_ofs, npages; |
|
|
|
|
|
BUG_ON(dir == PCI_DMA_NONE); |
|
|
|
|
|
if (dma_addr >= __direct_map_base |
|
|
&& dma_addr < __direct_map_base + __direct_map_size) { |
|
|
/* Nothing to do. */ |
|
|
|
|
|
DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n", |
|
|
dma_addr, size, __builtin_return_address(0)); |
|
|
|
|
|
return; |
|
|
} |
|
|
|
|
|
if (dma_addr > 0xffffffff) { |
|
|
DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n", |
|
|
dma_addr, size, __builtin_return_address(0)); |
|
|
return; |
|
|
} |
|
|
|
|
|
arena = hose->sg_pci; |
|
|
if (!arena || dma_addr < arena->dma_base) |
|
|
arena = hose->sg_isa; |
|
|
|
|
|
dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT; |
|
|
if (dma_ofs * PAGE_SIZE >= arena->size) { |
|
|
printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx " |
|
|
" base %llx size %x\n", |
|
|
dma_addr, arena->dma_base, arena->size); |
|
|
return; |
|
|
BUG(); |
|
|
} |
|
|
|
|
|
npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); |
|
|
|
|
|
spin_lock_irqsave(&arena->lock, flags); |
|
|
|
|
|
iommu_arena_free(arena, dma_ofs, npages); |
|
|
|
|
|
/* If we're freeing ptes above the `next_entry' pointer (they |
|
|
may have snuck back into the TLB since the last wrap flush), |
|
|
we need to flush the TLB before reallocating the latter. */ |
|
|
if (dma_ofs >= arena->next_entry) |
|
|
alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1); |
|
|
|
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
|
|
|
DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n", |
|
|
dma_addr, size, npages, __builtin_return_address(0)); |
|
|
} |
|
|
|
|
|
/* Allocate and map kernel buffer using consistent mode DMA for PCI |
|
|
device. Returns non-NULL cpu-view pointer to the buffer if |
|
|
successful and sets *DMA_ADDRP to the pci side dma address as well, |
|
|
else DMA_ADDRP is undefined. */ |
|
|
|
|
|
static void *alpha_pci_alloc_coherent(struct device *dev, size_t size, |
|
|
dma_addr_t *dma_addrp, gfp_t gfp, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
void *cpu_addr; |
|
|
long order = get_order(size); |
|
|
|
|
|
gfp &= ~GFP_DMA; |
|
|
|
|
|
try_again: |
|
|
cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order); |
|
|
if (! cpu_addr) { |
|
|
printk(KERN_INFO "pci_alloc_consistent: " |
|
|
"get_free_pages failed from %ps\n", |
|
|
__builtin_return_address(0)); |
|
|
/* ??? Really atomic allocation? Otherwise we could play |
|
|
with vmalloc and sg if we can't find contiguous memory. */ |
|
|
return NULL; |
|
|
} |
|
|
memset(cpu_addr, 0, size); |
|
|
|
|
|
*dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0); |
|
|
if (*dma_addrp == DMA_MAPPING_ERROR) { |
|
|
free_pages((unsigned long)cpu_addr, order); |
|
|
if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA)) |
|
|
return NULL; |
|
|
/* The address doesn't fit required mask and we |
|
|
do not have iommu. Try again with GFP_DMA. */ |
|
|
gfp |= GFP_DMA; |
|
|
goto try_again; |
|
|
} |
|
|
|
|
|
DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n", |
|
|
size, cpu_addr, *dma_addrp, __builtin_return_address(0)); |
|
|
|
|
|
return cpu_addr; |
|
|
} |
|
|
|
|
|
/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must |
|
|
be values that were returned from pci_alloc_consistent. SIZE must |
|
|
be the same as what as passed into pci_alloc_consistent. |
|
|
References to the memory and mappings associated with CPU_ADDR or |
|
|
DMA_ADDR past this call are illegal. */ |
|
|
|
|
|
static void alpha_pci_free_coherent(struct device *dev, size_t size, |
|
|
void *cpu_addr, dma_addr_t dma_addr, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); |
|
|
free_pages((unsigned long)cpu_addr, get_order(size)); |
|
|
|
|
|
DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n", |
|
|
dma_addr, size, __builtin_return_address(0)); |
|
|
} |
|
|
|
|
|
/* Classify the elements of the scatterlist. Write dma_address |
|
|
of each element with: |
|
|
0 : Followers all physically adjacent. |
|
|
1 : Followers all virtually adjacent. |
|
|
-1 : Not leader, physically adjacent to previous. |
|
|
-2 : Not leader, virtually adjacent to previous. |
|
|
Write dma_length of each leader with the combined lengths of |
|
|
the mergable followers. */ |
|
|
|
|
|
#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) |
|
|
#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) |
|
|
|
|
|
static void |
|
|
sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end, |
|
|
int virt_ok) |
|
|
{ |
|
|
unsigned long next_paddr; |
|
|
struct scatterlist *leader; |
|
|
long leader_flag, leader_length; |
|
|
unsigned int max_seg_size; |
|
|
|
|
|
leader = sg; |
|
|
leader_flag = 0; |
|
|
leader_length = leader->length; |
|
|
next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length; |
|
|
|
|
|
/* we will not marge sg without device. */ |
|
|
max_seg_size = dev ? dma_get_max_seg_size(dev) : 0; |
|
|
for (++sg; sg < end; ++sg) { |
|
|
unsigned long addr, len; |
|
|
addr = SG_ENT_PHYS_ADDRESS(sg); |
|
|
len = sg->length; |
|
|
|
|
|
if (leader_length + len > max_seg_size) |
|
|
goto new_segment; |
|
|
|
|
|
if (next_paddr == addr) { |
|
|
sg->dma_address = -1; |
|
|
leader_length += len; |
|
|
} else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) { |
|
|
sg->dma_address = -2; |
|
|
leader_flag = 1; |
|
|
leader_length += len; |
|
|
} else { |
|
|
new_segment: |
|
|
leader->dma_address = leader_flag; |
|
|
leader->dma_length = leader_length; |
|
|
leader = sg; |
|
|
leader_flag = 0; |
|
|
leader_length = len; |
|
|
} |
|
|
|
|
|
next_paddr = addr + len; |
|
|
} |
|
|
|
|
|
leader->dma_address = leader_flag; |
|
|
leader->dma_length = leader_length; |
|
|
} |
|
|
|
|
|
/* Given a scatterlist leader, choose an allocation method and fill |
|
|
in the blanks. */ |
|
|
|
|
|
static int |
|
|
sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, |
|
|
struct scatterlist *out, struct pci_iommu_arena *arena, |
|
|
dma_addr_t max_dma, int dac_allowed) |
|
|
{ |
|
|
unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader); |
|
|
long size = leader->dma_length; |
|
|
struct scatterlist *sg; |
|
|
unsigned long *ptes; |
|
|
long npages, dma_ofs, i; |
|
|
|
|
|
#if !DEBUG_NODIRECT |
|
|
/* If everything is physically contiguous, and the addresses |
|
|
fall into the direct-map window, use it. */ |
|
|
if (leader->dma_address == 0 |
|
|
&& paddr + size + __direct_map_base - 1 <= max_dma |
|
|
&& paddr + size <= __direct_map_size) { |
|
|
out->dma_address = paddr + __direct_map_base; |
|
|
out->dma_length = size; |
|
|
|
|
|
DBGA(" sg_fill: [%p,%lx] -> direct %llx\n", |
|
|
__va(paddr), size, out->dma_address); |
|
|
|
|
|
return 0; |
|
|
} |
|
|
#endif |
|
|
|
|
|
/* If physically contiguous and DAC is available, use it. */ |
|
|
if (leader->dma_address == 0 && dac_allowed) { |
|
|
out->dma_address = paddr + alpha_mv.pci_dac_offset; |
|
|
out->dma_length = size; |
|
|
|
|
|
DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n", |
|
|
__va(paddr), size, out->dma_address); |
|
|
|
|
|
return 0; |
|
|
} |
|
|
|
|
|
/* Otherwise, we'll use the iommu to make the pages virtually |
|
|
contiguous. */ |
|
|
|
|
|
paddr &= ~PAGE_MASK; |
|
|
npages = iommu_num_pages(paddr, size, PAGE_SIZE); |
|
|
dma_ofs = iommu_arena_alloc(dev, arena, npages, 0); |
|
|
if (dma_ofs < 0) { |
|
|
/* If we attempted a direct map above but failed, die. */ |
|
|
if (leader->dma_address == 0) |
|
|
return -1; |
|
|
|
|
|
/* Otherwise, break up the remaining virtually contiguous |
|
|
hunks into individual direct maps and retry. */ |
|
|
sg_classify(dev, leader, end, 0); |
|
|
return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed); |
|
|
} |
|
|
|
|
|
out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr; |
|
|
out->dma_length = size; |
|
|
|
|
|
DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n", |
|
|
__va(paddr), size, out->dma_address, npages); |
|
|
|
|
|
/* All virtually contiguous. We need to find the length of each |
|
|
physically contiguous subsegment to fill in the ptes. */ |
|
|
ptes = &arena->ptes[dma_ofs]; |
|
|
sg = leader; |
|
|
do { |
|
|
#if DEBUG_ALLOC > 0 |
|
|
struct scatterlist *last_sg = sg; |
|
|
#endif |
|
|
|
|
|
size = sg->length; |
|
|
paddr = SG_ENT_PHYS_ADDRESS(sg); |
|
|
|
|
|
while (sg+1 < end && (int) sg[1].dma_address == -1) { |
|
|
size += sg[1].length; |
|
|
sg = sg_next(sg); |
|
|
} |
|
|
|
|
|
npages = iommu_num_pages(paddr, size, PAGE_SIZE); |
|
|
|
|
|
paddr &= PAGE_MASK; |
|
|
for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) |
|
|
*ptes++ = mk_iommu_pte(paddr); |
|
|
|
|
|
#if DEBUG_ALLOC > 0 |
|
|
DBGA(" (%ld) [%p,%x] np %ld\n", |
|
|
last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), |
|
|
last_sg->length, npages); |
|
|
while (++last_sg <= sg) { |
|
|
DBGA(" (%ld) [%p,%x] cont\n", |
|
|
last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), |
|
|
last_sg->length); |
|
|
} |
|
|
#endif |
|
|
} while (++sg < end && (int) sg->dma_address < 0); |
|
|
|
|
|
return 1; |
|
|
} |
|
|
|
|
|
static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, |
|
|
int nents, enum dma_data_direction dir, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
struct scatterlist *start, *end, *out; |
|
|
struct pci_controller *hose; |
|
|
struct pci_iommu_arena *arena; |
|
|
dma_addr_t max_dma; |
|
|
int dac_allowed; |
|
|
|
|
|
BUG_ON(dir == PCI_DMA_NONE); |
|
|
|
|
|
dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; |
|
|
|
|
|
/* Fast path single entry scatterlists. */ |
|
|
if (nents == 1) { |
|
|
sg->dma_length = sg->length; |
|
|
sg->dma_address |
|
|
= pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg), |
|
|
sg->length, dac_allowed); |
|
|
return sg->dma_address != DMA_MAPPING_ERROR; |
|
|
} |
|
|
|
|
|
start = sg; |
|
|
end = sg + nents; |
|
|
|
|
|
/* First, prepare information about the entries. */ |
|
|
sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0); |
|
|
|
|
|
/* Second, figure out where we're going to map things. */ |
|
|
if (alpha_mv.mv_pci_tbi) { |
|
|
hose = pdev ? pdev->sysdata : pci_isa_hose; |
|
|
max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
|
|
arena = hose->sg_pci; |
|
|
if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
|
|
arena = hose->sg_isa; |
|
|
} else { |
|
|
max_dma = -1; |
|
|
arena = NULL; |
|
|
hose = NULL; |
|
|
} |
|
|
|
|
|
/* Third, iterate over the scatterlist leaders and allocate |
|
|
dma space as needed. */ |
|
|
for (out = sg; sg < end; ++sg) { |
|
|
if ((int) sg->dma_address < 0) |
|
|
continue; |
|
|
if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0) |
|
|
goto error; |
|
|
out++; |
|
|
} |
|
|
|
|
|
/* Mark the end of the list for pci_unmap_sg. */ |
|
|
if (out < end) |
|
|
out->dma_length = 0; |
|
|
|
|
|
if (out - start == 0) |
|
|
printk(KERN_WARNING "pci_map_sg failed: no entries?\n"); |
|
|
DBGA("pci_map_sg: %ld entries\n", out - start); |
|
|
|
|
|
return out - start; |
|
|
|
|
|
error: |
|
|
printk(KERN_WARNING "pci_map_sg failed: " |
|
|
"could not allocate dma page tables\n"); |
|
|
|
|
|
/* Some allocation failed while mapping the scatterlist |
|
|
entries. Unmap them now. */ |
|
|
if (out > start) |
|
|
pci_unmap_sg(pdev, start, out - start, dir); |
|
|
return 0; |
|
|
} |
|
|
|
|
|
/* Unmap a set of streaming mode DMA translations. Again, cpu read |
|
|
rules concerning calls here are the same as for pci_unmap_single() |
|
|
above. */ |
|
|
|
|
|
static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg, |
|
|
int nents, enum dma_data_direction dir, |
|
|
unsigned long attrs) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
unsigned long flags; |
|
|
struct pci_controller *hose; |
|
|
struct pci_iommu_arena *arena; |
|
|
struct scatterlist *end; |
|
|
dma_addr_t max_dma; |
|
|
dma_addr_t fbeg, fend; |
|
|
|
|
|
BUG_ON(dir == PCI_DMA_NONE); |
|
|
|
|
|
if (! alpha_mv.mv_pci_tbi) |
|
|
return; |
|
|
|
|
|
hose = pdev ? pdev->sysdata : pci_isa_hose; |
|
|
max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
|
|
arena = hose->sg_pci; |
|
|
if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
|
|
arena = hose->sg_isa; |
|
|
|
|
|
fbeg = -1, fend = 0; |
|
|
|
|
|
spin_lock_irqsave(&arena->lock, flags); |
|
|
|
|
|
for (end = sg + nents; sg < end; ++sg) { |
|
|
dma_addr_t addr; |
|
|
size_t size; |
|
|
long npages, ofs; |
|
|
dma_addr_t tend; |
|
|
|
|
|
addr = sg->dma_address; |
|
|
size = sg->dma_length; |
|
|
if (!size) |
|
|
break; |
|
|
|
|
|
if (addr > 0xffffffff) { |
|
|
/* It's a DAC address -- nothing to do. */ |
|
|
DBGA(" (%ld) DAC [%llx,%zx]\n", |
|
|
sg - end + nents, addr, size); |
|
|
continue; |
|
|
} |
|
|
|
|
|
if (addr >= __direct_map_base |
|
|
&& addr < __direct_map_base + __direct_map_size) { |
|
|
/* Nothing to do. */ |
|
|
DBGA(" (%ld) direct [%llx,%zx]\n", |
|
|
sg - end + nents, addr, size); |
|
|
continue; |
|
|
} |
|
|
|
|
|
DBGA(" (%ld) sg [%llx,%zx]\n", |
|
|
sg - end + nents, addr, size); |
|
|
|
|
|
npages = iommu_num_pages(addr, size, PAGE_SIZE); |
|
|
ofs = (addr - arena->dma_base) >> PAGE_SHIFT; |
|
|
iommu_arena_free(arena, ofs, npages); |
|
|
|
|
|
tend = addr + size - 1; |
|
|
if (fbeg > addr) fbeg = addr; |
|
|
if (fend < tend) fend = tend; |
|
|
} |
|
|
|
|
|
/* If we're freeing ptes above the `next_entry' pointer (they |
|
|
may have snuck back into the TLB since the last wrap flush), |
|
|
we need to flush the TLB before reallocating the latter. */ |
|
|
if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry) |
|
|
alpha_mv.mv_pci_tbi(hose, fbeg, fend); |
|
|
|
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
|
|
|
DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg)); |
|
|
} |
|
|
|
|
|
/* Return whether the given PCI device DMA address mask can be |
|
|
supported properly. */ |
|
|
|
|
|
static int alpha_pci_supported(struct device *dev, u64 mask) |
|
|
{ |
|
|
struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
|
|
struct pci_controller *hose; |
|
|
struct pci_iommu_arena *arena; |
|
|
|
|
|
/* If there exists a direct map, and the mask fits either |
|
|
the entire direct mapped space or the total system memory as |
|
|
shifted by the map base */ |
|
|
if (__direct_map_size != 0 |
|
|
&& (__direct_map_base + __direct_map_size - 1 <= mask || |
|
|
__direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask)) |
|
|
return 1; |
|
|
|
|
|
/* Check that we have a scatter-gather arena that fits. */ |
|
|
hose = pdev ? pdev->sysdata : pci_isa_hose; |
|
|
arena = hose->sg_isa; |
|
|
if (arena && arena->dma_base + arena->size - 1 <= mask) |
|
|
return 1; |
|
|
arena = hose->sg_pci; |
|
|
if (arena && arena->dma_base + arena->size - 1 <= mask) |
|
|
return 1; |
|
|
|
|
|
/* As last resort try ZONE_DMA. */ |
|
|
if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask) |
|
|
return 1; |
|
|
|
|
|
return 0; |
|
|
} |
|
|
|
|
|
|
|
|
/* |
|
|
* AGP GART extensions to the IOMMU |
|
|
*/ |
|
|
int |
|
|
iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) |
|
|
{ |
|
|
unsigned long flags; |
|
|
unsigned long *ptes; |
|
|
long i, p; |
|
|
|
|
|
if (!arena) return -EINVAL; |
|
|
|
|
|
spin_lock_irqsave(&arena->lock, flags); |
|
|
|
|
|
/* Search for N empty ptes. */ |
|
|
ptes = arena->ptes; |
|
|
p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask); |
|
|
if (p < 0) { |
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
return -1; |
|
|
} |
|
|
|
|
|
/* Success. Mark them all reserved (ie not zero and invalid) |
|
|
for the iommu tlb that could load them from under us. |
|
|
They will be filled in with valid bits by _bind() */ |
|
|
for (i = 0; i < pg_count; ++i) |
|
|
ptes[p+i] = IOMMU_RESERVED_PTE; |
|
|
|
|
|
arena->next_entry = p + pg_count; |
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
|
|
|
return p; |
|
|
} |
|
|
|
|
|
int |
|
|
iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count) |
|
|
{ |
|
|
unsigned long *ptes; |
|
|
long i; |
|
|
|
|
|
if (!arena) return -EINVAL; |
|
|
|
|
|
ptes = arena->ptes; |
|
|
|
|
|
/* Make sure they're all reserved first... */ |
|
|
for(i = pg_start; i < pg_start + pg_count; i++) |
|
|
if (ptes[i] != IOMMU_RESERVED_PTE) |
|
|
return -EBUSY; |
|
|
|
|
|
iommu_arena_free(arena, pg_start, pg_count); |
|
|
return 0; |
|
|
} |
|
|
|
|
|
int |
|
|
iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, |
|
|
struct page **pages) |
|
|
{ |
|
|
unsigned long flags; |
|
|
unsigned long *ptes; |
|
|
long i, j; |
|
|
|
|
|
if (!arena) return -EINVAL; |
|
|
|
|
|
spin_lock_irqsave(&arena->lock, flags); |
|
|
|
|
|
ptes = arena->ptes; |
|
|
|
|
|
for(j = pg_start; j < pg_start + pg_count; j++) { |
|
|
if (ptes[j] != IOMMU_RESERVED_PTE) { |
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
return -EBUSY; |
|
|
} |
|
|
} |
|
|
|
|
|
for(i = 0, j = pg_start; i < pg_count; i++, j++) |
|
|
ptes[j] = mk_iommu_pte(page_to_phys(pages[i])); |
|
|
|
|
|
spin_unlock_irqrestore(&arena->lock, flags); |
|
|
|
|
|
return 0; |
|
|
} |
|
|
|
|
|
int |
|
|
iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count) |
|
|
{ |
|
|
unsigned long *p; |
|
|
long i; |
|
|
|
|
|
if (!arena) return -EINVAL; |
|
|
|
|
|
p = arena->ptes + pg_start; |
|
|
for(i = 0; i < pg_count; i++) |
|
|
p[i] = IOMMU_RESERVED_PTE; |
|
|
|
|
|
return 0; |
|
|
} |
|
|
|
|
|
const struct dma_map_ops alpha_pci_ops = { |
|
|
.alloc = alpha_pci_alloc_coherent, |
|
|
.free = alpha_pci_free_coherent, |
|
|
.map_page = alpha_pci_map_page, |
|
|
.unmap_page = alpha_pci_unmap_page, |
|
|
.map_sg = alpha_pci_map_sg, |
|
|
.unmap_sg = alpha_pci_unmap_sg, |
|
|
.dma_supported = alpha_pci_supported, |
|
|
.mmap = dma_common_mmap, |
|
|
.get_sgtable = dma_common_get_sgtable, |
|
|
.alloc_pages = dma_common_alloc_pages, |
|
|
.free_pages = dma_common_free_pages, |
|
|
}; |
|
|
EXPORT_SYMBOL(alpha_pci_ops);
|
|
|
|