forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
488 lines
12 KiB
488 lines
12 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* |
|
* Copyright (C) 2013 Freescale Semiconductor, Inc. |
|
* Author: Varun Sethi <[email protected]> |
|
*/ |
|
|
|
#define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ |
|
|
|
#include "fsl_pamu_domain.h" |
|
|
|
#include <sysdev/fsl_pci.h> |
|
|
|
/* |
|
* Global spinlock that needs to be held while |
|
* configuring PAMU. |
|
*/ |
|
static DEFINE_SPINLOCK(iommu_lock); |
|
|
|
static struct kmem_cache *fsl_pamu_domain_cache; |
|
static struct kmem_cache *iommu_devinfo_cache; |
|
static DEFINE_SPINLOCK(device_domain_lock); |
|
|
|
struct iommu_device pamu_iommu; /* IOMMU core code handle */ |
|
|
|
static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom) |
|
{ |
|
return container_of(dom, struct fsl_dma_domain, iommu_domain); |
|
} |
|
|
|
static int __init iommu_init_mempool(void) |
|
{ |
|
fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", |
|
sizeof(struct fsl_dma_domain), |
|
0, |
|
SLAB_HWCACHE_ALIGN, |
|
NULL); |
|
if (!fsl_pamu_domain_cache) { |
|
pr_debug("Couldn't create fsl iommu_domain cache\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", |
|
sizeof(struct device_domain_info), |
|
0, |
|
SLAB_HWCACHE_ALIGN, |
|
NULL); |
|
if (!iommu_devinfo_cache) { |
|
pr_debug("Couldn't create devinfo cache\n"); |
|
kmem_cache_destroy(fsl_pamu_domain_cache); |
|
return -ENOMEM; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, |
|
u32 val) |
|
{ |
|
int ret = 0; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&iommu_lock, flags); |
|
ret = pamu_update_paace_stash(liodn, val); |
|
if (ret) { |
|
pr_debug("Failed to update SPAACE for liodn %d\n ", liodn); |
|
spin_unlock_irqrestore(&iommu_lock, flags); |
|
return ret; |
|
} |
|
|
|
spin_unlock_irqrestore(&iommu_lock, flags); |
|
|
|
return ret; |
|
} |
|
|
|
/* Set the geometry parameters for a LIODN */ |
|
static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev, |
|
int liodn) |
|
{ |
|
u32 omi_index = ~(u32)0; |
|
unsigned long flags; |
|
int ret; |
|
|
|
/* |
|
* Configure the omi_index at the geometry setup time. |
|
* This is a static value which depends on the type of |
|
* device and would not change thereafter. |
|
*/ |
|
get_ome_index(&omi_index, dev); |
|
|
|
spin_lock_irqsave(&iommu_lock, flags); |
|
ret = pamu_disable_liodn(liodn); |
|
if (ret) |
|
goto out_unlock; |
|
ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0); |
|
if (ret) |
|
goto out_unlock; |
|
ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id, |
|
PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE); |
|
out_unlock: |
|
spin_unlock_irqrestore(&iommu_lock, flags); |
|
if (ret) { |
|
pr_debug("PAACE configuration failed for liodn %d\n", |
|
liodn); |
|
} |
|
return ret; |
|
} |
|
|
|
static void remove_device_ref(struct device_domain_info *info) |
|
{ |
|
unsigned long flags; |
|
|
|
list_del(&info->link); |
|
spin_lock_irqsave(&iommu_lock, flags); |
|
pamu_disable_liodn(info->liodn); |
|
spin_unlock_irqrestore(&iommu_lock, flags); |
|
spin_lock_irqsave(&device_domain_lock, flags); |
|
dev_iommu_priv_set(info->dev, NULL); |
|
kmem_cache_free(iommu_devinfo_cache, info); |
|
spin_unlock_irqrestore(&device_domain_lock, flags); |
|
} |
|
|
|
static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) |
|
{ |
|
struct device_domain_info *info, *tmp; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&dma_domain->domain_lock, flags); |
|
/* Remove the device from the domain device list */ |
|
list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { |
|
if (!dev || (info->dev == dev)) |
|
remove_device_ref(info); |
|
} |
|
spin_unlock_irqrestore(&dma_domain->domain_lock, flags); |
|
} |
|
|
|
static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev) |
|
{ |
|
struct device_domain_info *info, *old_domain_info; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&device_domain_lock, flags); |
|
/* |
|
* Check here if the device is already attached to domain or not. |
|
* If the device is already attached to a domain detach it. |
|
*/ |
|
old_domain_info = dev_iommu_priv_get(dev); |
|
if (old_domain_info && old_domain_info->domain != dma_domain) { |
|
spin_unlock_irqrestore(&device_domain_lock, flags); |
|
detach_device(dev, old_domain_info->domain); |
|
spin_lock_irqsave(&device_domain_lock, flags); |
|
} |
|
|
|
info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC); |
|
|
|
info->dev = dev; |
|
info->liodn = liodn; |
|
info->domain = dma_domain; |
|
|
|
list_add(&info->link, &dma_domain->devices); |
|
/* |
|
* In case of devices with multiple LIODNs just store |
|
* the info for the first LIODN as all |
|
* LIODNs share the same domain |
|
*/ |
|
if (!dev_iommu_priv_get(dev)) |
|
dev_iommu_priv_set(dev, info); |
|
spin_unlock_irqrestore(&device_domain_lock, flags); |
|
} |
|
|
|
static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, |
|
dma_addr_t iova) |
|
{ |
|
if (iova < domain->geometry.aperture_start || |
|
iova > domain->geometry.aperture_end) |
|
return 0; |
|
return iova; |
|
} |
|
|
|
static bool fsl_pamu_capable(enum iommu_cap cap) |
|
{ |
|
return cap == IOMMU_CAP_CACHE_COHERENCY; |
|
} |
|
|
|
static void fsl_pamu_domain_free(struct iommu_domain *domain) |
|
{ |
|
struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); |
|
|
|
/* remove all the devices from the device list */ |
|
detach_device(NULL, dma_domain); |
|
kmem_cache_free(fsl_pamu_domain_cache, dma_domain); |
|
} |
|
|
|
static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) |
|
{ |
|
struct fsl_dma_domain *dma_domain; |
|
|
|
if (type != IOMMU_DOMAIN_UNMANAGED) |
|
return NULL; |
|
|
|
dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); |
|
if (!dma_domain) |
|
return NULL; |
|
|
|
dma_domain->stash_id = ~(u32)0; |
|
INIT_LIST_HEAD(&dma_domain->devices); |
|
spin_lock_init(&dma_domain->domain_lock); |
|
|
|
/* default geometry 64 GB i.e. maximum system address */ |
|
dma_domain->iommu_domain. geometry.aperture_start = 0; |
|
dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1; |
|
dma_domain->iommu_domain.geometry.force_aperture = true; |
|
|
|
return &dma_domain->iommu_domain; |
|
} |
|
|
|
/* Update stash destination for all LIODNs associated with the domain */ |
|
static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) |
|
{ |
|
struct device_domain_info *info; |
|
int ret = 0; |
|
|
|
list_for_each_entry(info, &dma_domain->devices, link) { |
|
ret = update_liodn_stash(info->liodn, dma_domain, val); |
|
if (ret) |
|
break; |
|
} |
|
|
|
return ret; |
|
} |
|
|
|
static int fsl_pamu_attach_device(struct iommu_domain *domain, |
|
struct device *dev) |
|
{ |
|
struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); |
|
unsigned long flags; |
|
int len, ret = 0, i; |
|
const u32 *liodn; |
|
struct pci_dev *pdev = NULL; |
|
struct pci_controller *pci_ctl; |
|
|
|
/* |
|
* Use LIODN of the PCI controller while attaching a |
|
* PCI device. |
|
*/ |
|
if (dev_is_pci(dev)) { |
|
pdev = to_pci_dev(dev); |
|
pci_ctl = pci_bus_to_host(pdev->bus); |
|
/* |
|
* make dev point to pci controller device |
|
* so we can get the LIODN programmed by |
|
* u-boot. |
|
*/ |
|
dev = pci_ctl->parent; |
|
} |
|
|
|
liodn = of_get_property(dev->of_node, "fsl,liodn", &len); |
|
if (!liodn) { |
|
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); |
|
return -EINVAL; |
|
} |
|
|
|
spin_lock_irqsave(&dma_domain->domain_lock, flags); |
|
for (i = 0; i < len / sizeof(u32); i++) { |
|
/* Ensure that LIODN value is valid */ |
|
if (liodn[i] >= PAACE_NUMBER_ENTRIES) { |
|
pr_debug("Invalid liodn %d, attach device failed for %pOF\n", |
|
liodn[i], dev->of_node); |
|
ret = -EINVAL; |
|
break; |
|
} |
|
|
|
attach_device(dma_domain, liodn[i], dev); |
|
ret = pamu_set_liodn(dma_domain, dev, liodn[i]); |
|
if (ret) |
|
break; |
|
ret = pamu_enable_liodn(liodn[i]); |
|
if (ret) |
|
break; |
|
} |
|
spin_unlock_irqrestore(&dma_domain->domain_lock, flags); |
|
return ret; |
|
} |
|
|
|
static void fsl_pamu_detach_device(struct iommu_domain *domain, |
|
struct device *dev) |
|
{ |
|
struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); |
|
const u32 *prop; |
|
int len; |
|
struct pci_dev *pdev = NULL; |
|
struct pci_controller *pci_ctl; |
|
|
|
/* |
|
* Use LIODN of the PCI controller while detaching a |
|
* PCI device. |
|
*/ |
|
if (dev_is_pci(dev)) { |
|
pdev = to_pci_dev(dev); |
|
pci_ctl = pci_bus_to_host(pdev->bus); |
|
/* |
|
* make dev point to pci controller device |
|
* so we can get the LIODN programmed by |
|
* u-boot. |
|
*/ |
|
dev = pci_ctl->parent; |
|
} |
|
|
|
prop = of_get_property(dev->of_node, "fsl,liodn", &len); |
|
if (prop) |
|
detach_device(dev, dma_domain); |
|
else |
|
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); |
|
} |
|
|
|
/* Set the domain stash attribute */ |
|
int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu) |
|
{ |
|
struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); |
|
unsigned long flags; |
|
int ret; |
|
|
|
spin_lock_irqsave(&dma_domain->domain_lock, flags); |
|
dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu); |
|
if (dma_domain->stash_id == ~(u32)0) { |
|
pr_debug("Invalid stash attributes\n"); |
|
spin_unlock_irqrestore(&dma_domain->domain_lock, flags); |
|
return -EINVAL; |
|
} |
|
ret = update_domain_stash(dma_domain, dma_domain->stash_id); |
|
spin_unlock_irqrestore(&dma_domain->domain_lock, flags); |
|
|
|
return ret; |
|
} |
|
|
|
static struct iommu_group *get_device_iommu_group(struct device *dev) |
|
{ |
|
struct iommu_group *group; |
|
|
|
group = iommu_group_get(dev); |
|
if (!group) |
|
group = iommu_group_alloc(); |
|
|
|
return group; |
|
} |
|
|
|
static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) |
|
{ |
|
u32 version; |
|
|
|
/* Check the PCI controller version number by readding BRR1 register */ |
|
version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); |
|
version &= PCI_FSL_BRR1_VER; |
|
/* If PCI controller version is >= 0x204 we can partition endpoints */ |
|
return version >= 0x204; |
|
} |
|
|
|
/* Get iommu group information from peer devices or devices on the parent bus */ |
|
static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev) |
|
{ |
|
struct pci_dev *tmp; |
|
struct iommu_group *group; |
|
struct pci_bus *bus = pdev->bus; |
|
|
|
/* |
|
* Traverese the pci bus device list to get |
|
* the shared iommu group. |
|
*/ |
|
while (bus) { |
|
list_for_each_entry(tmp, &bus->devices, bus_list) { |
|
if (tmp == pdev) |
|
continue; |
|
group = iommu_group_get(&tmp->dev); |
|
if (group) |
|
return group; |
|
} |
|
|
|
bus = bus->parent; |
|
} |
|
|
|
return NULL; |
|
} |
|
|
|
static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) |
|
{ |
|
struct pci_controller *pci_ctl; |
|
bool pci_endpt_partitioning; |
|
struct iommu_group *group = NULL; |
|
|
|
pci_ctl = pci_bus_to_host(pdev->bus); |
|
pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl); |
|
/* We can partition PCIe devices so assign device group to the device */ |
|
if (pci_endpt_partitioning) { |
|
group = pci_device_group(&pdev->dev); |
|
|
|
/* |
|
* PCIe controller is not a paritionable entity |
|
* free the controller device iommu_group. |
|
*/ |
|
if (pci_ctl->parent->iommu_group) |
|
iommu_group_remove_device(pci_ctl->parent); |
|
} else { |
|
/* |
|
* All devices connected to the controller will share the |
|
* PCI controllers device group. If this is the first |
|
* device to be probed for the pci controller, copy the |
|
* device group information from the PCI controller device |
|
* node and remove the PCI controller iommu group. |
|
* For subsequent devices, the iommu group information can |
|
* be obtained from sibling devices (i.e. from the bus_devices |
|
* link list). |
|
*/ |
|
if (pci_ctl->parent->iommu_group) { |
|
group = get_device_iommu_group(pci_ctl->parent); |
|
iommu_group_remove_device(pci_ctl->parent); |
|
} else { |
|
group = get_shared_pci_device_group(pdev); |
|
} |
|
} |
|
|
|
if (!group) |
|
group = ERR_PTR(-ENODEV); |
|
|
|
return group; |
|
} |
|
|
|
static struct iommu_group *fsl_pamu_device_group(struct device *dev) |
|
{ |
|
struct iommu_group *group = ERR_PTR(-ENODEV); |
|
int len; |
|
|
|
/* |
|
* For platform devices we allocate a separate group for |
|
* each of the devices. |
|
*/ |
|
if (dev_is_pci(dev)) |
|
group = get_pci_device_group(to_pci_dev(dev)); |
|
else if (of_get_property(dev->of_node, "fsl,liodn", &len)) |
|
group = get_device_iommu_group(dev); |
|
|
|
return group; |
|
} |
|
|
|
static struct iommu_device *fsl_pamu_probe_device(struct device *dev) |
|
{ |
|
return &pamu_iommu; |
|
} |
|
|
|
static void fsl_pamu_release_device(struct device *dev) |
|
{ |
|
} |
|
|
|
static const struct iommu_ops fsl_pamu_ops = { |
|
.capable = fsl_pamu_capable, |
|
.domain_alloc = fsl_pamu_domain_alloc, |
|
.domain_free = fsl_pamu_domain_free, |
|
.attach_dev = fsl_pamu_attach_device, |
|
.detach_dev = fsl_pamu_detach_device, |
|
.iova_to_phys = fsl_pamu_iova_to_phys, |
|
.probe_device = fsl_pamu_probe_device, |
|
.release_device = fsl_pamu_release_device, |
|
.device_group = fsl_pamu_device_group, |
|
}; |
|
|
|
int __init pamu_domain_init(void) |
|
{ |
|
int ret = 0; |
|
|
|
ret = iommu_init_mempool(); |
|
if (ret) |
|
return ret; |
|
|
|
ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0"); |
|
if (ret) |
|
return ret; |
|
|
|
ret = iommu_device_register(&pamu_iommu, &fsl_pamu_ops, NULL); |
|
if (ret) { |
|
iommu_device_sysfs_remove(&pamu_iommu); |
|
pr_err("Can't register iommu device\n"); |
|
return ret; |
|
} |
|
|
|
bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); |
|
bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); |
|
|
|
return ret; |
|
}
|
|
|