mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1234 lines
33 KiB
1234 lines
33 KiB
/* |
|
* Copyright(c) 2015 - 2020 Intel Corporation. |
|
* |
|
* This file is provided under a dual BSD/GPLv2 license. When using or |
|
* redistributing this file, you may do so under either license. |
|
* |
|
* GPL LICENSE SUMMARY |
|
* |
|
* This program is free software; you can redistribute it and/or modify |
|
* it under the terms of version 2 of the GNU General Public License as |
|
* published by the Free Software Foundation. |
|
* |
|
* This program is distributed in the hope that it will be useful, but |
|
* WITHOUT ANY WARRANTY; without even the implied warranty of |
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
|
* General Public License for more details. |
|
* |
|
* BSD LICENSE |
|
* |
|
* Redistribution and use in source and binary forms, with or without |
|
* modification, are permitted provided that the following conditions |
|
* are met: |
|
* |
|
* - Redistributions of source code must retain the above copyright |
|
* notice, this list of conditions and the following disclaimer. |
|
* - Redistributions in binary form must reproduce the above copyright |
|
* notice, this list of conditions and the following disclaimer in |
|
* the documentation and/or other materials provided with the |
|
* distribution. |
|
* - Neither the name of Intel Corporation nor the names of its |
|
* contributors may be used to endorse or promote products derived |
|
* from this software without specific prior written permission. |
|
* |
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
* |
|
*/ |
|
#include <linux/topology.h> |
|
#include <linux/cpumask.h> |
|
#include <linux/module.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/numa.h> |
|
|
|
#include "hfi.h" |
|
#include "affinity.h" |
|
#include "sdma.h" |
|
#include "trace.h" |
|
|
|
struct hfi1_affinity_node_list node_affinity = { |
|
.list = LIST_HEAD_INIT(node_affinity.list), |
|
.lock = __MUTEX_INITIALIZER(node_affinity.lock) |
|
}; |
|
|
|
/* Name of IRQ types, indexed by enum irq_type */ |
|
static const char * const irq_type_names[] = { |
|
"SDMA", |
|
"RCVCTXT", |
|
"NETDEVCTXT", |
|
"GENERAL", |
|
"OTHER", |
|
}; |
|
|
|
/* Per NUMA node count of HFI devices */ |
|
static unsigned int *hfi1_per_node_cntr; |
|
|
|
static inline void init_cpu_mask_set(struct cpu_mask_set *set) |
|
{ |
|
cpumask_clear(&set->mask); |
|
cpumask_clear(&set->used); |
|
set->gen = 0; |
|
} |
|
|
|
/* Increment generation of CPU set if needed */ |
|
static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set) |
|
{ |
|
if (cpumask_equal(&set->mask, &set->used)) { |
|
/* |
|
* We've used up all the CPUs, bump up the generation |
|
* and reset the 'used' map |
|
*/ |
|
set->gen++; |
|
cpumask_clear(&set->used); |
|
} |
|
} |
|
|
|
static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set) |
|
{ |
|
if (cpumask_empty(&set->used) && set->gen) { |
|
set->gen--; |
|
cpumask_copy(&set->used, &set->mask); |
|
} |
|
} |
|
|
|
/* Get the first CPU from the list of unused CPUs in a CPU set data structure */ |
|
static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff) |
|
{ |
|
int cpu; |
|
|
|
if (!diff || !set) |
|
return -EINVAL; |
|
|
|
_cpu_mask_set_gen_inc(set); |
|
|
|
/* Find out CPUs left in CPU mask */ |
|
cpumask_andnot(diff, &set->mask, &set->used); |
|
|
|
cpu = cpumask_first(diff); |
|
if (cpu >= nr_cpu_ids) /* empty */ |
|
cpu = -EINVAL; |
|
else |
|
cpumask_set_cpu(cpu, &set->used); |
|
|
|
return cpu; |
|
} |
|
|
|
static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu) |
|
{ |
|
if (!set) |
|
return; |
|
|
|
cpumask_clear_cpu(cpu, &set->used); |
|
_cpu_mask_set_gen_dec(set); |
|
} |
|
|
|
/* Initialize non-HT cpu cores mask */ |
|
void init_real_cpu_mask(void) |
|
{ |
|
int possible, curr_cpu, i, ht; |
|
|
|
cpumask_clear(&node_affinity.real_cpu_mask); |
|
|
|
/* Start with cpu online mask as the real cpu mask */ |
|
cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask); |
|
|
|
/* |
|
* Remove HT cores from the real cpu mask. Do this in two steps below. |
|
*/ |
|
possible = cpumask_weight(&node_affinity.real_cpu_mask); |
|
ht = cpumask_weight(topology_sibling_cpumask( |
|
cpumask_first(&node_affinity.real_cpu_mask))); |
|
/* |
|
* Step 1. Skip over the first N HT siblings and use them as the |
|
* "real" cores. Assumes that HT cores are not enumerated in |
|
* succession (except in the single core case). |
|
*/ |
|
curr_cpu = cpumask_first(&node_affinity.real_cpu_mask); |
|
for (i = 0; i < possible / ht; i++) |
|
curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); |
|
/* |
|
* Step 2. Remove the remaining HT siblings. Use cpumask_next() to |
|
* skip any gaps. |
|
*/ |
|
for (; i < possible; i++) { |
|
cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask); |
|
curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); |
|
} |
|
} |
|
|
|
int node_affinity_init(void) |
|
{ |
|
int node; |
|
struct pci_dev *dev = NULL; |
|
const struct pci_device_id *ids = hfi1_pci_tbl; |
|
|
|
cpumask_clear(&node_affinity.proc.used); |
|
cpumask_copy(&node_affinity.proc.mask, cpu_online_mask); |
|
|
|
node_affinity.proc.gen = 0; |
|
node_affinity.num_core_siblings = |
|
cpumask_weight(topology_sibling_cpumask( |
|
cpumask_first(&node_affinity.proc.mask) |
|
)); |
|
node_affinity.num_possible_nodes = num_possible_nodes(); |
|
node_affinity.num_online_nodes = num_online_nodes(); |
|
node_affinity.num_online_cpus = num_online_cpus(); |
|
|
|
/* |
|
* The real cpu mask is part of the affinity struct but it has to be |
|
* initialized early. It is needed to calculate the number of user |
|
* contexts in set_up_context_variables(). |
|
*/ |
|
init_real_cpu_mask(); |
|
|
|
hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes, |
|
sizeof(*hfi1_per_node_cntr), GFP_KERNEL); |
|
if (!hfi1_per_node_cntr) |
|
return -ENOMEM; |
|
|
|
while (ids->vendor) { |
|
dev = NULL; |
|
while ((dev = pci_get_device(ids->vendor, ids->device, dev))) { |
|
node = pcibus_to_node(dev->bus); |
|
if (node < 0) |
|
goto out; |
|
|
|
hfi1_per_node_cntr[node]++; |
|
} |
|
ids++; |
|
} |
|
|
|
return 0; |
|
|
|
out: |
|
/* |
|
* Invalid PCI NUMA node information found, note it, and populate |
|
* our database 1:1. |
|
*/ |
|
pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n"); |
|
pr_err("HFI: System BIOS may need to be upgraded\n"); |
|
for (node = 0; node < node_affinity.num_possible_nodes; node++) |
|
hfi1_per_node_cntr[node] = 1; |
|
|
|
return 0; |
|
} |
|
|
|
static void node_affinity_destroy(struct hfi1_affinity_node *entry) |
|
{ |
|
free_percpu(entry->comp_vect_affinity); |
|
kfree(entry); |
|
} |
|
|
|
void node_affinity_destroy_all(void) |
|
{ |
|
struct list_head *pos, *q; |
|
struct hfi1_affinity_node *entry; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
list_for_each_safe(pos, q, &node_affinity.list) { |
|
entry = list_entry(pos, struct hfi1_affinity_node, |
|
list); |
|
list_del(pos); |
|
node_affinity_destroy(entry); |
|
} |
|
mutex_unlock(&node_affinity.lock); |
|
kfree(hfi1_per_node_cntr); |
|
} |
|
|
|
static struct hfi1_affinity_node *node_affinity_allocate(int node) |
|
{ |
|
struct hfi1_affinity_node *entry; |
|
|
|
entry = kzalloc(sizeof(*entry), GFP_KERNEL); |
|
if (!entry) |
|
return NULL; |
|
entry->node = node; |
|
entry->comp_vect_affinity = alloc_percpu(u16); |
|
INIT_LIST_HEAD(&entry->list); |
|
|
|
return entry; |
|
} |
|
|
|
/* |
|
* It appends an entry to the list. |
|
* It *must* be called with node_affinity.lock held. |
|
*/ |
|
static void node_affinity_add_tail(struct hfi1_affinity_node *entry) |
|
{ |
|
list_add_tail(&entry->list, &node_affinity.list); |
|
} |
|
|
|
/* It must be called with node_affinity.lock held */ |
|
static struct hfi1_affinity_node *node_affinity_lookup(int node) |
|
{ |
|
struct list_head *pos; |
|
struct hfi1_affinity_node *entry; |
|
|
|
list_for_each(pos, &node_affinity.list) { |
|
entry = list_entry(pos, struct hfi1_affinity_node, list); |
|
if (entry->node == node) |
|
return entry; |
|
} |
|
|
|
return NULL; |
|
} |
|
|
|
static int per_cpu_affinity_get(cpumask_var_t possible_cpumask, |
|
u16 __percpu *comp_vect_affinity) |
|
{ |
|
int curr_cpu; |
|
u16 cntr; |
|
u16 prev_cntr; |
|
int ret_cpu; |
|
|
|
if (!possible_cpumask) { |
|
ret_cpu = -EINVAL; |
|
goto fail; |
|
} |
|
|
|
if (!comp_vect_affinity) { |
|
ret_cpu = -EINVAL; |
|
goto fail; |
|
} |
|
|
|
ret_cpu = cpumask_first(possible_cpumask); |
|
if (ret_cpu >= nr_cpu_ids) { |
|
ret_cpu = -EINVAL; |
|
goto fail; |
|
} |
|
|
|
prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu); |
|
for_each_cpu(curr_cpu, possible_cpumask) { |
|
cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu); |
|
|
|
if (cntr < prev_cntr) { |
|
ret_cpu = curr_cpu; |
|
prev_cntr = cntr; |
|
} |
|
} |
|
|
|
*per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1; |
|
|
|
fail: |
|
return ret_cpu; |
|
} |
|
|
|
static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask, |
|
u16 __percpu *comp_vect_affinity) |
|
{ |
|
int curr_cpu; |
|
int max_cpu; |
|
u16 cntr; |
|
u16 prev_cntr; |
|
|
|
if (!possible_cpumask) |
|
return -EINVAL; |
|
|
|
if (!comp_vect_affinity) |
|
return -EINVAL; |
|
|
|
max_cpu = cpumask_first(possible_cpumask); |
|
if (max_cpu >= nr_cpu_ids) |
|
return -EINVAL; |
|
|
|
prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu); |
|
for_each_cpu(curr_cpu, possible_cpumask) { |
|
cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu); |
|
|
|
if (cntr > prev_cntr) { |
|
max_cpu = curr_cpu; |
|
prev_cntr = cntr; |
|
} |
|
} |
|
|
|
*per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1; |
|
|
|
return max_cpu; |
|
} |
|
|
|
/* |
|
* Non-interrupt CPUs are used first, then interrupt CPUs. |
|
* Two already allocated cpu masks must be passed. |
|
*/ |
|
static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd, |
|
struct hfi1_affinity_node *entry, |
|
cpumask_var_t non_intr_cpus, |
|
cpumask_var_t available_cpus) |
|
__must_hold(&node_affinity.lock) |
|
{ |
|
int cpu; |
|
struct cpu_mask_set *set = dd->comp_vect; |
|
|
|
lockdep_assert_held(&node_affinity.lock); |
|
if (!non_intr_cpus) { |
|
cpu = -1; |
|
goto fail; |
|
} |
|
|
|
if (!available_cpus) { |
|
cpu = -1; |
|
goto fail; |
|
} |
|
|
|
/* Available CPUs for pinning completion vectors */ |
|
_cpu_mask_set_gen_inc(set); |
|
cpumask_andnot(available_cpus, &set->mask, &set->used); |
|
|
|
/* Available CPUs without SDMA engine interrupts */ |
|
cpumask_andnot(non_intr_cpus, available_cpus, |
|
&entry->def_intr.used); |
|
|
|
/* If there are non-interrupt CPUs available, use them first */ |
|
if (!cpumask_empty(non_intr_cpus)) |
|
cpu = cpumask_first(non_intr_cpus); |
|
else /* Otherwise, use interrupt CPUs */ |
|
cpu = cpumask_first(available_cpus); |
|
|
|
if (cpu >= nr_cpu_ids) { /* empty */ |
|
cpu = -1; |
|
goto fail; |
|
} |
|
cpumask_set_cpu(cpu, &set->used); |
|
|
|
fail: |
|
return cpu; |
|
} |
|
|
|
static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu) |
|
{ |
|
struct cpu_mask_set *set = dd->comp_vect; |
|
|
|
if (cpu < 0) |
|
return; |
|
|
|
cpu_mask_set_put(set, cpu); |
|
} |
|
|
|
/* _dev_comp_vect_mappings_destroy() is reentrant */ |
|
static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd) |
|
{ |
|
int i, cpu; |
|
|
|
if (!dd->comp_vect_mappings) |
|
return; |
|
|
|
for (i = 0; i < dd->comp_vect_possible_cpus; i++) { |
|
cpu = dd->comp_vect_mappings[i]; |
|
_dev_comp_vect_cpu_put(dd, cpu); |
|
dd->comp_vect_mappings[i] = -1; |
|
hfi1_cdbg(AFFINITY, |
|
"[%s] Release CPU %d from completion vector %d", |
|
rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i); |
|
} |
|
|
|
kfree(dd->comp_vect_mappings); |
|
dd->comp_vect_mappings = NULL; |
|
} |
|
|
|
/* |
|
* This function creates the table for looking up CPUs for completion vectors. |
|
* num_comp_vectors needs to have been initilized before calling this function. |
|
*/ |
|
static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd, |
|
struct hfi1_affinity_node *entry) |
|
__must_hold(&node_affinity.lock) |
|
{ |
|
int i, cpu, ret; |
|
cpumask_var_t non_intr_cpus; |
|
cpumask_var_t available_cpus; |
|
|
|
lockdep_assert_held(&node_affinity.lock); |
|
|
|
if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL)) |
|
return -ENOMEM; |
|
|
|
if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) { |
|
free_cpumask_var(non_intr_cpus); |
|
return -ENOMEM; |
|
} |
|
|
|
dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus, |
|
sizeof(*dd->comp_vect_mappings), |
|
GFP_KERNEL); |
|
if (!dd->comp_vect_mappings) { |
|
ret = -ENOMEM; |
|
goto fail; |
|
} |
|
for (i = 0; i < dd->comp_vect_possible_cpus; i++) |
|
dd->comp_vect_mappings[i] = -1; |
|
|
|
for (i = 0; i < dd->comp_vect_possible_cpus; i++) { |
|
cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus, |
|
available_cpus); |
|
if (cpu < 0) { |
|
ret = -EINVAL; |
|
goto fail; |
|
} |
|
|
|
dd->comp_vect_mappings[i] = cpu; |
|
hfi1_cdbg(AFFINITY, |
|
"[%s] Completion Vector %d -> CPU %d", |
|
rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu); |
|
} |
|
|
|
free_cpumask_var(available_cpus); |
|
free_cpumask_var(non_intr_cpus); |
|
return 0; |
|
|
|
fail: |
|
free_cpumask_var(available_cpus); |
|
free_cpumask_var(non_intr_cpus); |
|
_dev_comp_vect_mappings_destroy(dd); |
|
|
|
return ret; |
|
} |
|
|
|
int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd) |
|
{ |
|
int ret; |
|
struct hfi1_affinity_node *entry; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
entry = node_affinity_lookup(dd->node); |
|
if (!entry) { |
|
ret = -EINVAL; |
|
goto unlock; |
|
} |
|
ret = _dev_comp_vect_mappings_create(dd, entry); |
|
unlock: |
|
mutex_unlock(&node_affinity.lock); |
|
|
|
return ret; |
|
} |
|
|
|
void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd) |
|
{ |
|
_dev_comp_vect_mappings_destroy(dd); |
|
} |
|
|
|
int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect) |
|
{ |
|
struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); |
|
struct hfi1_devdata *dd = dd_from_dev(verbs_dev); |
|
|
|
if (!dd->comp_vect_mappings) |
|
return -EINVAL; |
|
if (comp_vect >= dd->comp_vect_possible_cpus) |
|
return -EINVAL; |
|
|
|
return dd->comp_vect_mappings[comp_vect]; |
|
} |
|
|
|
/* |
|
* It assumes dd->comp_vect_possible_cpus is available. |
|
*/ |
|
static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd, |
|
struct hfi1_affinity_node *entry, |
|
bool first_dev_init) |
|
__must_hold(&node_affinity.lock) |
|
{ |
|
int i, j, curr_cpu; |
|
int possible_cpus_comp_vect = 0; |
|
struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask; |
|
|
|
lockdep_assert_held(&node_affinity.lock); |
|
/* |
|
* If there's only one CPU available for completion vectors, then |
|
* there will only be one completion vector available. Othewise, |
|
* the number of completion vector available will be the number of |
|
* available CPUs divide it by the number of devices in the |
|
* local NUMA node. |
|
*/ |
|
if (cpumask_weight(&entry->comp_vect_mask) == 1) { |
|
possible_cpus_comp_vect = 1; |
|
dd_dev_warn(dd, |
|
"Number of kernel receive queues is too large for completion vector affinity to be effective\n"); |
|
} else { |
|
possible_cpus_comp_vect += |
|
cpumask_weight(&entry->comp_vect_mask) / |
|
hfi1_per_node_cntr[dd->node]; |
|
|
|
/* |
|
* If the completion vector CPUs available doesn't divide |
|
* evenly among devices, then the first device device to be |
|
* initialized gets an extra CPU. |
|
*/ |
|
if (first_dev_init && |
|
cpumask_weight(&entry->comp_vect_mask) % |
|
hfi1_per_node_cntr[dd->node] != 0) |
|
possible_cpus_comp_vect++; |
|
} |
|
|
|
dd->comp_vect_possible_cpus = possible_cpus_comp_vect; |
|
|
|
/* Reserving CPUs for device completion vector */ |
|
for (i = 0; i < dd->comp_vect_possible_cpus; i++) { |
|
curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask, |
|
entry->comp_vect_affinity); |
|
if (curr_cpu < 0) |
|
goto fail; |
|
|
|
cpumask_set_cpu(curr_cpu, dev_comp_vect_mask); |
|
} |
|
|
|
hfi1_cdbg(AFFINITY, |
|
"[%s] Completion vector affinity CPU set(s) %*pbl", |
|
rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), |
|
cpumask_pr_args(dev_comp_vect_mask)); |
|
|
|
return 0; |
|
|
|
fail: |
|
for (j = 0; j < i; j++) |
|
per_cpu_affinity_put_max(&entry->comp_vect_mask, |
|
entry->comp_vect_affinity); |
|
|
|
return curr_cpu; |
|
} |
|
|
|
/* |
|
* It assumes dd->comp_vect_possible_cpus is available. |
|
*/ |
|
static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd, |
|
struct hfi1_affinity_node *entry) |
|
__must_hold(&node_affinity.lock) |
|
{ |
|
int i, cpu; |
|
|
|
lockdep_assert_held(&node_affinity.lock); |
|
if (!dd->comp_vect_possible_cpus) |
|
return; |
|
|
|
for (i = 0; i < dd->comp_vect_possible_cpus; i++) { |
|
cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask, |
|
entry->comp_vect_affinity); |
|
/* Clearing CPU in device completion vector cpu mask */ |
|
if (cpu >= 0) |
|
cpumask_clear_cpu(cpu, &dd->comp_vect->mask); |
|
} |
|
|
|
dd->comp_vect_possible_cpus = 0; |
|
} |
|
|
|
/* |
|
* Interrupt affinity. |
|
* |
|
* non-rcv avail gets a default mask that |
|
* starts as possible cpus with threads reset |
|
* and each rcv avail reset. |
|
* |
|
* rcv avail gets node relative 1 wrapping back |
|
* to the node relative 1 as necessary. |
|
* |
|
*/ |
|
int hfi1_dev_affinity_init(struct hfi1_devdata *dd) |
|
{ |
|
struct hfi1_affinity_node *entry; |
|
const struct cpumask *local_mask; |
|
int curr_cpu, possible, i, ret; |
|
bool new_entry = false; |
|
|
|
local_mask = cpumask_of_node(dd->node); |
|
if (cpumask_first(local_mask) >= nr_cpu_ids) |
|
local_mask = topology_core_cpumask(0); |
|
|
|
mutex_lock(&node_affinity.lock); |
|
entry = node_affinity_lookup(dd->node); |
|
|
|
/* |
|
* If this is the first time this NUMA node's affinity is used, |
|
* create an entry in the global affinity structure and initialize it. |
|
*/ |
|
if (!entry) { |
|
entry = node_affinity_allocate(dd->node); |
|
if (!entry) { |
|
dd_dev_err(dd, |
|
"Unable to allocate global affinity node\n"); |
|
ret = -ENOMEM; |
|
goto fail; |
|
} |
|
new_entry = true; |
|
|
|
init_cpu_mask_set(&entry->def_intr); |
|
init_cpu_mask_set(&entry->rcv_intr); |
|
cpumask_clear(&entry->comp_vect_mask); |
|
cpumask_clear(&entry->general_intr_mask); |
|
/* Use the "real" cpu mask of this node as the default */ |
|
cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask, |
|
local_mask); |
|
|
|
/* fill in the receive list */ |
|
possible = cpumask_weight(&entry->def_intr.mask); |
|
curr_cpu = cpumask_first(&entry->def_intr.mask); |
|
|
|
if (possible == 1) { |
|
/* only one CPU, everyone will use it */ |
|
cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask); |
|
cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); |
|
} else { |
|
/* |
|
* The general/control context will be the first CPU in |
|
* the default list, so it is removed from the default |
|
* list and added to the general interrupt list. |
|
*/ |
|
cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask); |
|
cpumask_set_cpu(curr_cpu, &entry->general_intr_mask); |
|
curr_cpu = cpumask_next(curr_cpu, |
|
&entry->def_intr.mask); |
|
|
|
/* |
|
* Remove the remaining kernel receive queues from |
|
* the default list and add them to the receive list. |
|
*/ |
|
for (i = 0; |
|
i < (dd->n_krcv_queues - 1) * |
|
hfi1_per_node_cntr[dd->node]; |
|
i++) { |
|
cpumask_clear_cpu(curr_cpu, |
|
&entry->def_intr.mask); |
|
cpumask_set_cpu(curr_cpu, |
|
&entry->rcv_intr.mask); |
|
curr_cpu = cpumask_next(curr_cpu, |
|
&entry->def_intr.mask); |
|
if (curr_cpu >= nr_cpu_ids) |
|
break; |
|
} |
|
|
|
/* |
|
* If there ends up being 0 CPU cores leftover for SDMA |
|
* engines, use the same CPU cores as general/control |
|
* context. |
|
*/ |
|
if (cpumask_weight(&entry->def_intr.mask) == 0) |
|
cpumask_copy(&entry->def_intr.mask, |
|
&entry->general_intr_mask); |
|
} |
|
|
|
/* Determine completion vector CPUs for the entire node */ |
|
cpumask_and(&entry->comp_vect_mask, |
|
&node_affinity.real_cpu_mask, local_mask); |
|
cpumask_andnot(&entry->comp_vect_mask, |
|
&entry->comp_vect_mask, |
|
&entry->rcv_intr.mask); |
|
cpumask_andnot(&entry->comp_vect_mask, |
|
&entry->comp_vect_mask, |
|
&entry->general_intr_mask); |
|
|
|
/* |
|
* If there ends up being 0 CPU cores leftover for completion |
|
* vectors, use the same CPU core as the general/control |
|
* context. |
|
*/ |
|
if (cpumask_weight(&entry->comp_vect_mask) == 0) |
|
cpumask_copy(&entry->comp_vect_mask, |
|
&entry->general_intr_mask); |
|
} |
|
|
|
ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry); |
|
if (ret < 0) |
|
goto fail; |
|
|
|
if (new_entry) |
|
node_affinity_add_tail(entry); |
|
|
|
dd->affinity_entry = entry; |
|
mutex_unlock(&node_affinity.lock); |
|
|
|
return 0; |
|
|
|
fail: |
|
if (new_entry) |
|
node_affinity_destroy(entry); |
|
mutex_unlock(&node_affinity.lock); |
|
return ret; |
|
} |
|
|
|
void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd) |
|
{ |
|
struct hfi1_affinity_node *entry; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
if (!dd->affinity_entry) |
|
goto unlock; |
|
entry = node_affinity_lookup(dd->node); |
|
if (!entry) |
|
goto unlock; |
|
|
|
/* |
|
* Free device completion vector CPUs to be used by future |
|
* completion vectors |
|
*/ |
|
_dev_comp_vect_cpu_mask_clean_up(dd, entry); |
|
unlock: |
|
dd->affinity_entry = NULL; |
|
mutex_unlock(&node_affinity.lock); |
|
} |
|
|
|
/* |
|
* Function updates the irq affinity hint for msix after it has been changed |
|
* by the user using the /proc/irq interface. This function only accepts |
|
* one cpu in the mask. |
|
*/ |
|
static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu) |
|
{ |
|
struct sdma_engine *sde = msix->arg; |
|
struct hfi1_devdata *dd = sde->dd; |
|
struct hfi1_affinity_node *entry; |
|
struct cpu_mask_set *set; |
|
int i, old_cpu; |
|
|
|
if (cpu > num_online_cpus() || cpu == sde->cpu) |
|
return; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
entry = node_affinity_lookup(dd->node); |
|
if (!entry) |
|
goto unlock; |
|
|
|
old_cpu = sde->cpu; |
|
sde->cpu = cpu; |
|
cpumask_clear(&msix->mask); |
|
cpumask_set_cpu(cpu, &msix->mask); |
|
dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n", |
|
msix->irq, irq_type_names[msix->type], |
|
sde->this_idx, cpu); |
|
irq_set_affinity_hint(msix->irq, &msix->mask); |
|
|
|
/* |
|
* Set the new cpu in the hfi1_affinity_node and clean |
|
* the old cpu if it is not used by any other IRQ |
|
*/ |
|
set = &entry->def_intr; |
|
cpumask_set_cpu(cpu, &set->mask); |
|
cpumask_set_cpu(cpu, &set->used); |
|
for (i = 0; i < dd->msix_info.max_requested; i++) { |
|
struct hfi1_msix_entry *other_msix; |
|
|
|
other_msix = &dd->msix_info.msix_entries[i]; |
|
if (other_msix->type != IRQ_SDMA || other_msix == msix) |
|
continue; |
|
|
|
if (cpumask_test_cpu(old_cpu, &other_msix->mask)) |
|
goto unlock; |
|
} |
|
cpumask_clear_cpu(old_cpu, &set->mask); |
|
cpumask_clear_cpu(old_cpu, &set->used); |
|
unlock: |
|
mutex_unlock(&node_affinity.lock); |
|
} |
|
|
|
static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify, |
|
const cpumask_t *mask) |
|
{ |
|
int cpu = cpumask_first(mask); |
|
struct hfi1_msix_entry *msix = container_of(notify, |
|
struct hfi1_msix_entry, |
|
notify); |
|
|
|
/* Only one CPU configuration supported currently */ |
|
hfi1_update_sdma_affinity(msix, cpu); |
|
} |
|
|
|
static void hfi1_irq_notifier_release(struct kref *ref) |
|
{ |
|
/* |
|
* This is required by affinity notifier. We don't have anything to |
|
* free here. |
|
*/ |
|
} |
|
|
|
static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix) |
|
{ |
|
struct irq_affinity_notify *notify = &msix->notify; |
|
|
|
notify->irq = msix->irq; |
|
notify->notify = hfi1_irq_notifier_notify; |
|
notify->release = hfi1_irq_notifier_release; |
|
|
|
if (irq_set_affinity_notifier(notify->irq, notify)) |
|
pr_err("Failed to register sdma irq affinity notifier for irq %d\n", |
|
notify->irq); |
|
} |
|
|
|
static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix) |
|
{ |
|
struct irq_affinity_notify *notify = &msix->notify; |
|
|
|
if (irq_set_affinity_notifier(notify->irq, NULL)) |
|
pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n", |
|
notify->irq); |
|
} |
|
|
|
/* |
|
* Function sets the irq affinity for msix. |
|
* It *must* be called with node_affinity.lock held. |
|
*/ |
|
static int get_irq_affinity(struct hfi1_devdata *dd, |
|
struct hfi1_msix_entry *msix) |
|
{ |
|
cpumask_var_t diff; |
|
struct hfi1_affinity_node *entry; |
|
struct cpu_mask_set *set = NULL; |
|
struct sdma_engine *sde = NULL; |
|
struct hfi1_ctxtdata *rcd = NULL; |
|
char extra[64]; |
|
int cpu = -1; |
|
|
|
extra[0] = '\0'; |
|
cpumask_clear(&msix->mask); |
|
|
|
entry = node_affinity_lookup(dd->node); |
|
|
|
switch (msix->type) { |
|
case IRQ_SDMA: |
|
sde = (struct sdma_engine *)msix->arg; |
|
scnprintf(extra, 64, "engine %u", sde->this_idx); |
|
set = &entry->def_intr; |
|
break; |
|
case IRQ_GENERAL: |
|
cpu = cpumask_first(&entry->general_intr_mask); |
|
break; |
|
case IRQ_RCVCTXT: |
|
rcd = (struct hfi1_ctxtdata *)msix->arg; |
|
if (rcd->ctxt == HFI1_CTRL_CTXT) |
|
cpu = cpumask_first(&entry->general_intr_mask); |
|
else |
|
set = &entry->rcv_intr; |
|
scnprintf(extra, 64, "ctxt %u", rcd->ctxt); |
|
break; |
|
case IRQ_NETDEVCTXT: |
|
rcd = (struct hfi1_ctxtdata *)msix->arg; |
|
set = &entry->def_intr; |
|
scnprintf(extra, 64, "ctxt %u", rcd->ctxt); |
|
break; |
|
default: |
|
dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type); |
|
return -EINVAL; |
|
} |
|
|
|
/* |
|
* The general and control contexts are placed on a particular |
|
* CPU, which is set above. Skip accounting for it. Everything else |
|
* finds its CPU here. |
|
*/ |
|
if (cpu == -1 && set) { |
|
if (!zalloc_cpumask_var(&diff, GFP_KERNEL)) |
|
return -ENOMEM; |
|
|
|
cpu = cpu_mask_set_get_first(set, diff); |
|
if (cpu < 0) { |
|
free_cpumask_var(diff); |
|
dd_dev_err(dd, "Failure to obtain CPU for IRQ\n"); |
|
return cpu; |
|
} |
|
|
|
free_cpumask_var(diff); |
|
} |
|
|
|
cpumask_set_cpu(cpu, &msix->mask); |
|
dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n", |
|
msix->irq, irq_type_names[msix->type], |
|
extra, cpu); |
|
irq_set_affinity_hint(msix->irq, &msix->mask); |
|
|
|
if (msix->type == IRQ_SDMA) { |
|
sde->cpu = cpu; |
|
hfi1_setup_sdma_notifier(msix); |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix) |
|
{ |
|
int ret; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
ret = get_irq_affinity(dd, msix); |
|
mutex_unlock(&node_affinity.lock); |
|
return ret; |
|
} |
|
|
|
void hfi1_put_irq_affinity(struct hfi1_devdata *dd, |
|
struct hfi1_msix_entry *msix) |
|
{ |
|
struct cpu_mask_set *set = NULL; |
|
struct hfi1_ctxtdata *rcd; |
|
struct hfi1_affinity_node *entry; |
|
|
|
mutex_lock(&node_affinity.lock); |
|
entry = node_affinity_lookup(dd->node); |
|
|
|
switch (msix->type) { |
|
case IRQ_SDMA: |
|
set = &entry->def_intr; |
|
hfi1_cleanup_sdma_notifier(msix); |
|
break; |
|
case IRQ_GENERAL: |
|
/* Don't do accounting for general contexts */ |
|
break; |
|
case IRQ_RCVCTXT: |
|
rcd = (struct hfi1_ctxtdata *)msix->arg; |
|
/* Don't do accounting for control contexts */ |
|
if (rcd->ctxt != HFI1_CTRL_CTXT) |
|
set = &entry->rcv_intr; |
|
break; |
|
case IRQ_NETDEVCTXT: |
|
rcd = (struct hfi1_ctxtdata *)msix->arg; |
|
set = &entry->def_intr; |
|
break; |
|
default: |
|
mutex_unlock(&node_affinity.lock); |
|
return; |
|
} |
|
|
|
if (set) { |
|
cpumask_andnot(&set->used, &set->used, &msix->mask); |
|
_cpu_mask_set_gen_dec(set); |
|
} |
|
|
|
irq_set_affinity_hint(msix->irq, NULL); |
|
cpumask_clear(&msix->mask); |
|
mutex_unlock(&node_affinity.lock); |
|
} |
|
|
|
/* This should be called with node_affinity.lock held */ |
|
static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask, |
|
struct hfi1_affinity_node_list *affinity) |
|
{ |
|
int possible, curr_cpu, i; |
|
uint num_cores_per_socket = node_affinity.num_online_cpus / |
|
affinity->num_core_siblings / |
|
node_affinity.num_online_nodes; |
|
|
|
cpumask_copy(hw_thread_mask, &affinity->proc.mask); |
|
if (affinity->num_core_siblings > 0) { |
|
/* Removing other siblings not needed for now */ |
|
possible = cpumask_weight(hw_thread_mask); |
|
curr_cpu = cpumask_first(hw_thread_mask); |
|
for (i = 0; |
|
i < num_cores_per_socket * node_affinity.num_online_nodes; |
|
i++) |
|
curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); |
|
|
|
for (; i < possible; i++) { |
|
cpumask_clear_cpu(curr_cpu, hw_thread_mask); |
|
curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); |
|
} |
|
|
|
/* Identifying correct HW threads within physical cores */ |
|
cpumask_shift_left(hw_thread_mask, hw_thread_mask, |
|
num_cores_per_socket * |
|
node_affinity.num_online_nodes * |
|
hw_thread_no); |
|
} |
|
} |
|
|
|
int hfi1_get_proc_affinity(int node) |
|
{ |
|
int cpu = -1, ret, i; |
|
struct hfi1_affinity_node *entry; |
|
cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask; |
|
const struct cpumask *node_mask, |
|
*proc_mask = current->cpus_ptr; |
|
struct hfi1_affinity_node_list *affinity = &node_affinity; |
|
struct cpu_mask_set *set = &affinity->proc; |
|
|
|
/* |
|
* check whether process/context affinity has already |
|
* been set |
|
*/ |
|
if (current->nr_cpus_allowed == 1) { |
|
hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl", |
|
current->pid, current->comm, |
|
cpumask_pr_args(proc_mask)); |
|
/* |
|
* Mark the pre-set CPU as used. This is atomic so we don't |
|
* need the lock |
|
*/ |
|
cpu = cpumask_first(proc_mask); |
|
cpumask_set_cpu(cpu, &set->used); |
|
goto done; |
|
} else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) { |
|
hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl", |
|
current->pid, current->comm, |
|
cpumask_pr_args(proc_mask)); |
|
goto done; |
|
} |
|
|
|
/* |
|
* The process does not have a preset CPU affinity so find one to |
|
* recommend using the following algorithm: |
|
* |
|
* For each user process that is opening a context on HFI Y: |
|
* a) If all cores are filled, reinitialize the bitmask |
|
* b) Fill real cores first, then HT cores (First set of HT |
|
* cores on all physical cores, then second set of HT core, |
|
* and, so on) in the following order: |
|
* |
|
* 1. Same NUMA node as HFI Y and not running an IRQ |
|
* handler |
|
* 2. Same NUMA node as HFI Y and running an IRQ handler |
|
* 3. Different NUMA node to HFI Y and not running an IRQ |
|
* handler |
|
* 4. Different NUMA node to HFI Y and running an IRQ |
|
* handler |
|
* c) Mark core as filled in the bitmask. As user processes are |
|
* done, clear cores from the bitmask. |
|
*/ |
|
|
|
ret = zalloc_cpumask_var(&diff, GFP_KERNEL); |
|
if (!ret) |
|
goto done; |
|
ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL); |
|
if (!ret) |
|
goto free_diff; |
|
ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL); |
|
if (!ret) |
|
goto free_hw_thread_mask; |
|
ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL); |
|
if (!ret) |
|
goto free_available_mask; |
|
|
|
mutex_lock(&affinity->lock); |
|
/* |
|
* If we've used all available HW threads, clear the mask and start |
|
* overloading. |
|
*/ |
|
_cpu_mask_set_gen_inc(set); |
|
|
|
/* |
|
* If NUMA node has CPUs used by interrupt handlers, include them in the |
|
* interrupt handler mask. |
|
*/ |
|
entry = node_affinity_lookup(node); |
|
if (entry) { |
|
cpumask_copy(intrs_mask, (entry->def_intr.gen ? |
|
&entry->def_intr.mask : |
|
&entry->def_intr.used)); |
|
cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ? |
|
&entry->rcv_intr.mask : |
|
&entry->rcv_intr.used)); |
|
cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask); |
|
} |
|
hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl", |
|
cpumask_pr_args(intrs_mask)); |
|
|
|
cpumask_copy(hw_thread_mask, &set->mask); |
|
|
|
/* |
|
* If HT cores are enabled, identify which HW threads within the |
|
* physical cores should be used. |
|
*/ |
|
if (affinity->num_core_siblings > 0) { |
|
for (i = 0; i < affinity->num_core_siblings; i++) { |
|
find_hw_thread_mask(i, hw_thread_mask, affinity); |
|
|
|
/* |
|
* If there's at least one available core for this HW |
|
* thread number, stop looking for a core. |
|
* |
|
* diff will always be not empty at least once in this |
|
* loop as the used mask gets reset when |
|
* (set->mask == set->used) before this loop. |
|
*/ |
|
cpumask_andnot(diff, hw_thread_mask, &set->used); |
|
if (!cpumask_empty(diff)) |
|
break; |
|
} |
|
} |
|
hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl", |
|
cpumask_pr_args(hw_thread_mask)); |
|
|
|
node_mask = cpumask_of_node(node); |
|
hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node, |
|
cpumask_pr_args(node_mask)); |
|
|
|
/* Get cpumask of available CPUs on preferred NUMA */ |
|
cpumask_and(available_mask, hw_thread_mask, node_mask); |
|
cpumask_andnot(available_mask, available_mask, &set->used); |
|
hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node, |
|
cpumask_pr_args(available_mask)); |
|
|
|
/* |
|
* At first, we don't want to place processes on the same |
|
* CPUs as interrupt handlers. Then, CPUs running interrupt |
|
* handlers are used. |
|
* |
|
* 1) If diff is not empty, then there are CPUs not running |
|
* non-interrupt handlers available, so diff gets copied |
|
* over to available_mask. |
|
* 2) If diff is empty, then all CPUs not running interrupt |
|
* handlers are taken, so available_mask contains all |
|
* available CPUs running interrupt handlers. |
|
* 3) If available_mask is empty, then all CPUs on the |
|
* preferred NUMA node are taken, so other NUMA nodes are |
|
* used for process assignments using the same method as |
|
* the preferred NUMA node. |
|
*/ |
|
cpumask_andnot(diff, available_mask, intrs_mask); |
|
if (!cpumask_empty(diff)) |
|
cpumask_copy(available_mask, diff); |
|
|
|
/* If we don't have CPUs on the preferred node, use other NUMA nodes */ |
|
if (cpumask_empty(available_mask)) { |
|
cpumask_andnot(available_mask, hw_thread_mask, &set->used); |
|
/* Excluding preferred NUMA cores */ |
|
cpumask_andnot(available_mask, available_mask, node_mask); |
|
hfi1_cdbg(PROC, |
|
"Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl", |
|
cpumask_pr_args(available_mask)); |
|
|
|
/* |
|
* At first, we don't want to place processes on the same |
|
* CPUs as interrupt handlers. |
|
*/ |
|
cpumask_andnot(diff, available_mask, intrs_mask); |
|
if (!cpumask_empty(diff)) |
|
cpumask_copy(available_mask, diff); |
|
} |
|
hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl", |
|
cpumask_pr_args(available_mask)); |
|
|
|
cpu = cpumask_first(available_mask); |
|
if (cpu >= nr_cpu_ids) /* empty */ |
|
cpu = -1; |
|
else |
|
cpumask_set_cpu(cpu, &set->used); |
|
|
|
mutex_unlock(&affinity->lock); |
|
hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu); |
|
|
|
free_cpumask_var(intrs_mask); |
|
free_available_mask: |
|
free_cpumask_var(available_mask); |
|
free_hw_thread_mask: |
|
free_cpumask_var(hw_thread_mask); |
|
free_diff: |
|
free_cpumask_var(diff); |
|
done: |
|
return cpu; |
|
} |
|
|
|
void hfi1_put_proc_affinity(int cpu) |
|
{ |
|
struct hfi1_affinity_node_list *affinity = &node_affinity; |
|
struct cpu_mask_set *set = &affinity->proc; |
|
|
|
if (cpu < 0) |
|
return; |
|
|
|
mutex_lock(&affinity->lock); |
|
cpu_mask_set_put(set, cpu); |
|
hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu); |
|
mutex_unlock(&affinity->lock); |
|
}
|
|
|