mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
657 lines
17 KiB
657 lines
17 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Cell Broadband Engine OProfile Support |
|
* |
|
* (C) Copyright IBM Corporation 2006 |
|
* |
|
* Author: Maynard Johnson <[email protected]> |
|
*/ |
|
|
|
/* The purpose of this file is to handle SPU event task switching |
|
* and to record SPU context information into the OProfile |
|
* event buffer. |
|
* |
|
* Additionally, the spu_sync_buffer function is provided as a helper |
|
* for recoding actual SPU program counter samples to the event buffer. |
|
*/ |
|
#include <linux/dcookies.h> |
|
#include <linux/kref.h> |
|
#include <linux/mm.h> |
|
#include <linux/fs.h> |
|
#include <linux/file.h> |
|
#include <linux/module.h> |
|
#include <linux/notifier.h> |
|
#include <linux/numa.h> |
|
#include <linux/oprofile.h> |
|
#include <linux/slab.h> |
|
#include <linux/spinlock.h> |
|
#include "pr_util.h" |
|
|
|
#define RELEASE_ALL 9999 |
|
|
|
static DEFINE_SPINLOCK(buffer_lock); |
|
static DEFINE_SPINLOCK(cache_lock); |
|
static int num_spu_nodes; |
|
static int spu_prof_num_nodes; |
|
|
|
struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE]; |
|
struct delayed_work spu_work; |
|
static unsigned max_spu_buff; |
|
|
|
static void spu_buff_add(unsigned long int value, int spu) |
|
{ |
|
/* spu buff is a circular buffer. Add entries to the |
|
* head. Head is the index to store the next value. |
|
* The buffer is full when there is one available entry |
|
* in the queue, i.e. head and tail can't be equal. |
|
* That way we can tell the difference between the |
|
* buffer being full versus empty. |
|
* |
|
* ASSUMPTION: the buffer_lock is held when this function |
|
* is called to lock the buffer, head and tail. |
|
*/ |
|
int full = 1; |
|
|
|
if (spu_buff[spu].head >= spu_buff[spu].tail) { |
|
if ((spu_buff[spu].head - spu_buff[spu].tail) |
|
< (max_spu_buff - 1)) |
|
full = 0; |
|
|
|
} else if (spu_buff[spu].tail > spu_buff[spu].head) { |
|
if ((spu_buff[spu].tail - spu_buff[spu].head) |
|
> 1) |
|
full = 0; |
|
} |
|
|
|
if (!full) { |
|
spu_buff[spu].buff[spu_buff[spu].head] = value; |
|
spu_buff[spu].head++; |
|
|
|
if (spu_buff[spu].head >= max_spu_buff) |
|
spu_buff[spu].head = 0; |
|
} else { |
|
/* From the user's perspective make the SPU buffer |
|
* size management/overflow look like we are using |
|
* per cpu buffers. The user uses the same |
|
* per cpu parameter to adjust the SPU buffer size. |
|
* Increment the sample_lost_overflow to inform |
|
* the user the buffer size needs to be increased. |
|
*/ |
|
oprofile_cpu_buffer_inc_smpl_lost(); |
|
} |
|
} |
|
|
|
/* This function copies the per SPU buffers to the |
|
* OProfile kernel buffer. |
|
*/ |
|
static void sync_spu_buff(void) |
|
{ |
|
int spu; |
|
unsigned long flags; |
|
int curr_head; |
|
|
|
for (spu = 0; spu < num_spu_nodes; spu++) { |
|
/* In case there was an issue and the buffer didn't |
|
* get created skip it. |
|
*/ |
|
if (spu_buff[spu].buff == NULL) |
|
continue; |
|
|
|
/* Hold the lock to make sure the head/tail |
|
* doesn't change while spu_buff_add() is |
|
* deciding if the buffer is full or not. |
|
* Being a little paranoid. |
|
*/ |
|
spin_lock_irqsave(&buffer_lock, flags); |
|
curr_head = spu_buff[spu].head; |
|
spin_unlock_irqrestore(&buffer_lock, flags); |
|
|
|
/* Transfer the current contents to the kernel buffer. |
|
* data can still be added to the head of the buffer. |
|
*/ |
|
oprofile_put_buff(spu_buff[spu].buff, |
|
spu_buff[spu].tail, |
|
curr_head, max_spu_buff); |
|
|
|
spin_lock_irqsave(&buffer_lock, flags); |
|
spu_buff[spu].tail = curr_head; |
|
spin_unlock_irqrestore(&buffer_lock, flags); |
|
} |
|
|
|
} |
|
|
|
static void wq_sync_spu_buff(struct work_struct *work) |
|
{ |
|
/* move data from spu buffers to kernel buffer */ |
|
sync_spu_buff(); |
|
|
|
/* only reschedule if profiling is not done */ |
|
if (spu_prof_running) |
|
schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE); |
|
} |
|
|
|
/* Container for caching information about an active SPU task. */ |
|
struct cached_info { |
|
struct vma_to_fileoffset_map *map; |
|
struct spu *the_spu; /* needed to access pointer to local_store */ |
|
struct kref cache_ref; |
|
}; |
|
|
|
static struct cached_info *spu_info[MAX_NUMNODES * 8]; |
|
|
|
static void destroy_cached_info(struct kref *kref) |
|
{ |
|
struct cached_info *info; |
|
|
|
info = container_of(kref, struct cached_info, cache_ref); |
|
vma_map_free(info->map); |
|
kfree(info); |
|
module_put(THIS_MODULE); |
|
} |
|
|
|
/* Return the cached_info for the passed SPU number. |
|
* ATTENTION: Callers are responsible for obtaining the |
|
* cache_lock if needed prior to invoking this function. |
|
*/ |
|
static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num) |
|
{ |
|
struct kref *ref; |
|
struct cached_info *ret_info; |
|
|
|
if (spu_num >= num_spu_nodes) { |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: Invalid index %d into spu info cache\n", |
|
__func__, __LINE__, spu_num); |
|
ret_info = NULL; |
|
goto out; |
|
} |
|
if (!spu_info[spu_num] && the_spu) { |
|
ref = spu_get_profile_private_kref(the_spu->ctx); |
|
if (ref) { |
|
spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref); |
|
kref_get(&spu_info[spu_num]->cache_ref); |
|
} |
|
} |
|
|
|
ret_info = spu_info[spu_num]; |
|
out: |
|
return ret_info; |
|
} |
|
|
|
|
|
/* Looks for cached info for the passed spu. If not found, the |
|
* cached info is created for the passed spu. |
|
* Returns 0 for success; otherwise, -1 for error. |
|
*/ |
|
static int |
|
prepare_cached_spu_info(struct spu *spu, unsigned long objectId) |
|
{ |
|
unsigned long flags; |
|
struct vma_to_fileoffset_map *new_map; |
|
int retval = 0; |
|
struct cached_info *info; |
|
|
|
/* We won't bother getting cache_lock here since |
|
* don't do anything with the cached_info that's returned. |
|
*/ |
|
info = get_cached_info(spu, spu->number); |
|
|
|
if (info) { |
|
pr_debug("Found cached SPU info.\n"); |
|
goto out; |
|
} |
|
|
|
/* Create cached_info and set spu_info[spu->number] to point to it. |
|
* spu->number is a system-wide value, not a per-node value. |
|
*/ |
|
info = kzalloc(sizeof(*info), GFP_KERNEL); |
|
if (!info) { |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: create vma_map failed\n", |
|
__func__, __LINE__); |
|
retval = -ENOMEM; |
|
goto err_alloc; |
|
} |
|
new_map = create_vma_map(spu, objectId); |
|
if (!new_map) { |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: create vma_map failed\n", |
|
__func__, __LINE__); |
|
retval = -ENOMEM; |
|
goto err_alloc; |
|
} |
|
|
|
pr_debug("Created vma_map\n"); |
|
info->map = new_map; |
|
info->the_spu = spu; |
|
kref_init(&info->cache_ref); |
|
spin_lock_irqsave(&cache_lock, flags); |
|
spu_info[spu->number] = info; |
|
/* Increment count before passing off ref to SPUFS. */ |
|
kref_get(&info->cache_ref); |
|
|
|
/* We increment the module refcount here since SPUFS is |
|
* responsible for the final destruction of the cached_info, |
|
* and it must be able to access the destroy_cached_info() |
|
* function defined in the OProfile module. We decrement |
|
* the module refcount in destroy_cached_info. |
|
*/ |
|
try_module_get(THIS_MODULE); |
|
spu_set_profile_private_kref(spu->ctx, &info->cache_ref, |
|
destroy_cached_info); |
|
spin_unlock_irqrestore(&cache_lock, flags); |
|
goto out; |
|
|
|
err_alloc: |
|
kfree(info); |
|
out: |
|
return retval; |
|
} |
|
|
|
/* |
|
* NOTE: The caller is responsible for locking the |
|
* cache_lock prior to calling this function. |
|
*/ |
|
static int release_cached_info(int spu_index) |
|
{ |
|
int index, end; |
|
|
|
if (spu_index == RELEASE_ALL) { |
|
end = num_spu_nodes; |
|
index = 0; |
|
} else { |
|
if (spu_index >= num_spu_nodes) { |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: " |
|
"Invalid index %d into spu info cache\n", |
|
__func__, __LINE__, spu_index); |
|
goto out; |
|
} |
|
end = spu_index + 1; |
|
index = spu_index; |
|
} |
|
for (; index < end; index++) { |
|
if (spu_info[index]) { |
|
kref_put(&spu_info[index]->cache_ref, |
|
destroy_cached_info); |
|
spu_info[index] = NULL; |
|
} |
|
} |
|
|
|
out: |
|
return 0; |
|
} |
|
|
|
/* The source code for fast_get_dcookie was "borrowed" |
|
* from drivers/oprofile/buffer_sync.c. |
|
*/ |
|
|
|
/* Optimisation. We can manage without taking the dcookie sem |
|
* because we cannot reach this code without at least one |
|
* dcookie user still being registered (namely, the reader |
|
* of the event buffer). |
|
*/ |
|
static inline unsigned long fast_get_dcookie(const struct path *path) |
|
{ |
|
unsigned long cookie; |
|
|
|
if (path->dentry->d_flags & DCACHE_COOKIE) |
|
return (unsigned long)path->dentry; |
|
get_dcookie(path, &cookie); |
|
return cookie; |
|
} |
|
|
|
/* Look up the dcookie for the task's mm->exe_file, |
|
* which corresponds loosely to "application name". Also, determine |
|
* the offset for the SPU ELF object. If computed offset is |
|
* non-zero, it implies an embedded SPU object; otherwise, it's a |
|
* separate SPU binary, in which case we retrieve it's dcookie. |
|
* For the embedded case, we must determine if SPU ELF is embedded |
|
* in the executable application or another file (i.e., shared lib). |
|
* If embedded in a shared lib, we must get the dcookie and return |
|
* that to the caller. |
|
*/ |
|
static unsigned long |
|
get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, |
|
unsigned long *spu_bin_dcookie, |
|
unsigned long spu_ref) |
|
{ |
|
unsigned long app_cookie = 0; |
|
unsigned int my_offset = 0; |
|
struct vm_area_struct *vma; |
|
struct file *exe_file; |
|
struct mm_struct *mm = spu->mm; |
|
|
|
if (!mm) |
|
goto out; |
|
|
|
exe_file = get_mm_exe_file(mm); |
|
if (exe_file) { |
|
app_cookie = fast_get_dcookie(&exe_file->f_path); |
|
pr_debug("got dcookie for %pD\n", exe_file); |
|
fput(exe_file); |
|
} |
|
|
|
mmap_read_lock(mm); |
|
for (vma = mm->mmap; vma; vma = vma->vm_next) { |
|
if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) |
|
continue; |
|
my_offset = spu_ref - vma->vm_start; |
|
if (!vma->vm_file) |
|
goto fail_no_image_cookie; |
|
|
|
pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n", |
|
my_offset, spu_ref, vma->vm_file); |
|
*offsetp = my_offset; |
|
break; |
|
} |
|
|
|
*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path); |
|
pr_debug("got dcookie for %pD\n", vma->vm_file); |
|
|
|
mmap_read_unlock(mm); |
|
|
|
out: |
|
return app_cookie; |
|
|
|
fail_no_image_cookie: |
|
mmap_read_unlock(mm); |
|
|
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: Cannot find dcookie for SPU binary\n", |
|
__func__, __LINE__); |
|
goto out; |
|
} |
|
|
|
|
|
|
|
/* This function finds or creates cached context information for the |
|
* passed SPU and records SPU context information into the OProfile |
|
* event buffer. |
|
*/ |
|
static int process_context_switch(struct spu *spu, unsigned long objectId) |
|
{ |
|
unsigned long flags; |
|
int retval; |
|
unsigned int offset = 0; |
|
unsigned long spu_cookie = 0, app_dcookie; |
|
|
|
retval = prepare_cached_spu_info(spu, objectId); |
|
if (retval) |
|
goto out; |
|
|
|
/* Get dcookie first because a mutex_lock is taken in that |
|
* code path, so interrupts must not be disabled. |
|
*/ |
|
app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId); |
|
if (!app_dcookie || !spu_cookie) { |
|
retval = -ENOENT; |
|
goto out; |
|
} |
|
|
|
/* Record context info in event buffer */ |
|
spin_lock_irqsave(&buffer_lock, flags); |
|
spu_buff_add(ESCAPE_CODE, spu->number); |
|
spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number); |
|
spu_buff_add(spu->number, spu->number); |
|
spu_buff_add(spu->pid, spu->number); |
|
spu_buff_add(spu->tgid, spu->number); |
|
spu_buff_add(app_dcookie, spu->number); |
|
spu_buff_add(spu_cookie, spu->number); |
|
spu_buff_add(offset, spu->number); |
|
|
|
/* Set flag to indicate SPU PC data can now be written out. If |
|
* the SPU program counter data is seen before an SPU context |
|
* record is seen, the postprocessing will fail. |
|
*/ |
|
spu_buff[spu->number].ctx_sw_seen = 1; |
|
|
|
spin_unlock_irqrestore(&buffer_lock, flags); |
|
smp_wmb(); /* insure spu event buffer updates are written */ |
|
/* don't want entries intermingled... */ |
|
out: |
|
return retval; |
|
} |
|
|
|
/* |
|
* This function is invoked on either a bind_context or unbind_context. |
|
* If called for an unbind_context, the val arg is 0; otherwise, |
|
* it is the object-id value for the spu context. |
|
* The data arg is of type 'struct spu *'. |
|
*/ |
|
static int spu_active_notify(struct notifier_block *self, unsigned long val, |
|
void *data) |
|
{ |
|
int retval; |
|
unsigned long flags; |
|
struct spu *the_spu = data; |
|
|
|
pr_debug("SPU event notification arrived\n"); |
|
if (!val) { |
|
spin_lock_irqsave(&cache_lock, flags); |
|
retval = release_cached_info(the_spu->number); |
|
spin_unlock_irqrestore(&cache_lock, flags); |
|
} else { |
|
retval = process_context_switch(the_spu, val); |
|
} |
|
return retval; |
|
} |
|
|
|
static struct notifier_block spu_active = { |
|
.notifier_call = spu_active_notify, |
|
}; |
|
|
|
static int number_of_online_nodes(void) |
|
{ |
|
u32 cpu; u32 tmp; |
|
int nodes = 0; |
|
for_each_online_cpu(cpu) { |
|
tmp = cbe_cpu_to_node(cpu) + 1; |
|
if (tmp > nodes) |
|
nodes++; |
|
} |
|
return nodes; |
|
} |
|
|
|
static int oprofile_spu_buff_create(void) |
|
{ |
|
int spu; |
|
|
|
max_spu_buff = oprofile_get_cpu_buffer_size(); |
|
|
|
for (spu = 0; spu < num_spu_nodes; spu++) { |
|
/* create circular buffers to store the data in. |
|
* use locks to manage accessing the buffers |
|
*/ |
|
spu_buff[spu].head = 0; |
|
spu_buff[spu].tail = 0; |
|
|
|
/* |
|
* Create a buffer for each SPU. Can't reliably |
|
* create a single buffer for all spus due to not |
|
* enough contiguous kernel memory. |
|
*/ |
|
|
|
spu_buff[spu].buff = kzalloc((max_spu_buff |
|
* sizeof(unsigned long)), |
|
GFP_KERNEL); |
|
|
|
if (!spu_buff[spu].buff) { |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: oprofile_spu_buff_create " |
|
"failed to allocate spu buffer %d.\n", |
|
__func__, __LINE__, spu); |
|
|
|
/* release the spu buffers that have been allocated */ |
|
while (spu >= 0) { |
|
kfree(spu_buff[spu].buff); |
|
spu_buff[spu].buff = 0; |
|
spu--; |
|
} |
|
return -ENOMEM; |
|
} |
|
} |
|
return 0; |
|
} |
|
|
|
/* The main purpose of this function is to synchronize |
|
* OProfile with SPUFS by registering to be notified of |
|
* SPU task switches. |
|
* |
|
* NOTE: When profiling SPUs, we must ensure that only |
|
* spu_sync_start is invoked and not the generic sync_start |
|
* in drivers/oprofile/oprof.c. A return value of |
|
* SKIP_GENERIC_SYNC or SYNC_START_ERROR will |
|
* accomplish this. |
|
*/ |
|
int spu_sync_start(void) |
|
{ |
|
int spu; |
|
int ret = SKIP_GENERIC_SYNC; |
|
int register_ret; |
|
unsigned long flags = 0; |
|
|
|
spu_prof_num_nodes = number_of_online_nodes(); |
|
num_spu_nodes = spu_prof_num_nodes * 8; |
|
INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff); |
|
|
|
/* create buffer for storing the SPU data to put in |
|
* the kernel buffer. |
|
*/ |
|
ret = oprofile_spu_buff_create(); |
|
if (ret) |
|
goto out; |
|
|
|
spin_lock_irqsave(&buffer_lock, flags); |
|
for (spu = 0; spu < num_spu_nodes; spu++) { |
|
spu_buff_add(ESCAPE_CODE, spu); |
|
spu_buff_add(SPU_PROFILING_CODE, spu); |
|
spu_buff_add(num_spu_nodes, spu); |
|
} |
|
spin_unlock_irqrestore(&buffer_lock, flags); |
|
|
|
for (spu = 0; spu < num_spu_nodes; spu++) { |
|
spu_buff[spu].ctx_sw_seen = 0; |
|
spu_buff[spu].last_guard_val = 0; |
|
} |
|
|
|
/* Register for SPU events */ |
|
register_ret = spu_switch_event_register(&spu_active); |
|
if (register_ret) { |
|
ret = SYNC_START_ERROR; |
|
goto out; |
|
} |
|
|
|
pr_debug("spu_sync_start -- running.\n"); |
|
out: |
|
return ret; |
|
} |
|
|
|
/* Record SPU program counter samples to the oprofile event buffer. */ |
|
void spu_sync_buffer(int spu_num, unsigned int *samples, |
|
int num_samples) |
|
{ |
|
unsigned long long file_offset; |
|
unsigned long flags; |
|
int i; |
|
struct vma_to_fileoffset_map *map; |
|
struct spu *the_spu; |
|
unsigned long long spu_num_ll = spu_num; |
|
unsigned long long spu_num_shifted = spu_num_ll << 32; |
|
struct cached_info *c_info; |
|
|
|
/* We need to obtain the cache_lock here because it's |
|
* possible that after getting the cached_info, the SPU job |
|
* corresponding to this cached_info may end, thus resulting |
|
* in the destruction of the cached_info. |
|
*/ |
|
spin_lock_irqsave(&cache_lock, flags); |
|
c_info = get_cached_info(NULL, spu_num); |
|
if (!c_info) { |
|
/* This legitimately happens when the SPU task ends before all |
|
* samples are recorded. |
|
* No big deal -- so we just drop a few samples. |
|
*/ |
|
pr_debug("SPU_PROF: No cached SPU context " |
|
"for SPU #%d. Dropping samples.\n", spu_num); |
|
goto out; |
|
} |
|
|
|
map = c_info->map; |
|
the_spu = c_info->the_spu; |
|
spin_lock(&buffer_lock); |
|
for (i = 0; i < num_samples; i++) { |
|
unsigned int sample = *(samples+i); |
|
int grd_val = 0; |
|
file_offset = 0; |
|
if (sample == 0) |
|
continue; |
|
file_offset = vma_map_lookup( map, sample, the_spu, &grd_val); |
|
|
|
/* If overlays are used by this SPU application, the guard |
|
* value is non-zero, indicating which overlay section is in |
|
* use. We need to discard samples taken during the time |
|
* period which an overlay occurs (i.e., guard value changes). |
|
*/ |
|
if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) { |
|
spu_buff[spu_num].last_guard_val = grd_val; |
|
/* Drop the rest of the samples. */ |
|
break; |
|
} |
|
|
|
/* We must ensure that the SPU context switch has been written |
|
* out before samples for the SPU. Otherwise, the SPU context |
|
* information is not available and the postprocessing of the |
|
* SPU PC will fail with no available anonymous map information. |
|
*/ |
|
if (spu_buff[spu_num].ctx_sw_seen) |
|
spu_buff_add((file_offset | spu_num_shifted), |
|
spu_num); |
|
} |
|
spin_unlock(&buffer_lock); |
|
out: |
|
spin_unlock_irqrestore(&cache_lock, flags); |
|
} |
|
|
|
|
|
int spu_sync_stop(void) |
|
{ |
|
unsigned long flags = 0; |
|
int ret; |
|
int k; |
|
|
|
ret = spu_switch_event_unregister(&spu_active); |
|
|
|
if (ret) |
|
printk(KERN_ERR "SPU_PROF: " |
|
"%s, line %d: spu_switch_event_unregister " \ |
|
"returned %d\n", |
|
__func__, __LINE__, ret); |
|
|
|
/* flush any remaining data in the per SPU buffers */ |
|
sync_spu_buff(); |
|
|
|
spin_lock_irqsave(&cache_lock, flags); |
|
ret = release_cached_info(RELEASE_ALL); |
|
spin_unlock_irqrestore(&cache_lock, flags); |
|
|
|
/* remove scheduled work queue item rather then waiting |
|
* for every queued entry to execute. Then flush pending |
|
* system wide buffer to event buffer. |
|
*/ |
|
cancel_delayed_work(&spu_work); |
|
|
|
for (k = 0; k < num_spu_nodes; k++) { |
|
spu_buff[k].ctx_sw_seen = 0; |
|
|
|
/* |
|
* spu_sys_buff will be null if there was a problem |
|
* allocating the buffer. Only delete if it exists. |
|
*/ |
|
kfree(spu_buff[k].buff); |
|
spu_buff[k].buff = 0; |
|
} |
|
pr_debug("spu_sync_stop -- done.\n"); |
|
return ret; |
|
} |
|
|
|
|