forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
605 lines
16 KiB
605 lines
16 KiB
/* SPDX-License-Identifier: GPL-2.0 */ |
|
/* |
|
* arch/alpha/lib/ev6-memset.S |
|
* |
|
* This is an efficient (and relatively small) implementation of the C library |
|
* "memset()" function for the 21264 implementation of Alpha. |
|
* |
|
* 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com> |
|
* |
|
* Much of the information about 21264 scheduling/coding comes from: |
|
* Compiler Writer's Guide for the Alpha 21264 |
|
* abbreviated as 'CWG' in other comments here |
|
* ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html |
|
* Scheduling notation: |
|
* E - either cluster |
|
* U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 |
|
* L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 |
|
* The algorithm for the leading and trailing quadwords remains the same, |
|
* however the loop has been unrolled to enable better memory throughput, |
|
* and the code has been replicated for each of the entry points: __memset |
|
* and __memset16 to permit better scheduling to eliminate the stalling |
|
* encountered during the mask replication. |
|
* A future enhancement might be to put in a byte store loop for really |
|
* small (say < 32 bytes) memset()s. Whether or not that change would be |
|
* a win in the kernel would depend upon the contextual usage. |
|
* WARNING: Maintaining this is going to be more work than the above version, |
|
* as fixes will need to be made in multiple places. The performance gain |
|
* is worth it. |
|
*/ |
|
#include <asm/export.h> |
|
.set noat |
|
.set noreorder |
|
.text |
|
.globl memset |
|
.globl __memset |
|
.globl ___memset |
|
.globl __memset16 |
|
.globl __constant_c_memset |
|
|
|
.ent ___memset |
|
.align 5 |
|
___memset: |
|
.frame $30,0,$26,0 |
|
.prologue 0 |
|
|
|
/* |
|
* Serious stalling happens. The only way to mitigate this is to |
|
* undertake a major re-write to interleave the constant materialization |
|
* with other parts of the fall-through code. This is important, even |
|
* though it makes maintenance tougher. |
|
* Do this later. |
|
*/ |
|
and $17,255,$1 # E : 00000000000000ch |
|
insbl $17,1,$2 # U : 000000000000ch00 |
|
bis $16,$16,$0 # E : return value |
|
ble $18,end_b # U : zero length requested? |
|
|
|
addq $18,$16,$6 # E : max address to write to |
|
bis $1,$2,$17 # E : 000000000000chch |
|
insbl $1,2,$3 # U : 0000000000ch0000 |
|
insbl $1,3,$4 # U : 00000000ch000000 |
|
|
|
or $3,$4,$3 # E : 00000000chch0000 |
|
inswl $17,4,$5 # U : 0000chch00000000 |
|
xor $16,$6,$1 # E : will complete write be within one quadword? |
|
inswl $17,6,$2 # U : chch000000000000 |
|
|
|
or $17,$3,$17 # E : 00000000chchchch |
|
or $2,$5,$2 # E : chchchch00000000 |
|
bic $1,7,$1 # E : fit within a single quadword? |
|
and $16,7,$3 # E : Target addr misalignment |
|
|
|
or $17,$2,$17 # E : chchchchchchchch |
|
beq $1,within_quad_b # U : |
|
nop # E : |
|
beq $3,aligned_b # U : target is 0mod8 |
|
|
|
/* |
|
* Target address is misaligned, and won't fit within a quadword |
|
*/ |
|
ldq_u $4,0($16) # L : Fetch first partial |
|
bis $16,$16,$5 # E : Save the address |
|
insql $17,$16,$2 # U : Insert new bytes |
|
subq $3,8,$3 # E : Invert (for addressing uses) |
|
|
|
addq $18,$3,$18 # E : $18 is new count ($3 is negative) |
|
mskql $4,$16,$4 # U : clear relevant parts of the quad |
|
subq $16,$3,$16 # E : $16 is new aligned destination |
|
bis $2,$4,$1 # E : Final bytes |
|
|
|
nop |
|
stq_u $1,0($5) # L : Store result |
|
nop |
|
nop |
|
|
|
.align 4 |
|
aligned_b: |
|
/* |
|
* We are now guaranteed to be quad aligned, with at least |
|
* one partial quad to write. |
|
*/ |
|
|
|
sra $18,3,$3 # U : Number of remaining quads to write |
|
and $18,7,$18 # E : Number of trailing bytes to write |
|
bis $16,$16,$5 # E : Save dest address |
|
beq $3,no_quad_b # U : tail stuff only |
|
|
|
/* |
|
* it's worth the effort to unroll this and use wh64 if possible |
|
* Lifted a bunch of code from clear_user.S |
|
* At this point, entry values are: |
|
* $16 Current destination address |
|
* $5 A copy of $16 |
|
* $6 The max quadword address to write to |
|
* $18 Number trailer bytes |
|
* $3 Number quads to write |
|
*/ |
|
|
|
and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) |
|
subq $3, 16, $4 # E : Only try to unroll if > 128 bytes |
|
subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) |
|
blt $4, loop_b # U : |
|
|
|
/* |
|
* We know we've got at least 16 quads, minimum of one trip |
|
* through unrolled loop. Do a quad at a time to get us 0mod64 |
|
* aligned. |
|
*/ |
|
|
|
nop # E : |
|
nop # E : |
|
nop # E : |
|
beq $1, $bigalign_b # U : |
|
|
|
$alignmod64_b: |
|
stq $17, 0($5) # L : |
|
subq $3, 1, $3 # E : For consistency later |
|
addq $1, 8, $1 # E : Increment towards zero for alignment |
|
addq $5, 8, $4 # E : Initial wh64 address (filler instruction) |
|
|
|
nop |
|
nop |
|
addq $5, 8, $5 # E : Inc address |
|
blt $1, $alignmod64_b # U : |
|
|
|
$bigalign_b: |
|
/* |
|
* $3 - number quads left to go |
|
* $5 - target address (aligned 0mod64) |
|
* $17 - mask of stuff to store |
|
* Scratch registers available: $7, $2, $4, $1 |
|
* we know that we'll be taking a minimum of one trip through |
|
* CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle |
|
* Assumes the wh64 needs to be for 2 trips through the loop in the future |
|
* The wh64 is issued on for the starting destination address for trip +2 |
|
* through the loop, and if there are less than two trips left, the target |
|
* address will be for the current trip. |
|
*/ |
|
|
|
$do_wh64_b: |
|
wh64 ($4) # L1 : memory subsystem write hint |
|
subq $3, 24, $2 # E : For determining future wh64 addresses |
|
stq $17, 0($5) # L : |
|
nop # E : |
|
|
|
addq $5, 128, $4 # E : speculative target of next wh64 |
|
stq $17, 8($5) # L : |
|
stq $17, 16($5) # L : |
|
addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) |
|
|
|
stq $17, 24($5) # L : |
|
stq $17, 32($5) # L : |
|
cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle |
|
nop |
|
|
|
stq $17, 40($5) # L : |
|
stq $17, 48($5) # L : |
|
subq $3, 16, $2 # E : Repeat the loop at least once more? |
|
nop |
|
|
|
stq $17, 56($5) # L : |
|
addq $5, 64, $5 # E : |
|
subq $3, 8, $3 # E : |
|
bge $2, $do_wh64_b # U : |
|
|
|
nop |
|
nop |
|
nop |
|
beq $3, no_quad_b # U : Might have finished already |
|
|
|
.align 4 |
|
/* |
|
* Simple loop for trailing quadwords, or for small amounts |
|
* of data (where we can't use an unrolled loop and wh64) |
|
*/ |
|
loop_b: |
|
stq $17,0($5) # L : |
|
subq $3,1,$3 # E : Decrement number quads left |
|
addq $5,8,$5 # E : Inc address |
|
bne $3,loop_b # U : more? |
|
|
|
no_quad_b: |
|
/* |
|
* Write 0..7 trailing bytes. |
|
*/ |
|
nop # E : |
|
beq $18,end_b # U : All done? |
|
ldq $7,0($5) # L : |
|
mskqh $7,$6,$2 # U : Mask final quad |
|
|
|
insqh $17,$6,$4 # U : New bits |
|
bis $2,$4,$1 # E : Put it all together |
|
stq $1,0($5) # L : And back to memory |
|
ret $31,($26),1 # L0 : |
|
|
|
within_quad_b: |
|
ldq_u $1,0($16) # L : |
|
insql $17,$16,$2 # U : New bits |
|
mskql $1,$16,$4 # U : Clear old |
|
bis $2,$4,$2 # E : New result |
|
|
|
mskql $2,$6,$4 # U : |
|
mskqh $1,$6,$2 # U : |
|
bis $2,$4,$1 # E : |
|
stq_u $1,0($16) # L : |
|
|
|
end_b: |
|
nop |
|
nop |
|
nop |
|
ret $31,($26),1 # L0 : |
|
.end ___memset |
|
EXPORT_SYMBOL(___memset) |
|
|
|
/* |
|
* This is the original body of code, prior to replication and |
|
* rescheduling. Leave it here, as there may be calls to this |
|
* entry point. |
|
*/ |
|
.align 4 |
|
.ent __constant_c_memset |
|
__constant_c_memset: |
|
.frame $30,0,$26,0 |
|
.prologue 0 |
|
|
|
addq $18,$16,$6 # E : max address to write to |
|
bis $16,$16,$0 # E : return value |
|
xor $16,$6,$1 # E : will complete write be within one quadword? |
|
ble $18,end # U : zero length requested? |
|
|
|
bic $1,7,$1 # E : fit within a single quadword |
|
beq $1,within_one_quad # U : |
|
and $16,7,$3 # E : Target addr misalignment |
|
beq $3,aligned # U : target is 0mod8 |
|
|
|
/* |
|
* Target address is misaligned, and won't fit within a quadword |
|
*/ |
|
ldq_u $4,0($16) # L : Fetch first partial |
|
bis $16,$16,$5 # E : Save the address |
|
insql $17,$16,$2 # U : Insert new bytes |
|
subq $3,8,$3 # E : Invert (for addressing uses) |
|
|
|
addq $18,$3,$18 # E : $18 is new count ($3 is negative) |
|
mskql $4,$16,$4 # U : clear relevant parts of the quad |
|
subq $16,$3,$16 # E : $16 is new aligned destination |
|
bis $2,$4,$1 # E : Final bytes |
|
|
|
nop |
|
stq_u $1,0($5) # L : Store result |
|
nop |
|
nop |
|
|
|
.align 4 |
|
aligned: |
|
/* |
|
* We are now guaranteed to be quad aligned, with at least |
|
* one partial quad to write. |
|
*/ |
|
|
|
sra $18,3,$3 # U : Number of remaining quads to write |
|
and $18,7,$18 # E : Number of trailing bytes to write |
|
bis $16,$16,$5 # E : Save dest address |
|
beq $3,no_quad # U : tail stuff only |
|
|
|
/* |
|
* it's worth the effort to unroll this and use wh64 if possible |
|
* Lifted a bunch of code from clear_user.S |
|
* At this point, entry values are: |
|
* $16 Current destination address |
|
* $5 A copy of $16 |
|
* $6 The max quadword address to write to |
|
* $18 Number trailer bytes |
|
* $3 Number quads to write |
|
*/ |
|
|
|
and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) |
|
subq $3, 16, $4 # E : Only try to unroll if > 128 bytes |
|
subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) |
|
blt $4, loop # U : |
|
|
|
/* |
|
* We know we've got at least 16 quads, minimum of one trip |
|
* through unrolled loop. Do a quad at a time to get us 0mod64 |
|
* aligned. |
|
*/ |
|
|
|
nop # E : |
|
nop # E : |
|
nop # E : |
|
beq $1, $bigalign # U : |
|
|
|
$alignmod64: |
|
stq $17, 0($5) # L : |
|
subq $3, 1, $3 # E : For consistency later |
|
addq $1, 8, $1 # E : Increment towards zero for alignment |
|
addq $5, 8, $4 # E : Initial wh64 address (filler instruction) |
|
|
|
nop |
|
nop |
|
addq $5, 8, $5 # E : Inc address |
|
blt $1, $alignmod64 # U : |
|
|
|
$bigalign: |
|
/* |
|
* $3 - number quads left to go |
|
* $5 - target address (aligned 0mod64) |
|
* $17 - mask of stuff to store |
|
* Scratch registers available: $7, $2, $4, $1 |
|
* we know that we'll be taking a minimum of one trip through |
|
* CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle |
|
* Assumes the wh64 needs to be for 2 trips through the loop in the future |
|
* The wh64 is issued on for the starting destination address for trip +2 |
|
* through the loop, and if there are less than two trips left, the target |
|
* address will be for the current trip. |
|
*/ |
|
|
|
$do_wh64: |
|
wh64 ($4) # L1 : memory subsystem write hint |
|
subq $3, 24, $2 # E : For determining future wh64 addresses |
|
stq $17, 0($5) # L : |
|
nop # E : |
|
|
|
addq $5, 128, $4 # E : speculative target of next wh64 |
|
stq $17, 8($5) # L : |
|
stq $17, 16($5) # L : |
|
addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) |
|
|
|
stq $17, 24($5) # L : |
|
stq $17, 32($5) # L : |
|
cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle |
|
nop |
|
|
|
stq $17, 40($5) # L : |
|
stq $17, 48($5) # L : |
|
subq $3, 16, $2 # E : Repeat the loop at least once more? |
|
nop |
|
|
|
stq $17, 56($5) # L : |
|
addq $5, 64, $5 # E : |
|
subq $3, 8, $3 # E : |
|
bge $2, $do_wh64 # U : |
|
|
|
nop |
|
nop |
|
nop |
|
beq $3, no_quad # U : Might have finished already |
|
|
|
.align 4 |
|
/* |
|
* Simple loop for trailing quadwords, or for small amounts |
|
* of data (where we can't use an unrolled loop and wh64) |
|
*/ |
|
loop: |
|
stq $17,0($5) # L : |
|
subq $3,1,$3 # E : Decrement number quads left |
|
addq $5,8,$5 # E : Inc address |
|
bne $3,loop # U : more? |
|
|
|
no_quad: |
|
/* |
|
* Write 0..7 trailing bytes. |
|
*/ |
|
nop # E : |
|
beq $18,end # U : All done? |
|
ldq $7,0($5) # L : |
|
mskqh $7,$6,$2 # U : Mask final quad |
|
|
|
insqh $17,$6,$4 # U : New bits |
|
bis $2,$4,$1 # E : Put it all together |
|
stq $1,0($5) # L : And back to memory |
|
ret $31,($26),1 # L0 : |
|
|
|
within_one_quad: |
|
ldq_u $1,0($16) # L : |
|
insql $17,$16,$2 # U : New bits |
|
mskql $1,$16,$4 # U : Clear old |
|
bis $2,$4,$2 # E : New result |
|
|
|
mskql $2,$6,$4 # U : |
|
mskqh $1,$6,$2 # U : |
|
bis $2,$4,$1 # E : |
|
stq_u $1,0($16) # L : |
|
|
|
end: |
|
nop |
|
nop |
|
nop |
|
ret $31,($26),1 # L0 : |
|
.end __constant_c_memset |
|
EXPORT_SYMBOL(__constant_c_memset) |
|
|
|
/* |
|
* This is a replicant of the __constant_c_memset code, rescheduled |
|
* to mask stalls. Note that entry point names also had to change |
|
*/ |
|
.align 5 |
|
.ent __memset16 |
|
|
|
__memset16: |
|
.frame $30,0,$26,0 |
|
.prologue 0 |
|
|
|
inswl $17,0,$5 # U : 000000000000c1c2 |
|
inswl $17,2,$2 # U : 00000000c1c20000 |
|
bis $16,$16,$0 # E : return value |
|
addq $18,$16,$6 # E : max address to write to |
|
|
|
ble $18, end_w # U : zero length requested? |
|
inswl $17,4,$3 # U : 0000c1c200000000 |
|
inswl $17,6,$4 # U : c1c2000000000000 |
|
xor $16,$6,$1 # E : will complete write be within one quadword? |
|
|
|
or $2,$5,$2 # E : 00000000c1c2c1c2 |
|
or $3,$4,$17 # E : c1c2c1c200000000 |
|
bic $1,7,$1 # E : fit within a single quadword |
|
and $16,7,$3 # E : Target addr misalignment |
|
|
|
or $17,$2,$17 # E : c1c2c1c2c1c2c1c2 |
|
beq $1,within_quad_w # U : |
|
nop |
|
beq $3,aligned_w # U : target is 0mod8 |
|
|
|
/* |
|
* Target address is misaligned, and won't fit within a quadword |
|
*/ |
|
ldq_u $4,0($16) # L : Fetch first partial |
|
bis $16,$16,$5 # E : Save the address |
|
insql $17,$16,$2 # U : Insert new bytes |
|
subq $3,8,$3 # E : Invert (for addressing uses) |
|
|
|
addq $18,$3,$18 # E : $18 is new count ($3 is negative) |
|
mskql $4,$16,$4 # U : clear relevant parts of the quad |
|
subq $16,$3,$16 # E : $16 is new aligned destination |
|
bis $2,$4,$1 # E : Final bytes |
|
|
|
nop |
|
stq_u $1,0($5) # L : Store result |
|
nop |
|
nop |
|
|
|
.align 4 |
|
aligned_w: |
|
/* |
|
* We are now guaranteed to be quad aligned, with at least |
|
* one partial quad to write. |
|
*/ |
|
|
|
sra $18,3,$3 # U : Number of remaining quads to write |
|
and $18,7,$18 # E : Number of trailing bytes to write |
|
bis $16,$16,$5 # E : Save dest address |
|
beq $3,no_quad_w # U : tail stuff only |
|
|
|
/* |
|
* it's worth the effort to unroll this and use wh64 if possible |
|
* Lifted a bunch of code from clear_user.S |
|
* At this point, entry values are: |
|
* $16 Current destination address |
|
* $5 A copy of $16 |
|
* $6 The max quadword address to write to |
|
* $18 Number trailer bytes |
|
* $3 Number quads to write |
|
*/ |
|
|
|
and $16, 0x3f, $2 # E : Forward work (only useful for unrolled loop) |
|
subq $3, 16, $4 # E : Only try to unroll if > 128 bytes |
|
subq $2, 0x40, $1 # E : bias counter (aligning stuff 0mod64) |
|
blt $4, loop_w # U : |
|
|
|
/* |
|
* We know we've got at least 16 quads, minimum of one trip |
|
* through unrolled loop. Do a quad at a time to get us 0mod64 |
|
* aligned. |
|
*/ |
|
|
|
nop # E : |
|
nop # E : |
|
nop # E : |
|
beq $1, $bigalign_w # U : |
|
|
|
$alignmod64_w: |
|
stq $17, 0($5) # L : |
|
subq $3, 1, $3 # E : For consistency later |
|
addq $1, 8, $1 # E : Increment towards zero for alignment |
|
addq $5, 8, $4 # E : Initial wh64 address (filler instruction) |
|
|
|
nop |
|
nop |
|
addq $5, 8, $5 # E : Inc address |
|
blt $1, $alignmod64_w # U : |
|
|
|
$bigalign_w: |
|
/* |
|
* $3 - number quads left to go |
|
* $5 - target address (aligned 0mod64) |
|
* $17 - mask of stuff to store |
|
* Scratch registers available: $7, $2, $4, $1 |
|
* we know that we'll be taking a minimum of one trip through |
|
* CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle |
|
* Assumes the wh64 needs to be for 2 trips through the loop in the future |
|
* The wh64 is issued on for the starting destination address for trip +2 |
|
* through the loop, and if there are less than two trips left, the target |
|
* address will be for the current trip. |
|
*/ |
|
|
|
$do_wh64_w: |
|
wh64 ($4) # L1 : memory subsystem write hint |
|
subq $3, 24, $2 # E : For determining future wh64 addresses |
|
stq $17, 0($5) # L : |
|
nop # E : |
|
|
|
addq $5, 128, $4 # E : speculative target of next wh64 |
|
stq $17, 8($5) # L : |
|
stq $17, 16($5) # L : |
|
addq $5, 64, $7 # E : Fallback address for wh64 (== next trip addr) |
|
|
|
stq $17, 24($5) # L : |
|
stq $17, 32($5) # L : |
|
cmovlt $2, $7, $4 # E : Latency 2, extra mapping cycle |
|
nop |
|
|
|
stq $17, 40($5) # L : |
|
stq $17, 48($5) # L : |
|
subq $3, 16, $2 # E : Repeat the loop at least once more? |
|
nop |
|
|
|
stq $17, 56($5) # L : |
|
addq $5, 64, $5 # E : |
|
subq $3, 8, $3 # E : |
|
bge $2, $do_wh64_w # U : |
|
|
|
nop |
|
nop |
|
nop |
|
beq $3, no_quad_w # U : Might have finished already |
|
|
|
.align 4 |
|
/* |
|
* Simple loop for trailing quadwords, or for small amounts |
|
* of data (where we can't use an unrolled loop and wh64) |
|
*/ |
|
loop_w: |
|
stq $17,0($5) # L : |
|
subq $3,1,$3 # E : Decrement number quads left |
|
addq $5,8,$5 # E : Inc address |
|
bne $3,loop_w # U : more? |
|
|
|
no_quad_w: |
|
/* |
|
* Write 0..7 trailing bytes. |
|
*/ |
|
nop # E : |
|
beq $18,end_w # U : All done? |
|
ldq $7,0($5) # L : |
|
mskqh $7,$6,$2 # U : Mask final quad |
|
|
|
insqh $17,$6,$4 # U : New bits |
|
bis $2,$4,$1 # E : Put it all together |
|
stq $1,0($5) # L : And back to memory |
|
ret $31,($26),1 # L0 : |
|
|
|
within_quad_w: |
|
ldq_u $1,0($16) # L : |
|
insql $17,$16,$2 # U : New bits |
|
mskql $1,$16,$4 # U : Clear old |
|
bis $2,$4,$2 # E : New result |
|
|
|
mskql $2,$6,$4 # U : |
|
mskqh $1,$6,$2 # U : |
|
bis $2,$4,$1 # E : |
|
stq_u $1,0($16) # L : |
|
|
|
end_w: |
|
nop |
|
nop |
|
nop |
|
ret $31,($26),1 # L0 : |
|
|
|
.end __memset16 |
|
EXPORT_SYMBOL(__memset16) |
|
|
|
memset = ___memset |
|
__memset = ___memset |
|
EXPORT_SYMBOL(memset) |
|
EXPORT_SYMBOL(__memset)
|
|
|