mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
230 lines
4.2 KiB
230 lines
4.2 KiB
/* SPDX-License-Identifier: GPL-2.0-or-later */ |
|
/* |
|
* Copyright (C) 2002 Paul Mackerras, IBM Corp. |
|
*/ |
|
#include <asm/processor.h> |
|
#include <asm/ppc_asm.h> |
|
#include <asm/export.h> |
|
#include <asm/asm-compat.h> |
|
#include <asm/feature-fixups.h> |
|
#include <asm/kasan.h> |
|
|
|
#ifndef SELFTEST_CASE |
|
/* For big-endian, 0 == most CPUs, 1 == POWER6, 2 == Cell */ |
|
#define SELFTEST_CASE 0 |
|
#endif |
|
|
|
.align 7 |
|
_GLOBAL_TOC_KASAN(memcpy) |
|
BEGIN_FTR_SECTION |
|
#ifdef __LITTLE_ENDIAN__ |
|
cmpdi cr7,r5,0 |
|
#else |
|
std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* save destination pointer for return value */ |
|
#endif |
|
FTR_SECTION_ELSE |
|
#ifdef CONFIG_PPC_BOOK3S_64 |
|
b memcpy_power7 |
|
#endif |
|
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) |
|
#ifdef __LITTLE_ENDIAN__ |
|
/* dumb little-endian memcpy that will get replaced at runtime */ |
|
addi r9,r3,-1 |
|
addi r4,r4,-1 |
|
beqlr cr7 |
|
mtctr r5 |
|
1: lbzu r10,1(r4) |
|
stbu r10,1(r9) |
|
bdnz 1b |
|
blr |
|
#else |
|
PPC_MTOCRF(0x01,r5) |
|
cmpldi cr1,r5,16 |
|
neg r6,r3 # LS 3 bits = # bytes to 8-byte dest bdry |
|
andi. r6,r6,7 |
|
dcbt 0,r4 |
|
blt cr1,.Lshort_copy |
|
/* Below we want to nop out the bne if we're on a CPU that has the |
|
CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit |
|
cleared. |
|
At the time of writing the only CPU that has this combination of bits |
|
set is Power6. */ |
|
test_feature = (SELFTEST_CASE == 1) |
|
BEGIN_FTR_SECTION |
|
nop |
|
FTR_SECTION_ELSE |
|
bne .Ldst_unaligned |
|
ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \ |
|
CPU_FTR_UNALIGNED_LD_STD) |
|
.Ldst_aligned: |
|
addi r3,r3,-16 |
|
test_feature = (SELFTEST_CASE == 0) |
|
BEGIN_FTR_SECTION |
|
andi. r0,r4,7 |
|
bne .Lsrc_unaligned |
|
END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) |
|
srdi r7,r5,4 |
|
ld r9,0(r4) |
|
addi r4,r4,-8 |
|
mtctr r7 |
|
andi. r5,r5,7 |
|
bf cr7*4+0,2f |
|
addi r3,r3,8 |
|
addi r4,r4,8 |
|
mr r8,r9 |
|
blt cr1,3f |
|
1: ld r9,8(r4) |
|
std r8,8(r3) |
|
2: ldu r8,16(r4) |
|
stdu r9,16(r3) |
|
bdnz 1b |
|
3: std r8,8(r3) |
|
beq 3f |
|
addi r3,r3,16 |
|
.Ldo_tail: |
|
bf cr7*4+1,1f |
|
lwz r9,8(r4) |
|
addi r4,r4,4 |
|
stw r9,0(r3) |
|
addi r3,r3,4 |
|
1: bf cr7*4+2,2f |
|
lhz r9,8(r4) |
|
addi r4,r4,2 |
|
sth r9,0(r3) |
|
addi r3,r3,2 |
|
2: bf cr7*4+3,3f |
|
lbz r9,8(r4) |
|
stb r9,0(r3) |
|
3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
|
blr |
|
|
|
.Lsrc_unaligned: |
|
srdi r6,r5,3 |
|
addi r5,r5,-16 |
|
subf r4,r0,r4 |
|
srdi r7,r5,4 |
|
sldi r10,r0,3 |
|
cmpdi cr6,r6,3 |
|
andi. r5,r5,7 |
|
mtctr r7 |
|
subfic r11,r10,64 |
|
add r5,r5,r0 |
|
|
|
bt cr7*4+0,0f |
|
|
|
ld r9,0(r4) # 3+2n loads, 2+2n stores |
|
ld r0,8(r4) |
|
sld r6,r9,r10 |
|
ldu r9,16(r4) |
|
srd r7,r0,r11 |
|
sld r8,r0,r10 |
|
or r7,r7,r6 |
|
blt cr6,4f |
|
ld r0,8(r4) |
|
# s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12 |
|
b 2f |
|
|
|
0: ld r0,0(r4) # 4+2n loads, 3+2n stores |
|
ldu r9,8(r4) |
|
sld r8,r0,r10 |
|
addi r3,r3,-8 |
|
blt cr6,5f |
|
ld r0,8(r4) |
|
srd r12,r9,r11 |
|
sld r6,r9,r10 |
|
ldu r9,16(r4) |
|
or r12,r8,r12 |
|
srd r7,r0,r11 |
|
sld r8,r0,r10 |
|
addi r3,r3,16 |
|
beq cr6,3f |
|
|
|
# d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9 |
|
1: or r7,r7,r6 |
|
ld r0,8(r4) |
|
std r12,8(r3) |
|
2: srd r12,r9,r11 |
|
sld r6,r9,r10 |
|
ldu r9,16(r4) |
|
or r12,r8,r12 |
|
stdu r7,16(r3) |
|
srd r7,r0,r11 |
|
sld r8,r0,r10 |
|
bdnz 1b |
|
|
|
3: std r12,8(r3) |
|
or r7,r7,r6 |
|
4: std r7,16(r3) |
|
5: srd r12,r9,r11 |
|
or r12,r8,r12 |
|
std r12,24(r3) |
|
beq 4f |
|
cmpwi cr1,r5,8 |
|
addi r3,r3,32 |
|
sld r9,r9,r10 |
|
ble cr1,6f |
|
ld r0,8(r4) |
|
srd r7,r0,r11 |
|
or r9,r7,r9 |
|
6: |
|
bf cr7*4+1,1f |
|
rotldi r9,r9,32 |
|
stw r9,0(r3) |
|
addi r3,r3,4 |
|
1: bf cr7*4+2,2f |
|
rotldi r9,r9,16 |
|
sth r9,0(r3) |
|
addi r3,r3,2 |
|
2: bf cr7*4+3,3f |
|
rotldi r9,r9,8 |
|
stb r9,0(r3) |
|
3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
|
blr |
|
|
|
.Ldst_unaligned: |
|
PPC_MTOCRF(0x01,r6) # put #bytes to 8B bdry into cr7 |
|
subf r5,r6,r5 |
|
li r7,0 |
|
cmpldi cr1,r5,16 |
|
bf cr7*4+3,1f |
|
lbz r0,0(r4) |
|
stb r0,0(r3) |
|
addi r7,r7,1 |
|
1: bf cr7*4+2,2f |
|
lhzx r0,r7,r4 |
|
sthx r0,r7,r3 |
|
addi r7,r7,2 |
|
2: bf cr7*4+1,3f |
|
lwzx r0,r7,r4 |
|
stwx r0,r7,r3 |
|
3: PPC_MTOCRF(0x01,r5) |
|
add r4,r6,r4 |
|
add r3,r6,r3 |
|
b .Ldst_aligned |
|
|
|
.Lshort_copy: |
|
bf cr7*4+0,1f |
|
lwz r0,0(r4) |
|
lwz r9,4(r4) |
|
addi r4,r4,8 |
|
stw r0,0(r3) |
|
stw r9,4(r3) |
|
addi r3,r3,8 |
|
1: bf cr7*4+1,2f |
|
lwz r0,0(r4) |
|
addi r4,r4,4 |
|
stw r0,0(r3) |
|
addi r3,r3,4 |
|
2: bf cr7*4+2,3f |
|
lhz r0,0(r4) |
|
addi r4,r4,2 |
|
sth r0,0(r3) |
|
addi r3,r3,2 |
|
3: bf cr7*4+3,4f |
|
lbz r0,0(r4) |
|
stb r0,0(r3) |
|
4: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
|
blr |
|
#endif |
|
EXPORT_SYMBOL(memcpy) |
|
EXPORT_SYMBOL_KASAN(memcpy)
|
|
|