172 lines · plain
1/* SPDX-License-Identifier: GPL-2.0-only */2/* Copyright 2002 Andi Kleen */3 4#include <linux/export.h>5#include <linux/linkage.h>6#include <asm/errno.h>7#include <asm/cpufeatures.h>8#include <asm/alternative.h>9 10.section .noinstr.text, "ax"11 12/*13 * memcpy - Copy a memory block.14 *15 * Input:16 * rdi destination17 * rsi source18 * rdx count19 *20 * Output:21 * rax original destination22 *23 * The FSRM alternative should be done inline (avoiding the call and24 * the disgusting return handling), but that would require some help25 * from the compiler for better calling conventions.26 *27 * The 'rep movsb' itself is small enough to replace the call, but the28 * two register moves blow up the code. And one of them is "needed"29 * only for the return value that is the same as the source input,30 * which the compiler could/should do much better anyway.31 */32SYM_TYPED_FUNC_START(__memcpy)33 ALTERNATIVE "jmp memcpy_orig", "", X86_FEATURE_FSRM34 35 movq %rdi, %rax36 movq %rdx, %rcx37 rep movsb38 RET39SYM_FUNC_END(__memcpy)40EXPORT_SYMBOL(__memcpy)41 42SYM_FUNC_ALIAS_MEMFUNC(memcpy, __memcpy)43EXPORT_SYMBOL(memcpy)44 45SYM_FUNC_START_LOCAL(memcpy_orig)46 movq %rdi, %rax47 48 cmpq $0x20, %rdx49 jb .Lhandle_tail50 51 /*52 * We check whether memory false dependence could occur,53 * then jump to corresponding copy mode.54 */55 cmp %dil, %sil56 jl .Lcopy_backward57 subq $0x20, %rdx58.Lcopy_forward_loop:59 subq $0x20, %rdx60 61 /*62 * Move in blocks of 4x8 bytes:63 */64 movq 0*8(%rsi), %r865 movq 1*8(%rsi), %r966 movq 2*8(%rsi), %r1067 movq 3*8(%rsi), %r1168 leaq 4*8(%rsi), %rsi69 70 movq %r8, 0*8(%rdi)71 movq %r9, 1*8(%rdi)72 movq %r10, 2*8(%rdi)73 movq %r11, 3*8(%rdi)74 leaq 4*8(%rdi), %rdi75 jae .Lcopy_forward_loop76 addl $0x20, %edx77 jmp .Lhandle_tail78 79.Lcopy_backward:80 /*81 * Calculate copy position to tail.82 */83 addq %rdx, %rsi84 addq %rdx, %rdi85 subq $0x20, %rdx86 /*87 * At most 3 ALU operations in one cycle,88 * so append NOPS in the same 16 bytes trunk.89 */90 .p2align 491.Lcopy_backward_loop:92 subq $0x20, %rdx93 movq -1*8(%rsi), %r894 movq -2*8(%rsi), %r995 movq -3*8(%rsi), %r1096 movq -4*8(%rsi), %r1197 leaq -4*8(%rsi), %rsi98 movq %r8, -1*8(%rdi)99 movq %r9, -2*8(%rdi)100 movq %r10, -3*8(%rdi)101 movq %r11, -4*8(%rdi)102 leaq -4*8(%rdi), %rdi103 jae .Lcopy_backward_loop104 105 /*106 * Calculate copy position to head.107 */108 addl $0x20, %edx109 subq %rdx, %rsi110 subq %rdx, %rdi111.Lhandle_tail:112 cmpl $16, %edx113 jb .Lless_16bytes114 115 /*116 * Move data from 16 bytes to 31 bytes.117 */118 movq 0*8(%rsi), %r8119 movq 1*8(%rsi), %r9120 movq -2*8(%rsi, %rdx), %r10121 movq -1*8(%rsi, %rdx), %r11122 movq %r8, 0*8(%rdi)123 movq %r9, 1*8(%rdi)124 movq %r10, -2*8(%rdi, %rdx)125 movq %r11, -1*8(%rdi, %rdx)126 RET127 .p2align 4128.Lless_16bytes:129 cmpl $8, %edx130 jb .Lless_8bytes131 /*132 * Move data from 8 bytes to 15 bytes.133 */134 movq 0*8(%rsi), %r8135 movq -1*8(%rsi, %rdx), %r9136 movq %r8, 0*8(%rdi)137 movq %r9, -1*8(%rdi, %rdx)138 RET139 .p2align 4140.Lless_8bytes:141 cmpl $4, %edx142 jb .Lless_3bytes143 144 /*145 * Move data from 4 bytes to 7 bytes.146 */147 movl (%rsi), %ecx148 movl -4(%rsi, %rdx), %r8d149 movl %ecx, (%rdi)150 movl %r8d, -4(%rdi, %rdx)151 RET152 .p2align 4153.Lless_3bytes:154 subl $1, %edx155 jb .Lend156 /*157 * Move data from 1 bytes to 3 bytes.158 */159 movzbl (%rsi), %ecx160 jz .Lstore_1byte161 movzbq 1(%rsi), %r8162 movzbq (%rsi, %rdx), %r9163 movb %r8b, 1(%rdi)164 movb %r9b, (%rdi, %rdx)165.Lstore_1byte:166 movb %cl, (%rdi)167 168.Lend:169 RET170SYM_FUNC_END(memcpy_orig)171 172