118 lines · plain
1/* SPDX-License-Identifier: GPL-2.0 */2/* Copyright 2002 Andi Kleen, SuSE Labs */3 4#include <linux/export.h>5#include <linux/linkage.h>6#include <asm/cpufeatures.h>7#include <asm/alternative.h>8 9.section .noinstr.text, "ax"10 11/*12 * ISO C memset - set a memory block to a byte value. This function uses fast13 * string to get better performance than the original function. The code is14 * simpler and shorter than the original function as well.15 *16 * rdi destination17 * rsi value (char)18 * rdx count (bytes)19 *20 * rax original destination21 *22 * The FSRS alternative should be done inline (avoiding the call and23 * the disgusting return handling), but that would require some help24 * from the compiler for better calling conventions.25 *26 * The 'rep stosb' itself is small enough to replace the call, but all27 * the register moves blow up the code. And two of them are "needed"28 * only for the return value that is the same as the source input,29 * which the compiler could/should do much better anyway.30 */31SYM_FUNC_START(__memset)32 ALTERNATIVE "jmp memset_orig", "", X86_FEATURE_FSRS33 34 movq %rdi,%r935 movb %sil,%al36 movq %rdx,%rcx37 rep stosb38 movq %r9,%rax39 RET40SYM_FUNC_END(__memset)41EXPORT_SYMBOL(__memset)42 43SYM_FUNC_ALIAS_MEMFUNC(memset, __memset)44EXPORT_SYMBOL(memset)45 46SYM_FUNC_START_LOCAL(memset_orig)47 movq %rdi,%r1048 49 /* expand byte value */50 movzbl %sil,%ecx51 movabs $0x0101010101010101,%rax52 imulq %rcx,%rax53 54 /* align dst */55 movl %edi,%r9d56 andl $7,%r9d57 jnz .Lbad_alignment58.Lafter_bad_alignment:59 60 movq %rdx,%rcx61 shrq $6,%rcx62 jz .Lhandle_tail63 64 .p2align 465.Lloop_64:66 decq %rcx67 movq %rax,(%rdi)68 movq %rax,8(%rdi)69 movq %rax,16(%rdi)70 movq %rax,24(%rdi)71 movq %rax,32(%rdi)72 movq %rax,40(%rdi)73 movq %rax,48(%rdi)74 movq %rax,56(%rdi)75 leaq 64(%rdi),%rdi76 jnz .Lloop_6477 78 /* Handle tail in loops. The loops should be faster than hard79 to predict jump tables. */80 .p2align 481.Lhandle_tail:82 movl %edx,%ecx83 andl $63&(~7),%ecx84 jz .Lhandle_785 shrl $3,%ecx86 .p2align 487.Lloop_8:88 decl %ecx89 movq %rax,(%rdi)90 leaq 8(%rdi),%rdi91 jnz .Lloop_892 93.Lhandle_7:94 andl $7,%edx95 jz .Lende96 .p2align 497.Lloop_1:98 decl %edx99 movb %al,(%rdi)100 leaq 1(%rdi),%rdi101 jnz .Lloop_1102 103.Lende:104 movq %r10,%rax105 RET106 107.Lbad_alignment:108 cmpq $7,%rdx109 jbe .Lhandle_7110 movq %rax,(%rdi) /* unaligned store */111 movq $8,%r8112 subq %r9,%r8113 addq %r8,%rdi114 subq %r8,%rdx115 jmp .Lafter_bad_alignment116.Lfinal:117SYM_FUNC_END(memset_orig)118