diff --git a/libs/libc/machine/arm/armv8-m/Kconfig b/libs/libc/machine/arm/armv8-m/Kconfig index 825ee73d90d..ff06945ab1b 100644 --- a/libs/libc/machine/arm/armv8-m/Kconfig +++ b/libs/libc/machine/arm/armv8-m/Kconfig @@ -5,6 +5,60 @@ if ARCH_ARMV8M +config ARMV8M_MEMCHR + bool "Enable optimized memchr() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_MEMCHR + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific memchr() library function + +config ARMV8M_MEMCPY + bool "Enable optimized memcpy() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_MEMCPY + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific memcpy() library function + +config ARMV8M_MEMSET + bool "Enable optimized memset() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_MEMSET + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific memset() library function + +config ARMV8M_MEMMOVE + bool "Enable optimized memmove() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_MEMMOVE + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific memmove() library function + +config ARMV8M_STRCMP + bool "Enable optimized strcmp() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_STRCMP + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific strcmp() library function + +config ARMV8M_STRLEN + bool "Enable optimized strlen() for ARMv8-M" + default n + select MACHINE_OPTS_ARMV8M + select LIBC_ARCH_STRLEN + depends on ARCH_TOOLCHAIN_GNU + ---help--- + Enable optimized ARMv8-M specific strlen() library function + config ARMV8M_LIBM bool "Architecture specific optimizations" default n @@ -24,4 +78,8 @@ config ARMV8M_LIBM ---help--- Enable ARMv8 specific floating point optimizations. +config MACHINE_OPTS_ARMV8M + bool + default n + endif diff --git a/libs/libc/machine/arm/armv8-m/Make.defs b/libs/libc/machine/arm/armv8-m/Make.defs index f4cbaf64d19..9effc95f391 100644 --- a/libs/libc/machine/arm/armv8-m/Make.defs +++ b/libs/libc/machine/arm/armv8-m/Make.defs @@ -37,6 +37,35 @@ ifeq ($(CONFIG_LIBC_ARCH_ELF),y) CSRCS += arch_elf.c endif +ifeq ($(CONFIG_ARMV8M_MEMCHR),y) +ASRCS += arch_memchr.S +endif + +ifeq ($(CONFIG_ARMV8M_MEMCPY),y) +ASRCS += arch_memcpy.S +endif + +ifeq ($(CONFIG_ARMV8M_MEMSET),y) +ASRCS += arch_memset.S +endif + +ifeq ($(CONFIG_ARMV8M_MEMMOVE),y) +ASRCS += arch_memmove.S +endif + +ifeq ($(CONFIG_ARMV8M_STRCMP),y) +ASRCS += arch_strcmp.S +endif + +ifeq ($(CONFIG_ARMV8M_STRLEN),y) +ASRCS += arch_strlen.S +endif + +ifeq ($(CONFIG_MACHINE_OPTS_ARMV8M),y) +DEPPATH += --dep-path machine/arm/armv8-m/gnu +VPATH += :machine/arm/armv8-m/gnu +endif + ifeq ($(CONFIG_ARMV8M_LIBM),y) ifeq ($(LIBM_ARCH_CEIL),y) diff --git a/libs/libc/machine/arm/armv8-m/gnu/acle-compat.h b/libs/libc/machine/arm/armv8-m/gnu/acle-compat.h new file mode 100644 index 00000000000..beb6df9dd3d --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/acle-compat.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __ARM_ARCH + +/* ACLE standardises a set of pre-defines that describe the ARM architecture. + These were mostly implemented in GCC around GCC-4.8; older versions + have no, or only partial support. To provide a level of backwards + compatibility we try to work out what the definitions should be, given + the older pre-defines that GCC did produce. This isn't complete, but + it should be enough for use by routines that depend on this header. */ + +/* No need to handle ARMv8, GCC had ACLE support before that. */ + +# ifdef __ARM_ARCH_7__ +/* The common subset of ARMv7 in all profiles. */ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# endif + +# if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7R__) +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 15 +# define __ARM_FEATURE_UNALIGNED +# ifdef __ARM_ARCH_7A__ +# define __ARM_ARCH_PROFILE 'A' +# else +# define __ARM_ARCH_PROFILE 'R' +# endif +# endif + +# ifdef __ARM_ARCH_7EM__ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# define __ARM_ARCH_PROFILE 'M' +# endif + +# ifdef __ARM_ARCH_7M__ +# define __ARM_ARCH 7 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 7 +# define __ARM_FEATURE_UNALIGNED +# define __ARM_ARCH_PROFILE 'M' +# endif + +# ifdef __ARM_ARCH_6T2__ +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 2 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_LDREX 4 +# define __ARM_FEATURE_UNALIGNED +# endif + +# ifdef __ARM_ARCH_6M__ +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 1 +# define __ARM_ARCH_PROFILE 'M' +# endif + +# if defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) \ + || defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6Z__) \ + || defined (__ARM_ARCH_6ZK__) +# define __ARM_ARCH 6 +# define __ARM_ARCH_ISA_THUMB 1 +# define __ARM_ARCH_ISA_ARM +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_SIMD32 +# define __ARM_FEATURE_DSP +# define __ARM_FEATURE_QBIT +# define __ARM_FEATURE_SAT +# define __ARM_FEATURE_UNALIGNED +# ifndef __thumb__ +# if defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6ZK__) +# define __ARM_FEATURE_LDREX 15 +# else +# define __ARM_FEATURE_LDREX 4 +# endif +# endif +# endif + +# if defined (__ARM_ARCH_5TE__) || defined (__ARM_ARCH_5E__) +# define __ARM_ARCH 5 +# define __ARM_ARCH_ISA_ARM +# ifdef __ARM_ARCH_5TE__ +# define __ARM_ARCH_ISA_THUMB 1 +# endif +# define __ARM_FEATURE_CLZ +# define __ARM_FEATURE_DSP +# endif + +# if defined (__ARM_ARCH_5T__) || defined (__ARM_ARCH_5__) +# define __ARM_ARCH 5 +# define __ARM_ARCH_ISA_ARM +# ifdef __ARM_ARCH_5TE__ +# define __ARM_ARCH_ISA_THUMB 1 +# endif +# define __ARM_FEATURE_CLZ +# endif + +# ifdef __ARM_ARCH_4T__ +# define __ARM_ARCH 4 +# define __ARM_ARCH_ISA_ARM +# define __ARM_ARCH_ISA_THUMB 1 +# endif + +# ifdef __ARM_ARCH_4__ +# define __ARM_ARCH 4 +# define __ARM_ARCH_ISA_ARM +# endif + +# if defined (__ARM_ARCH_3__) || defined (__ARM_ARCH_3M__) +# define __ARM_ARCH 3 +# define __ARM_ARCH_ISA_ARM +# endif + +# ifdef __ARM_ARCH_2__ +# define __ARM_ARCH 2 +# define __ARM_ARCH_ISA_ARM +# endif + +# ifdef __ARMEB__ +# define __ARM_BIG_ENDIAN +# endif + +/* If we still don't know what the target architecture is, then we're + probably not using GCC. */ +# ifndef __ARM_ARCH +# error Unable to determine architecture version. +# endif + +#endif /* __ARM_ARCH */ \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_memchr.S b/libs/libc/machine/arm/armv8-m/gnu/arch_memchr.S new file mode 100644 index 00000000000..3c2b7e53a2d --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_memchr.S @@ -0,0 +1,386 @@ +/* Copyright (c) 2010-2011, Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Written by Dave Gilbert + + This memchr routine is optimised on a Cortex-A9 and should work on + all ARMv7 processors. It has a fast path for short sizes, and has + an optimised path for large data sets; the worst case is finding the + match early in a large data set. */ + +/* Copyright (c) 2015 ARM Ltd. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Linaro nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +@ 2011-02-07 david.gilbert@linaro.org +@ Extracted from local git a5b438d861 +@ 2011-07-14 david.gilbert@linaro.org +@ Import endianness fix from local git ea786f1b +@ 2011-10-11 david.gilbert@linaro.org +@ Import from cortex-strings bzr rev 63 +@ Flip to ldrd (as suggested by Greta Yorsh) +@ Make conditional on CPU type +@ tidy + +@ This code requires armv6t2 or later. Uses Thumb2. + + .syntax unified + +#include "acle-compat.h" + +@ NOTE: This ifdef MUST match the one in memchr-stub.c +#if defined (__ARM_NEON__) || defined (__ARM_NEON) +#if __ARM_ARCH >= 8 && __ARM_ARCH_PROFILE == 'R' + .arch armv8-r +#else + .arch armv7-a +#endif + .fpu neon + + +/* Arguments */ +#define srcin r0 +#define chrin r1 +#define cntin r2 + +/* Retval */ +#define result r0 /* Live range does not overlap with srcin */ + +/* Working registers */ +#define src r1 /* Live range does not overlap with chrin */ +#define tmp r3 +#define synd r0 /* No overlap with srcin or result */ +#define soff r12 + +/* Working NEON registers */ +#define vrepchr q0 +#define vdata0 q1 +#define vdata0_0 d2 /* Lower half of vdata0 */ +#define vdata0_1 d3 /* Upper half of vdata0 */ +#define vdata1 q2 +#define vdata1_0 d4 /* Lower half of vhas_chr0 */ +#define vdata1_1 d5 /* Upper half of vhas_chr0 */ +#define vrepmask q3 +#define vrepmask0 d6 +#define vrepmask1 d7 +#define vend q4 +#define vend0 d8 +#define vend1 d9 + +/* + * Core algorithm: + * + * For each 32-byte chunk we calculate a 32-bit syndrome value, with one bit per + * byte. Each bit is set if the relevant byte matched the requested character + * and cleared otherwise. Since the bits in the syndrome reflect exactly the + * order in which things occur in the original string, counting trailing zeros + * allows to identify exactly which byte has matched. + */ + + .text + .thumb_func + .align 4 + .p2align 4,,15 + .global memchr + .type memchr,%function + +memchr: + .cfi_sections .debug_frame + .cfi_startproc + /* Use a simple loop if there are less than 8 bytes to search. */ + cmp cntin, #7 + bhi .Llargestr + and chrin, chrin, #0xff + +.Lsmallstr: + subs cntin, cntin, #1 + blo .Lnotfound /* Return not found if reached end. */ + ldrb tmp, [srcin], #1 + cmp tmp, chrin + bne .Lsmallstr /* Loop again if not found. */ + /* Otherwise fixup address and return. */ + sub result, result, #1 + bx lr + + +.Llargestr: + vdup.8 vrepchr, chrin /* Duplicate char across all lanes. */ + /* + * Magic constant 0x8040201008040201 allows us to identify which lane + * matches the requested byte. + */ + movw tmp, #0x0201 + movt tmp, #0x0804 + lsl soff, tmp, #4 + vmov vrepmask0, tmp, soff + vmov vrepmask1, tmp, soff + /* Work with aligned 32-byte chunks */ + bic src, srcin, #31 + ands soff, srcin, #31 + beq .Lloopintro /* Go straight to main loop if it's aligned. */ + + /* + * Input string is not 32-byte aligned. We calculate the syndrome + * value for the aligned 32 bytes block containing the first bytes + * and mask the irrelevant part. + */ + vld1.8 {vdata0, vdata1}, [src:256]! + sub tmp, soff, #32 + adds cntin, cntin, tmp + vceq.i8 vdata0, vdata0, vrepchr + vceq.i8 vdata1, vdata1, vrepchr + vand vdata0, vdata0, vrepmask + vand vdata1, vdata1, vrepmask + vpadd.i8 vdata0_0, vdata0_0, vdata0_1 + vpadd.i8 vdata1_0, vdata1_0, vdata1_1 + vpadd.i8 vdata0_0, vdata0_0, vdata1_0 + vpadd.i8 vdata0_0, vdata0_0, vdata0_0 + vmov synd, vdata0_0[0] + + /* Clear the soff lower bits */ + lsr synd, synd, soff + lsl synd, synd, soff + /* The first block can also be the last */ + bls .Lmasklast + /* Have we found something already? */ + cbnz synd, .Ltail + + +.Lloopintro: + vpush {vend} + /* 264/265 correspond to d8/d9 for q4 */ + .cfi_adjust_cfa_offset 16 + .cfi_rel_offset 264, 0 + .cfi_rel_offset 265, 8 + .p2align 3,,7 +.Lloop: + vld1.8 {vdata0, vdata1}, [src:256]! + subs cntin, cntin, #32 + vceq.i8 vdata0, vdata0, vrepchr + vceq.i8 vdata1, vdata1, vrepchr + /* If we're out of data we finish regardless of the result. */ + bls .Lend + /* Use a fast check for the termination condition. */ + vorr vend, vdata0, vdata1 + vorr vend0, vend0, vend1 + vmov synd, tmp, vend0 + orrs synd, synd, tmp + /* We're not out of data, loop if we haven't found the character. */ + beq .Lloop + +.Lend: + vpop {vend} + .cfi_adjust_cfa_offset -16 + .cfi_restore 264 + .cfi_restore 265 + + /* Termination condition found, let's calculate the syndrome value. */ + vand vdata0, vdata0, vrepmask + vand vdata1, vdata1, vrepmask + vpadd.i8 vdata0_0, vdata0_0, vdata0_1 + vpadd.i8 vdata1_0, vdata1_0, vdata1_1 + vpadd.i8 vdata0_0, vdata0_0, vdata1_0 + vpadd.i8 vdata0_0, vdata0_0, vdata0_0 + vmov synd, vdata0_0[0] + cbz synd, .Lnotfound + bhi .Ltail + + +.Lmasklast: + /* Clear the (-cntin) upper bits to avoid out-of-bounds matches. */ + neg cntin, cntin + lsl synd, synd, cntin + lsrs synd, synd, cntin + it eq + moveq src, #0 /* If no match, set src to 0 so the retval is 0. */ + + +.Ltail: + /* Count the trailing zeros using bit reversing */ + rbit synd, synd + /* Compensate the last post-increment */ + sub src, src, #32 + /* Count the leading zeros */ + clz synd, synd + /* Compute the potential result and return */ + add result, src, synd + bx lr + + +.Lnotfound: + /* Set result to NULL if not found and return */ + mov result, #0 + bx lr + + .cfi_endproc + .size memchr, . - memchr + +#elif __ARM_ARCH_ISA_THUMB >= 2 && defined (__ARM_FEATURE_DSP) + +#if __ARM_ARCH_PROFILE == 'M' + .arch armv7e-m +#else + .arch armv6t2 +#endif + +@ this lets us check a flag in a 00/ff byte easily in either endianness +#ifdef __ARMEB__ +#define CHARTSTMASK(c) 1<<(31-(c*8)) +#else +#define CHARTSTMASK(c) 1<<(c*8) +#endif + .text + .thumb + +@ --------------------------------------------------------------------------- + .thumb_func + .align 2 + .p2align 4,,15 + .global memchr + .type memchr,%function +memchr: + @ r0 = start of memory to scan + @ r1 = character to look for + @ r2 = length + @ returns r0 = pointer to character or NULL if not found + and r1,r1,#0xff @ Don't trust the caller to pass a char + + cmp r2,#16 @ If short don't bother with anything clever + blt 20f + + tst r0, #7 @ If it's already aligned skip the next bit + beq 10f + + @ Work up to an aligned point +5: + ldrb r3, [r0],#1 + subs r2, r2, #1 + cmp r3, r1 + beq 50f @ If it matches exit found + tst r0, #7 + cbz r2, 40f @ If we run off the end, exit not found + bne 5b @ If not aligned yet then do next byte + +10: + @ We are aligned, we know we have at least 8 bytes to work with + push {r4,r5,r6,r7} + orr r1, r1, r1, lsl #8 @ expand the match word across all bytes + orr r1, r1, r1, lsl #16 + bic r4, r2, #7 @ Number of double words to work with * 8 + mvns r7, #0 @ all F's + movs r3, #0 + +15: + ldrd r5,r6,[r0],#8 + subs r4, r4, #8 + eor r5,r5, r1 @ r5,r6 have 00's where bytes match the target + eor r6,r6, r1 + uadd8 r5, r5, r7 @ Par add 0xff - sets GE bits for bytes!=0 + sel r5, r3, r7 @ bytes are 00 for none-00 bytes, + @ or ff for 00 bytes - NOTE INVERSION + uadd8 r6, r6, r7 @ Par add 0xff - sets GE bits for bytes!=0 + sel r6, r5, r7 @ chained....bytes are 00 for none-00 bytes + @ or ff for 00 bytes - NOTE INVERSION + cbnz r6, 60f + bne 15b @ (Flags from the subs above) + + pop {r4,r5,r6,r7} + and r1,r1,#0xff @ r1 back to a single character + and r2,r2,#7 @ Leave the count remaining as the number + @ after the double words have been done + +20: + cbz r2, 40f @ 0 length or hit the end already then not found + +21: @ Post aligned section, or just a short call + ldrb r3,[r0],#1 + subs r2,r2,#1 + eor r3,r3,r1 @ r3 = 0 if match - doesn't break flags from sub + cbz r3, 50f + bne 21b @ on r2 flags + +40: + movs r0,#0 @ not found + bx lr + +50: + subs r0,r0,#1 @ found + bx lr + +60: @ We're here because the fast path found a hit + @ now we have to track down exactly which word it was + @ r0 points to the start of the double word after the one tested + @ r5 has the 00/ff pattern for the first word, r6 has the chained value + cmp r5, #0 + itte eq + moveq r5, r6 @ the end is in the 2nd word + subeq r0,r0,#3 @ Points to 2nd byte of 2nd word + subne r0,r0,#7 @ or 2nd byte of 1st word + + @ r0 currently points to the 2nd byte of the word containing the hit + tst r5, # CHARTSTMASK(0) @ 1st character + bne 61f + adds r0,r0,#1 + tst r5, # CHARTSTMASK(1) @ 2nd character + ittt eq + addeq r0,r0,#1 + tsteq r5, # (3<<15) @ 2nd & 3rd character + @ If not the 3rd must be the last one + addeq r0,r0,#1 + +61: + pop {r4,r5,r6,r7} + subs r0,r0,#1 + bx lr +#else + /* Defined in memchr-stub.c. */ +#endif \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S b/libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S new file mode 100644 index 00000000000..7e7c752c3b2 --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_memcpy.S @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* This memcpy routine is optimised for Cortex-M3/M4 cores with/without + unaligned access. + + If compiled with GCC, this file should be enclosed within following + pre-processing check: + if defined (__ARM_ARCH_7M__) || defined (__ARM_ARCH_7EM__) + + Prototype: void *memcpy (void *dst, const void *src, size_t count); + + The job will be done in 5 steps. + Step 1: Align src/dest pointers, copy mis-aligned if fail to align both + Step 2: Repeatedly copy big block size of __OPT_BIG_BLOCK_SIZE + Step 3: Repeatedly copy big block size of __OPT_MID_BLOCK_SIZE + Step 4: Copy word by word + Step 5: Copy byte-to-byte + + Tunable options: + __OPT_BIG_BLOCK_SIZE: Size of big block in words. Default to 64. + __OPT_MID_BLOCK_SIZE: Size of big block in words. Default to 16. + */ +#ifndef __OPT_BIG_BLOCK_SIZE +#define __OPT_BIG_BLOCK_SIZE (4 * 16) +#endif + +#ifndef __OPT_MID_BLOCK_SIZE +#define __OPT_MID_BLOCK_SIZE (4 * 4) +#endif + +#if __OPT_BIG_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12 +#elif __OPT_BIG_BLOCK_SIZE == 32 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28 +#elif __OPT_BIG_BLOCK_SIZE == 64 +#define BEGIN_UNROLL_BIG_BLOCK \ + .irp offset, 0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60 +#else +#error "Illegal __OPT_BIG_BLOCK_SIZE" +#endif + +#if __OPT_MID_BLOCK_SIZE == 8 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4 +#elif __OPT_MID_BLOCK_SIZE == 16 +#define BEGIN_UNROLL_MID_BLOCK \ + .irp offset, 0,4,8,12 +#else +#error "Illegal __OPT_MID_BLOCK_SIZE" +#endif + +#define END_UNROLL .endr + + .syntax unified + .text + .align 2 + .global memcpy + .thumb + .thumb_func + .type memcpy, %function +memcpy: + @ r0: dst + @ r1: src + @ r2: len +#ifdef __ARM_FEATURE_UNALIGNED + /* In case of UNALIGNED access supported, ip is not used in + function body. */ + mov ip, r0 +#else + push {r0} +#endif + orr r3, r1, r0 + ands r3, r3, #3 + bne .Lmisaligned_copy + +.Lbig_block: + subs r2, __OPT_BIG_BLOCK_SIZE + blo .Lmid_block + + /* Kernel loop for big block copy */ + .align 2 +.Lbig_block_loop: + BEGIN_UNROLL_BIG_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_BIG_BLOCK_SIZE + adds r1, __OPT_BIG_BLOCK_SIZE +#endif + subs r2, __OPT_BIG_BLOCK_SIZE + bhs .Lbig_block_loop + +.Lmid_block: + adds r2, __OPT_BIG_BLOCK_SIZE - __OPT_MID_BLOCK_SIZE + blo .Lcopy_word_by_word + + /* Kernel loop for mid-block copy */ + .align 2 +.Lmid_block_loop: + BEGIN_UNROLL_MID_BLOCK +#ifdef __ARM_ARCH_7EM__ + ldr r3, [r1], #4 + str r3, [r0], #4 + END_UNROLL +#else /* __ARM_ARCH_7M__ */ + ldr r3, [r1, \offset] + str r3, [r0, \offset] + END_UNROLL + adds r0, __OPT_MID_BLOCK_SIZE + adds r1, __OPT_MID_BLOCK_SIZE +#endif + subs r2, __OPT_MID_BLOCK_SIZE + bhs .Lmid_block_loop + +.Lcopy_word_by_word: + adds r2, __OPT_MID_BLOCK_SIZE - 4 + blo .Lcopy_less_than_4 + + /* Kernel loop for small block copy */ + .align 2 +.Lcopy_word_by_word_loop: + ldr r3, [r1], #4 + str r3, [r0], #4 + subs r2, #4 + bhs .Lcopy_word_by_word_loop + +.Lcopy_less_than_4: + adds r2, #4 + beq .Ldone + + lsls r2, r2, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldone +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1] + strh r3, [r0] +#else + ldrb r3, [r1] + strb r3, [r0] + ldrb r3, [r1, #1] + strb r3, [r0, #1] +#endif /* __ARM_FEATURE_UNALIGNED */ + +.Ldone: +#ifdef __ARM_FEATURE_UNALIGNED + mov r0, ip +#else + pop {r0} +#endif + bx lr + + .align 2 +.Lmisaligned_copy: +#ifdef __ARM_FEATURE_UNALIGNED + /* Define label DST_ALIGNED to BIG_BLOCK. It will go to aligned copy + once destination is adjusted to aligned. */ +#define Ldst_aligned Lbig_block + + /* Copy word by word using LDR when alignment can be done in hardware, + i.e., SCTLR.A is set, supporting unaligned access in LDR and STR. */ + + cmp r2, #8 + blo .Lbyte_copy + + /* if src is aligned, just go to the big block loop. */ + lsls r3, r1, #30 + beq .Ldst_aligned +#else + /* if len < 12, misalignment adjustment has more overhead than + just byte-to-byte copy. Also, len must >=8 to guarantee code + afterward work correctly. */ + cmp r2, #12 + blo .Lbyte_copy +#endif /* __ARM_FEATURE_UNALIGNED */ + + /* Align dst only, not trying to align src. That is the because + handling of aligned src and misaligned dst need more overhead than + otherwise. By doing this the worst case is when initial src is aligned, + additional up to 4 byte additional copy will executed, which is + acceptable. */ + + ands r3, r0, #3 + beq .Ldst_aligned + + rsb r3, #4 + subs r2, r3 + + lsls r3, r3, #31 + itt ne + ldrbne r3, [r1], #1 + strbne r3, [r0], #1 + + bcc .Ldst_aligned + +#ifdef __ARM_FEATURE_UNALIGNED + ldrh r3, [r1], #2 + strh r3, [r0], #2 + b .Ldst_aligned +#else + ldrb r3, [r1], #1 + strb r3, [r0], #1 + ldrb r3, [r1], #1 + strb r3, [r0], #1 + /* Now that dst is aligned */ +.Ldst_aligned: + /* if r1 is aligned now, it means r0/r1 has the same misalignment, + and they are both aligned now. Go aligned copy. */ + ands r3, r1, #3 + beq .Lbig_block + + /* dst is aligned, but src isn't. Misaligned copy. */ + + push {r4, r5} + subs r2, #4 + + /* Backward r1 by misaligned bytes, to make r1 aligned. + Since we need to restore r1 to unaligned address after the loop, + we need keep the offset bytes to ip and sub it from r1 afterward. */ + subs r1, r3 + rsb ip, r3, #4 + + /* Pre-load on word */ + ldr r4, [r1], #4 + + cmp r3, #2 + beq .Lmisaligned_copy_2_2 + cmp r3, #3 + beq .Lmisaligned_copy_3_1 + + .macro mis_src_copy shift +1: +#ifdef __ARM_BIG_ENDIAN + lsls r4, r4, \shift +#else + lsrs r4, r4, \shift +#endif + ldr r3, [r1], #4 +#ifdef __ARM_BIG_ENDIAN + lsrs r5, r3, 32-\shift +#else + lsls r5, r3, 32-\shift +#endif + orr r4, r4, r5 + str r4, [r0], #4 + mov r4, r3 + subs r2, #4 + bhs 1b + .endm + +.Lmisaligned_copy_1_3: + mis_src_copy shift=8 + b .Lsrc_misaligned_tail + +.Lmisaligned_copy_3_1: + mis_src_copy shift=24 + b .Lsrc_misaligned_tail + +.Lmisaligned_copy_2_2: + /* For 2_2 misalignment, ldr is still faster than 2 x ldrh. */ + mis_src_copy shift=16 + +.Lsrc_misaligned_tail: + adds r2, #4 + subs r1, ip + pop {r4, r5} + +#endif /* __ARM_FEATURE_UNALIGNED */ + +.Lbyte_copy: + subs r2, #4 + blo .Lcopy_less_than_4 + +.Lbyte_copy_loop: + subs r2, #1 + ldrb r3, [r1], #1 + strb r3, [r0], #1 + bhs .Lbyte_copy_loop + + ldrb r3, [r1] + strb r3, [r0] + ldrb r3, [r1, #1] + strb r3, [r0, #1] + ldrb r3, [r1, #2] + strb r3, [r0, #2] + +#ifdef __ARM_FEATURE_UNALIGNED + mov r0, ip +#else + pop {r0} +#endif + bx lr + + .size memcpy, .-memcpy \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_memmove.S b/libs/libc/machine/arm/armv8-m/gnu/arch_memmove.S new file mode 100644 index 00000000000..8a54e908a82 --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_memmove.S @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2015 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + .thumb + .syntax unified + .global memmove + .type memmove, %function +memmove: + cmp r0, r1 + push {r4} + bls 3f + adds r3, r1, r2 + cmp r0, r3 + bcs 3f + adds r1, r0, r2 + cbz r2, 2f + subs r2, r3, r2 +1: + ldrb r4, [r3, #-1]! + cmp r2, r3 + strb r4, [r1, #-1]! + bne 1b +2: + pop {r4} + bx lr +3: + cmp r2, #0 + beq 2b + add r2, r2, r1 + subs r3, r0, #1 +4: + ldrb r4, [r1], #1 + cmp r2, r1 + strb r4, [r3, #1]! + bne 4b + pop {r4} + bx lr + .size memmove, . - memmove \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_memset.S b/libs/libc/machine/arm/armv8-m/gnu/arch_memset.S new file mode 100644 index 00000000000..fa68c171f2e --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_memset.S @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2015 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + .thumb + .syntax unified + .global memset + .type memset, %function +memset: + push {r4, r5, r6} + lsls r4, r0, #30 + beq 10f + subs r4, r1, #1 + cmp r1, #0 + beq 9f + uxtb r5, r2 + mov r3, r0 + b 2f +1: + subs r1, r4, #1 + cbz r4, 9f + mov r4, r1 +2: + strb r5, [r3], #1 + lsls r1, r3, #30 + bne 1b +3: + cmp r4, #3 + bls 7f + uxtb r5, r2 + orr r5, r5, r5, lsl #8 + cmp r4, #15 + orr r5, r5, r5, lsl #16 + bls 5f + mov r6, r4 + add r1, r3, #16 +4: + subs r6, r6, #16 + cmp r6, #15 + str r5, [r1, #-16] + str r5, [r1, #-12] + str r5, [r1, #-8] + str r5, [r1, #-4] + add r1, r1, #16 + bhi 4b + sub r1, r4, #16 + bic r1, r1, #15 + and r4, r4, #15 + adds r1, r1, #16 + cmp r4, #3 + add r3, r3, r1 + bls 7f +5: + mov r6, r3 + mov r1, r4 +6: + subs r1, r1, #4 + cmp r1, #3 + str r5, [r6], #4 + bhi 6b + subs r1, r4, #4 + bic r1, r1, #3 + adds r1, r1, #4 + add r3, r3, r1 + and r4, r4, #3 +7: + cbz r4, 9f + uxtb r2, r2 + add r4, r4, r3 +8: + strb r2, [r3], #1 + cmp r3, r4 + bne 8b +9: + pop {r4, r5, r6} + bx lr +10: + mov r4, r1 + mov r3, r0 + b 3b + .size memset, . - memset \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_strcmp.S b/libs/libc/machine/arm/armv8-m/gnu/arch_strcmp.S new file mode 100644 index 00000000000..9612297b73d --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_strcmp.S @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2012-2014 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef __ARM_BIG_ENDIAN +#define S2LO lsl +#define S2LOEQ lsleq +#define S2HI lsr +#define MSB 0x000000ff +#define LSB 0xff000000 +#define BYTE0_OFFSET 24 +#define BYTE1_OFFSET 16 +#define BYTE2_OFFSET 8 +#define BYTE3_OFFSET 0 +#else /* not __ARM_BIG_ENDIAN */ +#define S2LO lsr +#define S2LOEQ lsreq +#define S2HI lsl +#define BYTE0_OFFSET 0 +#define BYTE1_OFFSET 8 +#define BYTE2_OFFSET 16 +#define BYTE3_OFFSET 24 +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif /* not __ARM_BIG_ENDIAN */ + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm + +/* Very similar to the generic code, but uses Thumb2 as implemented + in ARMv7-M. */ + +/* Parameters and result. */ +#define src1 r0 +#define src2 r1 +#define result r0 /* Overlaps src1. */ + +/* Internal variables. */ +#define data1 r2 +#define data2 r3 +#define tmp2 r5 +#define tmp1 r12 +#define syndrome r12 /* Overlaps tmp1 */ + + .thumb + .syntax unified +def_fn strcmp + .cfi_sections .debug_frame + .cfi_startproc + eor tmp1, src1, src2 + tst tmp1, #3 + /* Strings not at same byte offset from a word boundary. */ + bne .Lstrcmp_unaligned + ands tmp1, src1, #3 + bic src1, src1, #3 + bic src2, src2, #3 + ldr data1, [src1], #4 + it eq + ldreq data2, [src2], #4 + beq 4f + /* Although s1 and s2 have identical initial alignment, they are + not currently word aligned. Rather than comparing bytes, + make sure that any bytes fetched from before the addressed + bytes are forced to 0xff. Then they will always compare + equal. */ + eor tmp1, tmp1, #3 + mvn data2, #MSB + lsl tmp1, tmp1, #3 + S2LO tmp1, data2, tmp1 + ldr data2, [src2], #4 + orr data1, data1, tmp1 + orr data2, data2, tmp1 + .p2align 2 + /* Critical loop. */ +4: + sub syndrome, data1, #0x01010101 + cmp data1, data2 + /* check for any zero bytes in first word */ + itttt eq + biceq syndrome, syndrome, data1 + tsteq syndrome, #0x80808080 + ldreq data1, [src1], #4 + ldreq data2, [src2], #4 + beq 4b +2: + /* There's a zero or a different byte in the word */ + S2HI result, data1, #24 + S2LO data1, data1, #8 + cmp result, #1 + it cs + cmpcs result, data2, S2HI #24 + it eq + S2LOEQ data2, data2, #8 + beq 2b + /* On a big-endian machine, RESULT contains the desired byte in bits + 0-7; on a little-endian machine they are in bits 24-31. In + both cases the other bits in RESULT are all zero. For DATA2 the + interesting byte is at the other end of the word, but the + other bits are not necessarily zero. We need a signed result + representing the differnece in the unsigned bytes, so for the + little-endian case we can't just shift the interesting bits + up. */ +#ifdef __ARM_BIG_ENDIAN + sub result, result, data2, lsr #24 +#else + and data2, data2, #255 + lsrs result, result, #24 + subs result, result, data2 +#endif + bx lr + + +#if 0 + /* The assembly code below is based on the following alogrithm. */ +#ifdef __ARM_BIG_ENDIAN +#define RSHIFT << +#define LSHIFT >> +#else +#define RSHIFT >> +#define LSHIFT << +#endif + +#define body(shift) \ + mask = 0xffffffffU RSHIFT shift; \ + data1 = *src1++; \ + data2 = *src2++; \ + do \ + { \ + tmp2 = data1 & mask; \ + if (__builtin_expect(tmp2 != data2 RSHIFT shift, 0)) \ + { \ + data2 RSHIFT= shift; \ + break; \ + } \ + if (__builtin_expect(((data1 - b1) & ~data1) & (b1 << 7), 0)) \ + { \ + /* See comment in assembler below re syndrome on big-endian */\ + if ((((data1 - b1) & ~data1) & (b1 << 7)) & mask) \ + data2 RSHIFT= shift; \ + else \ + { \ + data2 = *src2; \ + tmp2 = data1 RSHIFT (32 - shift); \ + data2 = (data2 LSHIFT (32 - shift)) RSHIFT (32 - shift); \ + } \ + break; \ + } \ + data2 = *src2++; \ + tmp2 ^= data1; \ + if (__builtin_expect(tmp2 != data2 LSHIFT (32 - shift), 0)) \ + { \ + tmp2 = data1 >> (32 - shift); \ + data2 = (data2 << (32 - shift)) RSHIFT (32 - shift); \ + break; \ + } \ + data1 = *src1++; \ + } while (1) + + const unsigned* src1; + const unsigned* src2; + unsigned data1, data2; + unsigned mask; + unsigned shift; + unsigned b1 = 0x01010101; + char c1, c2; + unsigned tmp2; + + while (((unsigned) s1) & 3) + { + c1 = *s1++; + c2 = *s2++; + if (c1 == 0 || c1 != c2) + return c1 - (int)c2; + } + src1 = (unsigned*) (((unsigned)s1) & ~3); + src2 = (unsigned*) (((unsigned)s2) & ~3); + tmp2 = ((unsigned) s2) & 3; + if (tmp2 == 1) + { + body(8); + } + else if (tmp2 == 2) + { + body(16); + } + else + { + body (24); + } + + do + { +#ifdef __ARM_BIG_ENDIAN + c1 = (char) tmp2 >> 24; + c2 = (char) data2 >> 24; +#else /* not __ARM_BIG_ENDIAN */ + c1 = (char) tmp2; + c2 = (char) data2; +#endif /* not __ARM_BIG_ENDIAN */ + tmp2 RSHIFT= 8; + data2 RSHIFT= 8; + } while (c1 != 0 && c1 == c2); + return c1 - c2; +#endif /* 0 */ + + + /* First of all, compare bytes until src1(sp1) is word-aligned. */ +.Lstrcmp_unaligned: + tst src1, #3 + beq 2f + ldrb data1, [src1], #1 + ldrb data2, [src2], #1 + cmp data1, #1 + it cs + cmpcs data1, data2 + beq .Lstrcmp_unaligned + sub result, data1, data2 + bx lr + +2: + stmfd sp!, {r5} + .cfi_def_cfa_offset 4 + .cfi_offset 5, -4 + + ldr data1, [src1], #4 + and tmp2, src2, #3 + bic src2, src2, #3 + ldr data2, [src2], #4 + cmp tmp2, #2 + beq .Loverlap2 + bhi .Loverlap1 + + /* Critical inner Loop: Block with 3 bytes initial overlap */ + .p2align 2 +.Loverlap3: + bic tmp2, data1, #MSB + cmp tmp2, data2, S2LO #8 + sub syndrome, data1, #0x01010101 + bic syndrome, syndrome, data1 + bne 4f + ands syndrome, syndrome, #0x80808080 + it eq + ldreq data2, [src2], #4 + bne 5f + eor tmp2, tmp2, data1 + cmp tmp2, data2, S2HI #24 + bne 6f + ldr data1, [src1], #4 + b .Loverlap3 +4: + S2LO data2, data2, #8 + b .Lstrcmp_tail + +5: +#ifdef __ARM_BIG_ENDIAN + /* The syndrome value may contain false ones if the string ends + with the bytes 0x01 0x00. */ + tst data1, #0xff000000 + itt ne + tstne data1, #0x00ff0000 + tstne data1, #0x0000ff00 + beq .Lstrcmp_done_equal +#else + bics syndrome, syndrome, #0xff000000 + bne .Lstrcmp_done_equal +#endif + ldrb data2, [src2] + S2LO tmp2, data1, #24 +#ifdef __ARM_BIG_ENDIAN + lsl data2, data2, #24 +#endif + b .Lstrcmp_tail + +6: + S2LO tmp2, data1, #24 + and data2, data2, #LSB + b .Lstrcmp_tail + + /* Critical inner Loop: Block with 2 bytes initial overlap. */ + .p2align 2 +.Loverlap2: + S2HI tmp2, data1, #16 + sub syndrome, data1, #0x01010101 + S2LO tmp2, tmp2, #16 + bic syndrome, syndrome, data1 + cmp tmp2, data2, S2LO #16 + bne 4f + ands syndrome, syndrome, #0x80808080 + it eq + ldreq data2, [src2], #4 + bne 5f + eor tmp2, tmp2, data1 + cmp tmp2, data2, S2HI #16 + bne 6f + ldr data1, [src1], #4 + b .Loverlap2 + +5: +#ifdef __ARM_BIG_ENDIAN + /* The syndrome value may contain false ones if the string ends + with the bytes 0x01 0x00 */ + tst data1, #0xff000000 + it ne + tstne data1, #0x00ff0000 + beq .Lstrcmp_done_equal +#else + lsls syndrome, syndrome, #16 + bne .Lstrcmp_done_equal +#endif + ldrh data2, [src2] + S2LO tmp2, data1, #16 +#ifdef __ARM_BIG_ENDIAN + lsl data2, data2, #16 +#endif + b .Lstrcmp_tail + +6: + S2HI data2, data2, #16 + S2LO tmp2, data1, #16 +4: + S2LO data2, data2, #16 + b .Lstrcmp_tail + + /* Critical inner Loop: Block with 1 byte initial overlap. */ + .p2align 2 +.Loverlap1: + and tmp2, data1, #LSB + cmp tmp2, data2, S2LO #24 + sub syndrome, data1, #0x01010101 + bic syndrome, syndrome, data1 + bne 4f + ands syndrome, syndrome, #0x80808080 + it eq + ldreq data2, [src2], #4 + bne 5f + eor tmp2, tmp2, data1 + cmp tmp2, data2, S2HI #8 + bne 6f + ldr data1, [src1], #4 + b .Loverlap1 +4: + S2LO data2, data2, #24 + b .Lstrcmp_tail +5: + /* The syndrome value may contain false ones if the string ends + with the bytes 0x01 0x00. */ + tst data1, #LSB + beq .Lstrcmp_done_equal + ldr data2, [src2], #4 +6: + S2LO tmp2, data1, #8 + bic data2, data2, #MSB + b .Lstrcmp_tail +.Lstrcmp_done_equal: + mov result, #0 + .cfi_remember_state + ldmfd sp!, {r5} + .cfi_restore 5 + .cfi_def_cfa_offset 0 + bx lr + +.Lstrcmp_tail: + .cfi_restore_state + and r2, tmp2, #LSB + and result, data2, #LSB + cmp result, #1 + it cs + cmpcs result, r2 + itt eq + S2LOEQ tmp2, tmp2, #8 + S2LOEQ data2, data2, #8 + beq .Lstrcmp_tail + sub result, r2, result + ldmfd sp!, {r5} + .cfi_restore 5 + .cfi_def_cfa_offset 0 + bx lr + .cfi_endproc + .size strcmp, . - strcmp \ No newline at end of file diff --git a/libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S b/libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S new file mode 100644 index 00000000000..a1e6c8b2bde --- /dev/null +++ b/libs/libc/machine/arm/armv8-m/gnu/arch_strlen.S @@ -0,0 +1,179 @@ +/* Copyright (c) 2010-2011,2013 Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Assumes: + ARMv6T2 or ARMv7E-M, AArch32 + */ + +/* Copyright (c) 2015 ARM Ltd. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Linaro nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +#include "acle-compat.h" + + .macro def_fn f p2align=0 + .text + .p2align \p2align + .global \f + .type \f, %function +\f: + .endm +#ifdef __ARMEB__ +#define S2LO lsl +#define S2HI lsr +#else +#define S2LO lsr +#define S2HI lsl +#endif + + /* This code requires Thumb. */ +#if __ARM_ARCH_PROFILE == 'M' + .arch armv7e-m +#else + .arch armv6t2 +#endif + .eabi_attribute Tag_ARM_ISA_use, 0 + .thumb + .syntax unified + +/* Parameters and result. */ +#define srcin r0 +#define result r0 + +/* Internal variables. */ +#define src r1 +#define data1a r2 +#define data1b r3 +#define const_m1 r12 +#define const_0 r4 +#define tmp1 r4 /* Overlaps const_0 */ +#define tmp2 r5 + +def_fn strlen p2align=6 + pld [srcin, #0] + strd r4, r5, [sp, #-8]! + bic src, srcin, #7 + mvn const_m1, #0 + ands tmp1, srcin, #7 /* (8 - bytes) to alignment. */ + pld [src, #32] + bne.w .Lmisaligned8 + mov const_0, #0 + mov result, #-8 +.Lloop_aligned: + /* Bytes 0-7. */ + ldrd data1a, data1b, [src] + pld [src, #64] + add result, result, #8 +.Lstart_realigned: + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, .Lnull_found + + /* Bytes 8-15. */ + ldrd data1a, data1b, [src, #8] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, .Lnull_found + + /* Bytes 16-23. */ + ldrd data1a, data1b, [src, #16] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, .Lnull_found + + /* Bytes 24-31. */ + ldrd data1a, data1b, [src, #24] + add src, src, #32 + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cmp data1b, #0 + beq .Lloop_aligned + +.Lnull_found: + cmp data1a, #0 + itt eq + addeq result, result, #4 + moveq data1a, data1b +#ifndef __ARMEB__ + rev data1a, data1a +#endif + clz data1a, data1a + ldrd r4, r5, [sp], #8 + add result, result, data1a, lsr #3 /* Bits -> Bytes. */ + bx lr + +.Lmisaligned8: + ldrd data1a, data1b, [src] + and tmp2, tmp1, #3 + rsb result, tmp1, #0 + lsl tmp2, tmp2, #3 /* Bytes -> bits. */ + tst tmp1, #4 + pld [src, #64] + S2HI tmp2, const_m1, tmp2 + orn data1a, data1a, tmp2 + itt ne + ornne data1b, data1b, tmp2 + movne data1a, const_m1 + mov const_0, #0 + b .Lstart_realigned + .size strlen, . - strlen \ No newline at end of file