platform_bionic/libc/arch-x86/dynamic_function_dispatch.cpp
ahs 919fb7f2e0 avx2 implementation for memset.
This patch includes handwritten avx2
assembly for memset 64-bit. Uses
non-temporal stores for very large sizes.
Also includes dynamic dispatch for APIs
having multiple implementations.

Convincing benchmark improvements for sizes above 512 bytes, and
although the slight regression for small sizes is unfortunate, it's
probably small enough to be okay?

Before:

  BM_string_memset/8/0            3.06 ns         3.04 ns    222703428 bytes_per_second=2.45261G/s
  BM_string_memset/16/0           3.50 ns         3.47 ns    202569932 bytes_per_second=4.29686G/s
  BM_string_memset/32/0           3.50 ns         3.48 ns    200064955 bytes_per_second=8.57386G/s
  BM_string_memset/64/0           3.49 ns         3.46 ns    201928186 bytes_per_second=17.2184G/s
  BM_string_memset/512/0          14.8 ns         14.7 ns     47776178 bytes_per_second=32.3887G/s
  BM_string_memset/1024/0         27.3 ns         27.1 ns     25884933 bytes_per_second=35.2515G/s
  BM_string_memset/8192/0          203 ns          201 ns      3476903 bytes_per_second=37.9311G/s
  BM_string_memset/16384/0         402 ns          399 ns      1750471 bytes_per_second=38.2725G/s
  BM_string_memset/32768/0         932 ns          925 ns       755750 bytes_per_second=33.0071G/s
  BM_string_memset/65536/0        2038 ns         2014 ns       347060 bytes_per_second=30.3057G/s
  BM_string_memset/131072/0       4012 ns         3980 ns       175186 bytes_per_second=30.6682G/s

After:

  BM_string_memset/8/0            3.32 ns         3.23 ns    208939089 bytes_per_second=2.3051G/s
  BM_string_memset/16/0           4.07 ns         3.98 ns    173479615 bytes_per_second=3.74822G/s
  BM_string_memset/32/0           4.07 ns         3.95 ns    177208119 bytes_per_second=7.54344G/s
  BM_string_memset/64/0           4.09 ns         4.00 ns    174729144 bytes_per_second=14.8878G/s
  BM_string_memset/512/0          10.7 ns         10.4 ns     65922763 bytes_per_second=45.6611G/s
  BM_string_memset/1024/0         18.0 ns         17.6 ns     40489136 bytes_per_second=54.3166G/s
  BM_string_memset/8192/0          109 ns          106 ns      6577711 bytes_per_second=71.7667G/s
  BM_string_memset/16384/0         221 ns          210 ns      3343800 bytes_per_second=72.684G/s
  BM_string_memset/32768/0         655 ns          623 ns      1153501 bytes_per_second=48.9781G/s
  BM_string_memset/65536/0        1547 ns         1495 ns       461702 bytes_per_second=40.8154G/s
  BM_string_memset/131072/0       2991 ns         2924 ns       240189 bytes_per_second=41.7438G/s

This patch drops the wmemset() code because we don't even have a
microbenchmark for it, we have as many implementations checked in as we
have non-test call sites (!), so at this point it seems like we've spent
more time maintaining wmemset() than running it!

Test: bionic/tests/run-on-host.sh 64
Signed-off-by: ahs <amrita.h.s@intel.com>
Change-Id: Ie5047df5300638c1e4c69f8285d33d034f79c83b
2022-07-22 21:48:50 +00:00

154 lines
5.6 KiB
C++

/*
* Copyright (C) 2008 The Android Open Source Project
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <stddef.h>
#include <private/bionic_ifuncs.h>
extern "C" {
typedef int memcmp_func(const void* __lhs, const void* __rhs, size_t __n);
DEFINE_IFUNC_FOR(memcmp) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func, memcmp_atom);
if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func, memcmp_sse4);
RETURN_FUNC(memcmp_func, memcmp_generic);
}
typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC_FOR(memset) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func, memset_atom);
RETURN_FUNC(memset_func, memset_generic);
}
typedef void* __memset_chk_func(void *s, int c, size_t n, size_t n2);
DEFINE_IFUNC_FOR(__memset_chk) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func, __memset_chk_atom);
RETURN_FUNC(__memset_chk_func, __memset_chk_generic);
}
typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC_FOR(memmove) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func, memmove_atom);
RETURN_FUNC(memmove_func, memmove_generic);
}
typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC_FOR(memcpy) {
return memmove_resolver();
}
typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcpy) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func, strcpy_atom);
RETURN_FUNC(strcpy_func, strcpy_generic);
}
typedef char* strncpy_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncpy) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func, strncpy_atom);
RETURN_FUNC(strncpy_func, strncpy_generic);
}
typedef size_t strlen_func(const char* __s);
DEFINE_IFUNC_FOR(strlen) {
__builtin_cpu_init();
if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func, strlen_atom);
RETURN_FUNC(strlen_func, strlen_generic);
}
typedef int wmemcmp_func(const wchar_t* __lhs, const wchar_t* __rhs, size_t __n);
DEFINE_IFUNC_FOR(wmemcmp) {
__builtin_cpu_init();
if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func, wmemcmp_sse4);
if (__builtin_cpu_is("atom")) RETURN_FUNC(wmemcmp_func, wmemcmp_atom);
RETURN_FUNC(wmemcmp_func, wmemcmp_freebsd);
}
typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_IFUNC_FOR(strcmp) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcmp_func, strcmp_ssse3);
RETURN_FUNC(strcmp_func, strcmp_generic);
}
typedef int strncmp_func(const char* __lhs, const char* __rhs, size_t __n);
DEFINE_IFUNC_FOR(strncmp) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncmp_func, strncmp_ssse3);
RETURN_FUNC(strncmp_func, strncmp_generic);
}
typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC_FOR(strcat) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strcat_func, strcat_ssse3);
RETURN_FUNC(strcat_func, strcat_generic);
}
typedef char* strncat_func(char* __dst, const char* __src, size_t __n);
DEFINE_IFUNC_FOR(strncat) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strncat_func, strncat_ssse3);
RETURN_FUNC(strncat_func, strncat_openbsd);
}
typedef size_t strlcat_func(char *dst, const char *src, size_t dsize);
DEFINE_IFUNC_FOR(strlcat) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcat_func, strlcat_ssse3);
RETURN_FUNC(strlcat_func, strlcat_openbsd);
}
typedef size_t strlcpy_func(char *dst, const char *src, size_t dsize);
DEFINE_IFUNC_FOR(strlcpy) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(strlcpy_func, strlcpy_ssse3);
RETURN_FUNC(strlcpy_func, strlcpy_openbsd);
}
typedef wchar_t* wcscat_func(wchar_t *s1, const wchar_t *s2);
DEFINE_IFUNC_FOR(wcscat) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscat_func, wcscat_ssse3);
RETURN_FUNC(wcscat_func, wcscat_freebsd);
}
typedef wchar_t* wcscpy_func(wchar_t *s1, const wchar_t *s2);
DEFINE_IFUNC_FOR(wcscpy) {
__builtin_cpu_init();
if (__builtin_cpu_supports("ssse3")) RETURN_FUNC(wcscpy_func, wcscpy_ssse3);
RETURN_FUNC(wcscpy_func, wcscpy_freebsd);
}
} // extern "C"