From 967c7c19b54fd0db820bbfa1cbe199a8ac9f5419 Mon Sep 17 00:00:00 2001 From: Richard Thier Date: Tue, 30 Sep 2025 22:12:22 +0200 Subject: [PATCH] thier3: write caching queues fixed - bug just makes it slower despite less cache misses --- thiersort3.h | 4 +- threepass_xbit.h | 117 +++++++++++++++++++++++++++++++++++++++-------- ypsu.cpp | 4 +- 3 files changed, 101 insertions(+), 24 deletions(-) diff --git a/thiersort3.h b/thiersort3.h index 546f2e4..fe37e2b 100644 --- a/thiersort3.h +++ b/thiersort3.h @@ -12,7 +12,7 @@ union th3_fu { typedef union th3_fu th3_fu; /** Tells from the key which bucket it is in. */ -static inline uint32_t witch_bucket3(uint32_t key) { +static inline uint32_t witch_bucket3(uint32_t key) TPBX_NOEXCEPT { /* This is hackz to misuse int->float converter HEAVILY and IEE for bucketing */ /* https://en.wikipedia.org/wiki/Single-precision_floating-point_format */ @@ -41,7 +41,7 @@ static inline uint32_t witch_bucket3(uint32_t key) { * @param n Number of elements in arr and temparr * @param rstate Create with sch_rand_state rstate = schwab_rand_state(junk_uint32_t); */ -static inline void thiersort3(uint32_t *arr, uint32_t *temparr, uint32_t n) { +static inline void thiersort3(uint32_t *arr, uint32_t *temparr, uint32_t n) TPBX_NOEXCEPT { uint32_t bucket[4096]; /* Inclusive */ uint32_t bucket_end[4096]; /* Not inclusive */ diff --git a/threepass_xbit.h b/threepass_xbit.h index 3e619a8..86cb2cf 100644 --- a/threepass_xbit.h +++ b/threepass_xbit.h @@ -8,38 +8,115 @@ /* - This means those bits in numbers DO get used when we are running as internal sort of thier3 */ /* - But the numbers high bits (29..32th bits) stay the same inside each bucket so we spare it! */ /* - If you know your data does not have all the range of 32 bits you can #define these by #define CUSTOM_TPBX_BITS */ + +/* Only define these to be smaller than these in case your input data shows you can - never change orders or make bigger! */ #ifndef CUSTOM_TPBX_BITS #define TPBX1 10 // top #define TPBX2 9 // mid #define TPBX3 9 // bottom #endif /* CUSTOM_TPBX_BITS */ -static inline constexpr uint32_t min3u32_xb(uint32_t a, uint32_t b, uint32_t c) noexcept { + +#ifndef TPBX_NO_DEF_WCACHE +#define TPBX_WCACHE 2 /* depth of the write-cache - I let the user */ +#endif /* TPBX_NO_DEF_WCACHE */ + +#ifdef __cplusplus +#define TPBX_NOEXCEPT noexcept +#define TPBX_CONSTEXPR constexpr +#else +#define TPBX_NOEXCEPT +#define TPBX_CONSTEXPR const +#endif /* __cplusplus */ + +static inline TPBX_CONSTEXPR uint32_t min3u32_xb(uint32_t a, uint32_t b, uint32_t c) TPBX_NOEXCEPT { return (a <= b) ? ((a <= c) ? a : c) : ((b <= c) ? b : c); } -/** Copy the elements to their respective radics-place (f->t copy) */ -static inline void copy_radics_tpxp(uint32_t *f, uint32_t *t, uint32_t *bucket, uint32_t shr, uint32_t mask, uint32_t n) { +static inline TPBX_CONSTEXPR uint32_t max3u32_xb(uint32_t a, uint32_t b, uint32_t c) TPBX_NOEXCEPT { + return (a >= b) ? + ((a >= c) ? a : c) : + ((b >= c) ? b : c); +} + +/** Copy the elements to their respective radics-place (f->t copy) bit should be the TPBX1,2,3 */ +static inline void copy_radics_tpxp( + uint32_t *f, + uint32_t *t, + uint32_t *bucket, + uint32_t shr, + uint32_t mask, + uint32_t n, + uint32_t bit) { +#ifndef TPBX_WCACHE // right-to-left to ensure already sorted digits order we keep for iterations #pragma GCC unroll 48 for(uint32_t i = n; i > 0; --i) { // Prefetch caches //__builtin_prefetch(&a[i-8]); // Get num and its new offset / location - auto num = f[i - 1]; - auto bkeyni = (num >> shr) & mask; - auto offset = --bucket[bkeyni]; + uint32_t num = f[i - 1]; + uint32_t bkeyni = (num >> shr) & mask; + uint32_t offset = --bucket[bkeyni]; // Add to the proper target location t[offset] = num; } +#else + int stride = (1 << bit); + static __thread uint32_t write_cache[1024 * TPBX_WCACHE]; + static __thread uint8_t write_qc[1024]; + + for(uint32_t i = 0; i < 1024; ++i) { + write_qc[i] = 0; + } + + /* right-to-left to ensure already sorted digits order we keep for iterations */ + #pragma GCC unroll 4 + for(uint32_t i = n; i > 0; --i) { + /* calculate which bucket this would belong and where the queue is */ + uint32_t num = f[i - 1]; + uint32_t bkeyni = (num >> shr) & mask; + uint8_t qc = write_qc[bkeyni]; + + if(qc < TPBX_WCACHE) { + /* Cache is not full yet */ + write_cache[bkeyni + stride * qc] = num; + ++write_qc[bkeyni]; + } else { + /* Cache is full, make a real writeout - all from this queue */ + + /* We also directly move the current element too - micro optimized */ + uint32_t offset = (bucket[bkeyni] -= (TPBX_WCACHE + 1)); + t[offset++] = num; + /* Copy from the cache - order counts here because keeping opposites! */ + for(uint32_t j = 0; j < TPBX_WCACHE; ++j) { + uint32_t cached = write_cache[bkeyni + stride * (TPBX_WCACHE - 1 - j)]; + t[offset + j] = cached; + } + + /* Reset this queue */ + write_qc[bkeyni] = 0; + } + } + + /* Write out any data remaining in the write cache */ + for(uint32_t bkeyni = 0; bkeyni < stride; ++bkeyni) { + uint32_t qc = write_qc[bkeyni]; + uint32_t offset = (bucket[bkeyni] -= qc); + if(qc > 0) for(uint32_t j = 0; j < qc; ++j) { + uint32_t cached = write_cache[bkeyni + stride * (qc - 1 - j)]; + t[offset + j] = cached; + } + } +#endif /* TPBX_WCACHE */ } /* I pulled these out only for better flame graph support */ /** Count occurences (can count together with good ILP) */ -static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, uint32_t *bucket3, const uint32_t shr1, const uint32_t shr2, const uint32_t shr3, const uint32_t mask1, const uint32_t mask2, const uint32_t mask3, uint32_t *a, uint32_t n) noexcept { +static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, uint32_t *bucket3, const uint32_t shr1, const uint32_t shr2, const uint32_t shr3, const uint32_t mask1, const uint32_t mask2, const uint32_t mask3, uint32_t *a, uint32_t n) TPBX_NOEXCEPT { #pragma GCC unroll 64 for(uint32_t i = 0; i < n; ++i) { ++bucket1[(a[i] >> shr1) & mask1]; @@ -55,23 +132,23 @@ static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, u * @param buf Result array with the same size - result will be here * @param n The number of elements */ -static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) noexcept { +static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) TPBX_NOEXCEPT { assert(buf != NULL); - constexpr uint32_t shr1 = TPBX3 + TPBX2; - constexpr uint32_t shr2 = TPBX3; - constexpr uint32_t shr3 = 0; - constexpr uint32_t mask1 = (1 << TPBX1) - 1; - constexpr uint32_t mask2 = (1 << TPBX2) - 1; - constexpr uint32_t mask3 = (1 << TPBX3) - 1; + TPBX_CONSTEXPR uint32_t shr1 = TPBX3 + TPBX2; + TPBX_CONSTEXPR uint32_t shr2 = TPBX3; + TPBX_CONSTEXPR uint32_t shr3 = 0; + TPBX_CONSTEXPR uint32_t mask1 = (1 << TPBX1) - 1; + TPBX_CONSTEXPR uint32_t mask2 = (1 << TPBX2) - 1; + TPBX_CONSTEXPR uint32_t mask3 = (1 << TPBX3) - 1; /* helper buffers. */ uint32_t sz = n * sizeof(a[0]); - static thread_local uint32_t bucket1[1 << TPBX1]; + static __thread uint32_t bucket1[1 << TPBX1]; memset(bucket1, 0, (1 << TPBX1) * sizeof(uint32_t)); - static thread_local uint32_t bucket2[1 << TPBX2]; + static __thread uint32_t bucket2[1 << TPBX2]; memset(bucket2, 0, (1 << TPBX2) * sizeof(uint32_t)); - static thread_local uint32_t bucket3[1 << TPBX3]; + static __thread uint32_t bucket3[1 << TPBX3]; memset(bucket3, 0, (1 << TPBX3) * sizeof(uint32_t)); count_occurences_tpxp(bucket1, bucket2, bucket3, shr1, shr2, shr3, mask1, mask2, mask3, a, n); @@ -112,13 +189,13 @@ static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) noexcept } // Bottom digit a->buf - copy_radics_tpxp(a, buf, bucket3, shr3, mask3, n); + copy_radics_tpxp(a, buf, bucket3, shr3, mask3, n, TPBX3); // Mid digit buf->a - copy_radics_tpxp(buf, a, bucket2, shr2, mask2, n); + copy_radics_tpxp(buf, a, bucket2, shr2, mask2, n, TPBX2); // Top digit a->buf - copy_radics_tpxp(a, buf, bucket1, shr1, mask1, n); + copy_radics_tpxp(a, buf, bucket1, shr1, mask1, n, TPBX1); } #endif /* THREE_PASS_XB_H */ diff --git a/ypsu.cpp b/ypsu.cpp index 48c6988..9520774 100644 --- a/ypsu.cpp +++ b/ypsu.cpp @@ -896,8 +896,8 @@ int main(int argc, char **argv) { printf("Sorting %d elements:\n\n", n); // Uncomment this for profiling and alg! - measure_single(n); - return 0; + //measure_single(n); + //return 0; for (auto inputtype : inputtypes) { printf("%10s", inputtype.c_str());