thier3: write caching queues fixed - bug just makes it slower despite less cache misses

thiersort3.h

@@ -12,7 +12,7 @@ union th3_fu {
 typedef union th3_fu th3_fu;
 
 /** Tells from the key which bucket it is in. */
-static inline uint32_t witch_bucket3(uint32_t key) {
+static inline uint32_t witch_bucket3(uint32_t key) TPBX_NOEXCEPT {
 	/* This is hackz to misuse int->float converter HEAVILY and IEE for bucketing */
 	/* https://en.wikipedia.org/wiki/Single-precision_floating-point_format */
 
@@ -41,7 +41,7 @@ static inline uint32_t witch_bucket3(uint32_t key) {
  * @param n Number of elements in arr and temparr
  * @param rstate Create with sch_rand_state rstate = schwab_rand_state(junk_uint32_t);
  */
-static inline void thiersort3(uint32_t *arr, uint32_t *temparr, uint32_t n) {
+static inline void thiersort3(uint32_t *arr, uint32_t *temparr, uint32_t n) TPBX_NOEXCEPT {
 	uint32_t bucket[4096]; /* Inclusive */
 	uint32_t bucket_end[4096]; /* Not inclusive */
 

threepass_xbit.h

@@ -8,38 +8,115 @@
 /* - This means those bits in numbers DO get used when we are running as internal sort of thier3 */
 /* - But the numbers high bits (29..32th bits) stay the same inside each bucket so we spare it! */
 /* - If you know your data does not have all the range of 32 bits you can #define these by #define CUSTOM_TPBX_BITS */
+
+/* Only define these to be smaller than these in case your input data shows you can - never change orders or make bigger! */
 #ifndef CUSTOM_TPBX_BITS
 #define TPBX1 10 // top
 #define TPBX2 9 // mid
 #define TPBX3 9 // bottom
 #endif /* CUSTOM_TPBX_BITS */
 
-static inline constexpr uint32_t min3u32_xb(uint32_t a, uint32_t b, uint32_t c) noexcept {
+
+#ifndef TPBX_NO_DEF_WCACHE
+#define TPBX_WCACHE 2 /* depth of the write-cache - I let the user */
+#endif /* TPBX_NO_DEF_WCACHE */
+
+#ifdef __cplusplus
+#define TPBX_NOEXCEPT noexcept
+#define TPBX_CONSTEXPR constexpr
+#else
+#define TPBX_NOEXCEPT
+#define TPBX_CONSTEXPR const
+#endif /* __cplusplus */
+
+static inline TPBX_CONSTEXPR uint32_t min3u32_xb(uint32_t a, uint32_t b, uint32_t c) TPBX_NOEXCEPT {
 	return (a <= b) ?
 		((a <= c) ? a : c) :
 		((b <= c) ? b : c);
 }
 
-/** Copy the elements to their respective radics-place (f->t  copy) */
+static inline TPBX_CONSTEXPR uint32_t max3u32_xb(uint32_t a, uint32_t b, uint32_t c) TPBX_NOEXCEPT {
-static inline void copy_radics_tpxp(uint32_t *f, uint32_t *t, uint32_t *bucket, uint32_t shr, uint32_t mask, uint32_t n) {
+	return (a >= b) ?
+		((a >= c) ? a : c) :
+		((b >= c) ? b : c);
+}
+
+/** Copy the elements to their respective radics-place (f->t  copy) bit should be the TPBX1,2,3 */
+static inline void copy_radics_tpxp(
+		uint32_t *f,
+		uint32_t *t,
+		uint32_t *bucket,
+		uint32_t shr,
+		uint32_t mask,
+		uint32_t n,
+		uint32_t bit) {
+#ifndef TPBX_WCACHE
 	// right-to-left to ensure already sorted digits order we keep for iterations
 	#pragma GCC unroll 48
 	for(uint32_t i = n; i > 0; --i) {
 		// Prefetch caches
 		//__builtin_prefetch(&a[i-8]);
 		// Get num and its new offset / location
-		auto num = f[i - 1];
+		uint32_t num = f[i - 1];
-		auto bkeyni = (num >> shr) & mask;
+		uint32_t bkeyni = (num >> shr) & mask;
-		auto offset = --bucket[bkeyni];
+		uint32_t offset = --bucket[bkeyni];
 
 		// Add to the proper target location
 		t[offset] = num;
 	}
+#else
+	int stride = (1 << bit);
+	static __thread uint32_t write_cache[1024 * TPBX_WCACHE];
+	static __thread uint8_t write_qc[1024];
+
+	for(uint32_t i = 0; i < 1024; ++i) {
+		write_qc[i] = 0;
+	}
+
+	/* right-to-left to ensure already sorted digits order we keep for iterations */
+	#pragma GCC unroll 4
+	for(uint32_t i = n; i > 0; --i) {
+		/* calculate which bucket this would belong and where the queue is */
+		uint32_t num = f[i - 1];
+		uint32_t bkeyni = (num >> shr) & mask;
+		uint8_t qc = write_qc[bkeyni];
+
+		if(qc < TPBX_WCACHE) {
+			/* Cache is not full yet */
+			write_cache[bkeyni + stride * qc] = num;
+			++write_qc[bkeyni];
+		} else {
+			/* Cache is full, make a real writeout - all from this queue */
+
+			/* We also directly move the current element too - micro optimized */
+			uint32_t offset = (bucket[bkeyni] -= (TPBX_WCACHE + 1));
+			t[offset++] = num;
+			/* Copy from the cache - order counts here because keeping opposites! */
+			for(uint32_t j = 0; j < TPBX_WCACHE; ++j) {
+				uint32_t cached = write_cache[bkeyni + stride * (TPBX_WCACHE - 1 - j)];
+				t[offset + j] = cached;
+			}
+
+			/* Reset this queue */
+			write_qc[bkeyni] = 0;
+		}
+	}
+
+	/* Write out any data remaining in the write cache */
+	for(uint32_t bkeyni = 0; bkeyni < stride; ++bkeyni) {
+		uint32_t qc = write_qc[bkeyni];
+		uint32_t offset = (bucket[bkeyni] -= qc);
+		if(qc > 0) for(uint32_t j = 0; j < qc; ++j) {
+			uint32_t cached = write_cache[bkeyni + stride * (qc - 1 - j)];
+			t[offset + j] = cached;
+		}
+	}
+#endif /* TPBX_WCACHE */
 }
 
 /* I pulled these out only for better flame graph support */
 /** Count occurences (can count together with good ILP) */
-static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, uint32_t *bucket3, const uint32_t shr1, const uint32_t shr2, const uint32_t shr3, const uint32_t mask1, const uint32_t mask2, const uint32_t mask3, uint32_t *a, uint32_t n) noexcept {
+static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, uint32_t *bucket3, const uint32_t shr1, const uint32_t shr2, const uint32_t shr3, const uint32_t mask1, const uint32_t mask2, const uint32_t mask3, uint32_t *a, uint32_t n) TPBX_NOEXCEPT {
 	#pragma GCC unroll 64
 	for(uint32_t i = 0; i < n; ++i) {
 		++bucket1[(a[i] >> shr1) & mask1];
@@ -55,23 +132,23 @@ static inline void count_occurences_tpxp(uint32_t *bucket1, uint32_t *bucket2, u
  * @param buf Result array with the same size - result will be here
  * @param n The number of elements
  */
-static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) noexcept {
+static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) TPBX_NOEXCEPT {
 	assert(buf != NULL);
-	constexpr uint32_t shr1 = TPBX3 + TPBX2;
+	TPBX_CONSTEXPR uint32_t shr1 = TPBX3 + TPBX2;
-	constexpr uint32_t shr2 = TPBX3;
+	TPBX_CONSTEXPR uint32_t shr2 = TPBX3;
-	constexpr uint32_t shr3 = 0;
+	TPBX_CONSTEXPR uint32_t shr3 = 0;
-	constexpr uint32_t mask1 = (1 << TPBX1) - 1;
+	TPBX_CONSTEXPR uint32_t mask1 = (1 << TPBX1) - 1;
-	constexpr uint32_t mask2 = (1 << TPBX2) - 1;
+	TPBX_CONSTEXPR uint32_t mask2 = (1 << TPBX2) - 1;
-	constexpr uint32_t mask3 = (1 << TPBX3) - 1;
+	TPBX_CONSTEXPR uint32_t mask3 = (1 << TPBX3) - 1;
 
 	/* helper buffers. */
 	uint32_t sz = n * sizeof(a[0]);
 
-	static thread_local uint32_t bucket1[1 << TPBX1];
+	static __thread uint32_t bucket1[1 << TPBX1];
 	memset(bucket1, 0, (1 << TPBX1) * sizeof(uint32_t));
-	static thread_local uint32_t bucket2[1 << TPBX2];
+	static __thread uint32_t bucket2[1 << TPBX2];
 	memset(bucket2, 0, (1 << TPBX2) * sizeof(uint32_t));
-	static thread_local uint32_t bucket3[1 << TPBX3];
+	static __thread uint32_t bucket3[1 << TPBX3];
 	memset(bucket3, 0, (1 << TPBX3) * sizeof(uint32_t));
 
 	count_occurences_tpxp(bucket1, bucket2, bucket3, shr1, shr2, shr3, mask1, mask2, mask3, a, n);
@@ -112,13 +189,13 @@ static inline void threepass_xb(uint32_t *a, uint32_t *buf, uint32_t n) noexcept
 	}
 
 	// Bottom digit a->buf
-	copy_radics_tpxp(a, buf, bucket3, shr3, mask3, n);
+	copy_radics_tpxp(a, buf, bucket3, shr3, mask3, n, TPBX3);
 
 	// Mid digit buf->a
-	copy_radics_tpxp(buf, a, bucket2, shr2, mask2, n);
+	copy_radics_tpxp(buf, a, bucket2, shr2, mask2, n, TPBX2);
 
 	// Top digit a->buf
-	copy_radics_tpxp(a, buf, bucket1, shr1, mask1, n);
+	copy_radics_tpxp(a, buf, bucket1, shr1, mask1, n, TPBX1);
 }
 
 #endif /* THREE_PASS_XB_H */

ypsu.cpp

@@ -896,8 +896,8 @@ int main(int argc, char **argv) {
   printf("Sorting %d elements:\n\n", n);
 
   // Uncomment this for profiling and alg!
-  measure_single(n);
+  //measure_single(n);
-  return 0;
+  //return 0;
 
   for (auto inputtype : inputtypes) {
     printf("%10s", inputtype.c_str());