--- a/aggregate.c +++ b/aggregate.c @@ -1,18 +1,33 @@ /* - * EXERCISE 05 — BROKEN STARTING POINT. Do not edit this file; copy it. + * EXERCISE 05 — FIXED VARIANT. Three changes, all aimed at the same thing: + * hold the lock for less time, and take it far less often. * - * ============================ UNSAFE CODE WARNING ============================ - * Nothing here is memory-unsafe and nothing deadlocks. The defect is a - * PERFORMANCE one, and it is the kind that gets worse the more hardware you - * throw at it. Never ship a critical section shaped like the one below. - * ============================================================================= - * * THE SCENARIO * A metrics aggregator digests events. Each event is scored and the score is - * folded into a running total. One mutex guards the total. The team's fix for - * "the aggregator is too slow" was to raise the worker count from 4 to 16. - * It got slower. + * folded into a running total. * + * THE REPAIR + * 1. The scoring moves OUT of the critical section. Nothing about + * score_event() needs the lock: it reads no shared state and writes none. + * Only the fold into the total does. + * 2. Each worker accumulates into a LOCAL total and folds it in once per + * BATCH events. That divides the acquisition count by BATCH — from one per + * event to one per 64 events — without changing the answer, because + * addition is associative. + * 3. The single hot total becomes SHARDS independent totals, each on its own + * cache line, summed once at the end. Contention is divided rather than + * removed. + * + * The padding in shard_t is not decoration. Without it the shards share a + * cache line, every fold invalidates every other shard, and the measurement + * becomes one of false sharing rather than lock contention — a classic way to + * "prove" that sharding does not help. + * + * Note what sharding did NOT solve: it works here because each worker owns a + * shard and the only shared invariant is a sum, which is associative. Sharding + * a structure with an invariant ACROSS shards buys you nothing, because you + * would have to hold every shard's lock to preserve it. + * * WHAT THIS PROGRAM MEASURES * A FIXED amount of total work — TOTAL_OPS events, whatever the thread count * — split across 1, 2, 4, 8 and 16 threads. Perfect scaling would keep wall @@ -23,11 +38,12 @@ * The headline number is the 16-thread time divided by the 1-thread time. * Anything above 1.0 means the lock, not the work, is the bottleneck. * - * Build: clang -O2 -g -Wall -Wextra aggregate.c -o aggregate_broken - * Run: ./aggregate_broken [total_ops] (default 1000000) + * Build: clang -O2 -g -Wall -Wextra aggregate.c -o aggregate_fixed + * Run: ./aggregate_fixed [total_ops] (default 1000000) * - * Expected: a slowdown well above 1.0 at 16 threads, and an acquisitions count - * equal to the op count — one lock acquisition per event. Exit status 0. + * Expected: wall time that FALLS as threads are added, a ratio well below 1.0, + * and an acquisition count roughly totalOps/64 instead of totalOps. + * Exit status 0. * * The total is deterministic and does not depend on the thread count, so the * broken and fixed builds must print the SAME checksum. A "speed-up" that @@ -45,6 +61,8 @@ #define MAX_THREADS 16 #define REPEATS 5 #define SCORE_ROUNDS 48 +#define SHARDS 16 +#define BATCH 64 static void *watchdog(void *arg) { sleep((unsigned)(long)arg); @@ -75,29 +93,44 @@ return h; } -/* ------------------------------- shared state ------------------------------ */ +/* ------------------------------- shared state ------------------------------ + * One shard per worker, each padded to its own cache line so that folding into + * one shard does not invalidate the line holding another. + * -------------------------------------------------------------------------- */ -static pthread_mutex_t total_lock = PTHREAD_MUTEX_INITIALIZER; -static unsigned long running_total; -static unsigned long acquisitions; +typedef struct { + pthread_mutex_t m; + unsigned long total; + unsigned long acquisitions; + char pad[128 - sizeof(pthread_mutex_t) - 2 * sizeof(unsigned long)]; +} shard_t; +static shard_t shards[SHARDS]; + typedef struct { int id; unsigned long first, count; } work_t; /* THE CRITICAL SECTION * - * The lock is taken once per event, and the scoring happens while it is held. - * Every thread therefore spends its time queued behind whichever thread is - * currently scoring — the lock is held for far longer than it needs to be, and - * it is acquired far more often than it needs to be. */ + * It now contains one addition and one increment, and it is entered once per + * BATCH events rather than once per event. */ +static void fold(int id, unsigned long *local) { + shard_t *sh = &shards[id % SHARDS]; + pthread_mutex_lock(&sh->m); + sh->total += *local; + sh->acquisitions++; + pthread_mutex_unlock(&sh->m); + *local = 0; +} + static void *worker(void *arg) { work_t *w = (work_t *)arg; + unsigned long local = 0; for (unsigned long i = 0; i < w->count; i++) { - pthread_mutex_lock(&total_lock); - unsigned long s = score_event(w->first + i); - running_total += s & 0xFFUL; - acquisitions++; - pthread_mutex_unlock(&total_lock); + unsigned long s = score_event(w->first + i); /* OUTSIDE the lock */ + local += s & 0xFFUL; + if ((i % BATCH) == BATCH - 1) fold(w->id, &local); } + fold(w->id, &local); /* the remainder */ return NULL; } @@ -109,9 +142,11 @@ work_t w[MAX_THREADS]; unsigned long per = total_ops / (unsigned long)threads; - total_lock = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; - running_total = 0; - acquisitions = 0; + for (int i = 0; i < SHARDS; i++) { + pthread_mutex_init(&shards[i].m, NULL); + shards[i].total = 0; + shards[i].acquisitions = 0; + } double t0 = now_s(); for (int i = 0; i < threads; i++) { @@ -123,8 +158,10 @@ for (int i = 0; i < threads; i++) pthread_join(t[i], NULL); double elapsed = now_s() - t0; - *out_total = running_total; - *out_acq = acquisitions; + unsigned long total = 0, acq = 0; + for (int i = 0; i < SHARDS; i++) { total += shards[i].total; acq += shards[i].acquisitions; } + *out_total = total; + *out_acq = acq; return elapsed; } @@ -154,7 +191,7 @@ const int counts[] = { 1, 2, 4, 8, 16 }; const int ncounts = (int)(sizeof(counts) / sizeof(counts[0])); - printf("EX05 build=broken totalOps=%lu (fixed total work, split across N threads)\n", total); + printf("EX05 build=fixed totalOps=%lu (fixed total work, split across N threads)\n", total); printf(" each cell is the median of %d runs; perfect scaling would keep it FLAT\n\n", REPEATS); printf(" %-10s", "threads"); for (int i = 0; i < ncounts; i++) printf("%10d", counts[i]); @@ -180,7 +217,7 @@ } printf("\n\n"); - printf("EX05 build=broken totalOps=%lu checksum=%lu acquisitions=%lu " + printf("EX05 build=fixed totalOps=%lu checksum=%lu acquisitions=%lu " "oneThreadMs=%.0f sixteenThreadMs=%.0f slowdown=%.2f " "worstSlowdown=%.2f worstAtThreads=%d\n", total, checksum, one_acq, one * 1000.0, sixteen * 1000.0,