diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 187bee54..5d6a1879 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -541,9 +541,10 @@ NVN_API(nvnSyncWait) { const GfxBackend *b = gfx_backend(); uint64_t want = s->fence, lag = s->fence; // Only the game's per-frame wait lags, and not a poll (timeout 0). - int mode = timeout && c->x[30] == frame_wait_lr() ? sync_mode() : 0; + int mode = timeout && c->x[30] == frame_wait_lr() ? sync_mode() : 0, behind = b->fence_done() < want; if (mode == 2) vk_ahead_frame_entry(); - if (mode && b->fence_done() < want) { + // check: every frame as if the GPU were behind (the question is what the game writes, not this machine's timing) + if (mode == 2 || (mode && behind)) { uint64_t p0 = __atomic_load_n(&pa_fence[0], __ATOMIC_ACQUIRE); uint64_t p1 = __atomic_load_n(&pa_fence[1], __ATOMIC_ACQUIRE); if (want <= p0 && want > p1) lag = p1; @@ -560,7 +561,7 @@ NVN_API(nvnSyncWait) { vk_mem_shadow_fence(); // a shadow sync that read the old guard finishes before the game writes again } if (b->fence_done() >= want) { - if (mode == 2) vk_ahead_frame_lag(s->fence, lag); + if (mode == 2) vk_ahead_frame_lag(s->fence, lag, behind); b->cpu_synced(want == s->fence ? s->fence : b->fence_done()); if (mode == 2) vk_ahead_frame_done(1); ret_x(c, NVN_SYNC_WAIT_RESULT_ALREADY_SIGNALED); @@ -571,7 +572,7 @@ NVN_API(nvnSyncWait) { NVN_BLOCKING(); ok = b->fence_wait(want, timeout); } - if (mode == 2) vk_ahead_frame_lag(s->fence, lag); + if (mode == 2) vk_ahead_frame_lag(s->fence, lag, behind); if (ok) b->cpu_synced(want == s->fence ? s->fence : b->fence_done()); if (mode == 2) vk_ahead_frame_done(ok); ret_x(c, ok ? NVN_SYNC_WAIT_RESULT_CONDITION_SATISFIED : NVN_SYNC_WAIT_RESULT_TIMEOUT_EXPIRED); diff --git a/runtime/vk.h b/runtime/vk.h index 9600b528..b9179a1e 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -272,12 +272,15 @@ void vk_mem_pick(VkMemSpan *sp); void vk_ahead_span(const VkMemSpan *sp, uint64_t addr, uint64_t size, int use); #define VK_AHEAD_READ(sp, addr, size, use) \ do { if (__builtin_expect(vk_ahead_on, 0)) vk_ahead_span(&(sp), (addr), (size), (use)); } while (0) -int vk_mem_span_inplace(const VkMemSpan *sp); // the span is guest memory itself (no mirror, no shadow, no device memory) +// 1: the span is guest memory itself (no mirror, no shadow, no device memory); 2: an imported pool's VRAM mirror; else 0 +int vk_mem_span_inplace(const VkMemSpan *sp); // The write watch over the ranges: every write made so far stamped into the page generations; untracked[i] 1 where // range i has no write watch. Returns the newest generation. uint32_t vk_mem_ahead_observe(const uint64_t (*iv)[2], int n, uint8_t *untracked); -// Bytes of [lo, hi) in pages stamped after generation since; *first: the first such byte. -uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, uint64_t *first); +// Bytes of [lo, hi) in pages stamped after generation since; fn (may be NULL) gets each such page's part of the range, +// in address order. +uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, void (*fn)(uint64_t, uint64_t, void *), + void *arg); int vk_mem_pool_info(uint64_t addr, uint64_t *lo, uint64_t *size, uint32_t *flags, int *mirrored); // 0: no pool void vk_mem_gpu_wrote(const VkMemSpan *sp, uint64_t size); // A shader's storage-buffer write: goes back through the write-back merge (vk_memory.c), which keeps the words the GPU diff --git a/runtime/vk_ahead.c b/runtime/vk_ahead.c index d11e7a0b..dc0b24e0 100644 --- a/runtime/vk_ahead.c +++ b/runtime/vk_ahead.c @@ -33,7 +33,7 @@ int vk_ahead_on; static const char *const cls_name[AH_N + 1] = {"vb", "ib", "ubo", "ssbo", "ind", "copy", "msync", "deswz", "wb", "wbm", - "all"}; + "bound", "all"}; #define NOTE_MAX 65536 // notes not yet taken by a frame wait #define SET_MAX 8192 // merged ranges of one frame set, per class and the union together @@ -47,9 +47,10 @@ static const char *const cls_name[AH_N + 1] = {"vb", "ib", "ubo", "ssbo", "ind", #define SMALL_FRAMES 16 #define OFF_N 32 // offender table #define OFF_LOG 24 // first-sighting lines +#define COPY_MAX (32ull << 20) // bytes of a set's union copied at its snap, for the byte compare typedef struct { uint64_t lo, hi, fence; uint8_t cls; } Note; -typedef struct { uint64_t lo, hi; uint8_t cls, untracked; uint32_t hash0, nhash; } SetR; +typedef struct { uint64_t lo, hi, coff; uint8_t cls, untracked; uint32_t hash0, nhash; } SetR; // coff: in copybuf typedef struct { uint64_t lo, hi, fence; uint8_t kind; } GWrite; static PlatMutex ah_lock = PLAT_MUTEX_INIT; // notes, gw, rd, small, lag context @@ -72,19 +73,24 @@ static SetR set[SET_MAX]; static int nset; static uint64_t iv[IV_MAX][2]; static uint8_t iv_untr[IV_MAX]; -static int niv, armed; +static int niv, armed, nclass; // set: [0, nclass) per class by (class, address), then the union by address +static int bound0, bound1; // set[bound0, bound1): the BOUND ranges +static uint8_t *copybuf; static uint32_t set_gen; static uint64_t hashes[HASH_MAX]; static int nhash; typedef struct { - uint64_t frames, lagged, read[AH_N + 1], hit[AH_N + 1], hit_frames[AH_N + 1], hit_max[AH_N + 1]; + // hit: bytes in pages written after the wait; hitb: bytes that changed; mb: MSYNC bytes N also binds (BOUND) + uint64_t frames, lagged, behind, read[AH_N + 1], hit[AH_N + 1], hit_frames[AH_N + 1], hit_max[AH_N + 1]; + uint64_t hitb[AH_N + 1], hitb_frames[AH_N + 1], hitb_max[AH_N + 1], read_mb, hit_mb, hit_mb_frames, hit_mb_max; + uint64_t uncopied; uint64_t inval, inval_stale, inval_stale_unknown, inval_stale_bytes, late[AHW_N], small, small_stale, small_oneoff; uint64_t cost_ns, set_joined, set_lost, untracked, hashed, hash_skipped, ivq; } Stat; static Stat win, tot; // ah_lock for the readback counters, game thread for the rest (summed under ah_lock) -typedef struct { uint64_t pool, size, bytes, frames, first; uint32_t flags; int cls, mirrored; } Offender; +typedef struct { uint64_t pool, size, bytes, changed, frames, first; uint32_t flags; int cls, mirrored; } Offender; static Offender off[OFF_N]; static int noff, nofflog; static uint64_t last_log_ns; @@ -154,7 +160,9 @@ void vk_ahead_read(uint64_t addr, uint64_t size, int cls, uint64_t fence) { } void vk_ahead_span(const VkMemSpan *sp, uint64_t addr, uint64_t size, int use) { - if (!vk_mem_span_inplace(sp)) return; + int k = vk_mem_span_inplace(sp); + if (!k || (k == 2 && use == VK_MUSE_GPU_WRITE)) return; // a mirror's writes go back as WB / WBM + if (k == 2) { vk_ahead_read(addr, size, AH_BOUND, 0); return; } int cls = use == VK_MUSE_VB ? AH_VB : use == VK_MUSE_IB ? AH_IB : use == VK_MUSE_UBO ? AH_UBO : use == VK_MUSE_IND ? AH_IND : use == VK_MUSE_COPY_SRC ? AH_COPY : use == VK_MUSE_GPU_WRITE ? AH_WB : AH_SSBO; vk_ahead_read(addr, size, cls, 0); @@ -271,7 +279,7 @@ static uint64_t hash_pass(SetR *s, int check) { return changed; } -static void offender(int cls, uint64_t addr, uint64_t bytes) { +static void offender(int cls, uint64_t addr, uint64_t bytes, uint64_t changed) { uint64_t lo = 0, size = 0; uint32_t flags = 0; int mirrored = 0; @@ -284,33 +292,88 @@ static void offender(int cls, uint64_t addr, uint64_t bytes) { for (int k = 1; k < noff; k++) if (off[k].bytes < off[m].bytes) m = k; i = m; } else noff++; - off[i] = (Offender){lo, size, 0, 0, addr - lo, flags, cls, mirrored}; + off[i] = (Offender){lo, size, 0, 0, 0, addr - lo, flags, cls, mirrored}; if (nofflog++ < OFF_LOG) rt_log("sync-ahead check: frame %llu: %s range written after the wait: pool %#llx (%.1f MiB, flags %#x, %s) " - "+%#llx, %llu bytes", (unsigned long long)frame_no, cls_name[cls], (unsigned long long)lo, - size / 1048576.0, flags, mirrored ? "mirrored" : "in place", (unsigned long long)(addr - lo), - (unsigned long long)bytes); + "+%#llx, %llu bytes in written pages, %llu changed", (unsigned long long)frame_no, cls_name[cls], + (unsigned long long)lo, size / 1048576.0, flags, mirrored ? "mirrored" : "in place", + (unsigned long long)(addr - lo), (unsigned long long)bytes, (unsigned long long)changed); } - off[i].bytes += bytes, off[i].frames++; + off[i].bytes += bytes, off[i].changed += changed, off[i].frames++; } +// The union range holding a (set[nclass, nset) by address), NULL: none. +static const SetR *union_at(uint64_t a) { + int lo = nclass, hi = nset; + while (lo < hi) { + int m = (lo + hi) / 2; + if (set[m].hi <= a) lo = m + 1; + else hi = m; + } + return lo < nset && set[lo].lo <= a ? &set[lo] : NULL; +} + +// Bytes of [x, y) that differ from the copy taken at the snap (0 where there is none or the memory is gone). +static uint64_t changed_bytes(uint64_t x, uint64_t y) { + const SetR *u = union_at(x); + if (!u || u->coff == UINT64_MAX || y > u->hi || !readable(x, y)) return 0; + const uint8_t *m = (const uint8_t *)(uintptr_t)x, *c = copybuf + u->coff + (x - u->lo); + uint64_t n = y - x, d = 0, i = 0; + for (; i + 8 <= n; i += 8) { + uint64_t a, b; + memcpy(&a, m + i, 8), memcpy(&b, c + i, 8); + if (a == b) continue; + for (int k = 0; k < 8; k++) d += m[i + k] != c[i + k]; + } + for (; i < n; i++) d += m[i] != c[i]; + return d; +} + +typedef struct { int cls; uint64_t first, changed, changed_bound; } Seg; + +// A written page's part [x, y) of a set range: its changed bytes, and for MSYNC those inside a BOUND range too. +static void seg_cb(uint64_t x, uint64_t y, void *arg) { + Seg *g = arg; + if (g->first == UINT64_MAX) g->first = x; + g->changed += changed_bytes(x, y); + if (g->cls != AH_MSYNC) return; + int a = bound0, b = bound1; // the first BOUND range ending after x + while (a < b) { + int m = (a + b) / 2; + if (set[m].hi <= x) a = m + 1; + else b = m; + } + for (int i = a; i < bound1 && set[i].lo < y; i++) { + uint64_t p = set[i].lo > x ? set[i].lo : x, q = set[i].hi < y ? set[i].hi : y; + if (p < q) g->changed_bound += changed_bytes(p, q); + } +} + +static void max_into(uint64_t *m, uint64_t v) { if (v > *m) *m = v; } + // The armed set against the writes since its snap. static void check_set(void) { vk_mem_ahead_observe((const uint64_t(*)[2])iv, niv, iv_untr); win.ivq += (uint64_t)niv; - uint64_t hit[AH_N + 1] = {0}; + uint64_t hit[AH_N + 1] = {0}, hitb[AH_N + 1] = {0}, mb = 0; for (int i = 0; i < nset; i++) { SetR *s = &set[i]; - uint64_t first = s->lo, b = s->untracked ? hash_pass(s, 1) : vk_mem_ahead_written(s->lo, s->hi, set_gen, &first); + if (s->cls == AH_BOUND) continue; + Seg g = {s->cls, UINT64_MAX, 0, 0}; + uint64_t b; + if (s->untracked) b = g.changed = hash_pass(s, 1), g.first = s->lo; + else b = vk_mem_ahead_written(s->lo, s->hi, set_gen, seg_cb, &g); if (!b) continue; - hit[s->cls] += b; - if (s->cls != AH_N) offender(s->cls, first, b); + hit[s->cls] += b, hitb[s->cls] += g.changed, mb += g.changed_bound; + if (s->cls != AH_N) offender(s->cls, g.first, b, g.changed); } for (int c = 0; c <= AH_N; c++) { - win.hit[c] += hit[c]; - if (hit[c]) win.hit_frames[c]++; - if (hit[c] > win.hit_max[c]) win.hit_max[c] = hit[c]; + win.hit[c] += hit[c], win.hitb[c] += hitb[c]; + win.hit_frames[c] += hit[c] != 0, win.hitb_frames[c] += hitb[c] != 0; + max_into(&win.hit_max[c], hit[c]), max_into(&win.hitb_max[c], hitb[c]); } + win.hit_mb += mb, win.hit_mb_frames += mb != 0; + max_into(&win.hit_mb_max, mb); armed = 0; } @@ -331,19 +394,36 @@ static void build_set(int n) { } else win.set_lost += take[i].hi - take[i].lo; continue; } - set[nset++] = (SetR){take[i].lo, take[i].hi, take[i].cls, 0, 0, 0}; + set[nset++] = (SetR){take[i].lo, take[i].hi, UINT64_MAX, take[i].cls, 0, 0, 0}; } int nc = nset; - for (int i = 0; i < nc; i++) win.read[set[i].cls] += set[i].hi - set[i].lo; - // The union over all classes, after them. - SetR *u = set + nc; - memcpy(u, set, sizeof *set * (size_t)nc); - qsort(u, (size_t)nc, sizeof *u, lo_cmp); - int nu = 0; + nclass = nc, bound0 = bound1 = 0; for (int i = 0; i < nc; i++) { + win.read[set[i].cls] += set[i].hi - set[i].lo; + if (set[i].cls == AH_BOUND) bound1 = i + 1; + else if (set[i].cls < AH_BOUND) bound0 = i + 1; + } + if (bound1 < bound0) bound1 = bound0; + // MSYNC bytes N also binds: both lists are sorted by address. + for (int i = 0, j = bound0; i < nc && j < bound1; i++) { + if (set[i].cls != AH_MSYNC) continue; + while (j < bound1 && set[j].hi <= set[i].lo) j++; + for (int k = j; k < bound1 && set[k].lo < set[i].hi; k++) { + uint64_t p = set[k].lo > set[i].lo ? set[k].lo : set[i].lo, q = set[k].hi < set[i].hi ? set[k].hi : set[i].hi; + if (p < q) win.read_mb += q - p; + } + } + // The union over the classes that race (not BOUND), after them. + SetR *u = set + nc; + int nr = 0; + for (int i = 0; i < nc; i++) + if (set[i].cls != AH_BOUND) u[nr++] = set[i]; + qsort(u, (size_t)nr, sizeof *u, lo_cmp); + int nu = 0; + for (int i = 0; i < nr; i++) { if (nu && u[i].lo <= u[nu - 1].hi) { if (u[i].hi > u[nu - 1].hi) u[nu - 1].hi = u[i].hi; - } else u[nu++] = (SetR){u[i].lo, u[i].hi, AH_N, 0, 0, 0}; + } else u[nu++] = (SetR){u[i].lo, u[i].hi, UINT64_MAX, AH_N, 0, 0, 0}; } nset = nc + nu; for (int i = 0; i < nu; i++) win.read[AH_N] += u[i].hi - u[i].lo; @@ -377,8 +457,19 @@ static void snap_set(void) { set_gen = vk_mem_ahead_observe((const uint64_t(*)[2])iv, niv, iv_untr); win.ivq += (uint64_t)niv; nhash = 0; + if (!copybuf && !(copybuf = malloc(COPY_MAX))) rt_fatal("vk: out of memory (sync-ahead check)"); + uint64_t used = 0; + for (int i = nclass; i < nset; i++) { + SetR *s = &set[i]; + uint64_t n = s->hi - s->lo; + if (used + n <= COPY_MAX && readable(s->lo, s->hi)) { + memcpy(copybuf + used, (const void *)(uintptr_t)s->lo, n); + s->coff = used, used += n; + } else win.uncopied += n; + } for (int i = 0; i < nset; i++) { SetR *s = &set[i]; + if (s->cls == AH_BOUND) continue; s->untracked = (uint8_t)untracked_at(s->lo); if (!s->untracked) continue; win.untracked += s->cls == AH_N ? s->hi - s->lo : 0; @@ -391,15 +482,20 @@ static void log_line(const Stat *s, const char *what) { char b[2048]; // about 1,200 bytes with every class int k = 0; double f = s->frames ? (double)s->frames : 1; - k += snprintf(b + k, sizeof b - (size_t)k, "sync-ahead check: %s %llu frames (%llu lagged): written after the wait " - "KiB/frame", what, (unsigned long long)s->frames, (unsigned long long)s->lagged); + k += snprintf(b + k, sizeof b - (size_t)k, "sync-ahead check: %s %llu frames (%llu lagged, the GPU " + "behind in %llu): written after the wait KiB/frame", what, (unsigned long long)s->frames, + (unsigned long long)s->lagged, (unsigned long long)s->behind); for (int c = 0; c <= AH_N; c++) if (s->hit[c] || c == AH_N) - k += snprintf(b + k, sizeof b - (size_t)k, " %s %.1f (in %llu frames, max %.0f)", cls_name[c], - s->hit[c] / 1024.0 / f, (unsigned long long)s->hit_frames[c], s->hit_max[c] / 1024.0); + k += snprintf(b + k, sizeof b - (size_t)k, " %s %.1f (in %llu frames, max %.0f; changed %.1f in %llu, max %.0f)", + cls_name[c], s->hit[c] / 1024.0 / f, (unsigned long long)s->hit_frames[c], s->hit_max[c] / 1024.0, + s->hitb[c] / 1024.0 / f, (unsigned long long)s->hitb_frames[c], s->hitb_max[c] / 1024.0); + k += snprintf(b + k, sizeof b - (size_t)k, " msync&bound changed %.1f (in %llu, max %.0f)", s->hit_mb / 1024.0 / f, + (unsigned long long)s->hit_mb_frames, s->hit_mb_max / 1024.0); k += snprintf(b + k, sizeof b - (size_t)k, "; read KiB/frame"); for (int c = 0; c <= AH_N; c++) if (s->read[c] || c == AH_N) k += snprintf(b + k, sizeof b - (size_t)k, " %s %.1f", cls_name[c], s->read[c] / 1024.0 / f); + k += snprintf(b + k, sizeof b - (size_t)k, " msync&bound %.1f", s->read_mb / 1024.0 / f); k += snprintf(b + k, sizeof b - (size_t)k, "; stale: invalidates %llu of %llu (%llu not read lately, %.1f KiB), late reports counter %llu samples " "%llu event %llu, writes %llu, small write-backs %llu of %llu (%llu one-off)", @@ -408,10 +504,10 @@ static void log_line(const Stat *s, const char *what) { (unsigned long long)s->late[AHW_EVENT], (unsigned long long)s->late[AHW_WRITE], (unsigned long long)s->small_stale, (unsigned long long)s->small, (unsigned long long)s->small_oneoff); snprintf(b + k, sizeof b - (size_t)k, "; game-thread cost %.3f ms/frame, %.1f queries/frame, untracked %.1f " - "KiB/frame (hashed %.1f, skipped %.1f), set over its size: %.1f KiB joined, %.1f KiB left out; note compactions %llu, gpu-write list " - "overflow %llu", s->cost_ns / 1e6 / f, s->ivq / f, s->untracked / 1024.0 / f, s->hashed / 1024.0 / f, - s->hash_skipped / 1024.0 / f, s->set_joined / 1024.0, s->set_lost / 1024.0, (unsigned long long)n_compact, - (unsigned long long)gw_over); + "KiB/frame (hashed %.1f, skipped %.1f), not copied %.1f KiB, set over its size: %.1f KiB joined, %.1f KiB " + "left out; note compactions %llu, gpu-write list overflow %llu", s->cost_ns / 1e6 / f, s->ivq / f, + s->untracked / 1024.0 / f, s->hashed / 1024.0 / f, s->hash_skipped / 1024.0 / f, s->uncopied / 1024.0, + s->set_joined / 1024.0, s->set_lost / 1024.0, (unsigned long long)n_compact, (unsigned long long)gw_over); rt_log("%s", b); } @@ -420,9 +516,10 @@ static void stat_add(Stat *d, const Stat *s) { uint64_t *y = (uint64_t *)d; for (size_t i = 0; i < sizeof *s / sizeof(uint64_t); i++) y[i] += x[i]; for (int c = 0; c <= AH_N; c++) { // maxima, not sums - d->hit_max[c] -= s->hit_max[c]; - if (s->hit_max[c] > d->hit_max[c]) d->hit_max[c] = s->hit_max[c]; + d->hit_max[c] -= s->hit_max[c], max_into(&d->hit_max[c], s->hit_max[c]); + d->hitb_max[c] -= s->hitb_max[c], max_into(&d->hitb_max[c], s->hitb_max[c]); } + d->hit_mb_max -= s->hit_mb_max, max_into(&d->hit_mb_max, s->hit_mb_max); } void vk_ahead_frame_entry(void) { @@ -432,8 +529,9 @@ void vk_ahead_frame_entry(void) { win.cost_ns += plat_now_ns() - t0; } -void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged) { +void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged, int behind) { if (!vk_ahead_on) return; + if (behind && lagged < asked) win.behind++; plat_mutex_lock(&ah_lock); frame_no++; cur_asked = asked, cur_lagged = lagged, lag_live = 1; @@ -496,9 +594,10 @@ void vk_ahead_log_final(void) { log_line(&tot, "run:"); qsort(off, (size_t)noff, sizeof *off, off_cmp); for (int i = 0; i < noff && i < 12; i++) - rt_log("sync-ahead check: offender %d: %s pool %#llx (%.1f MiB, flags %#x, %s) first at +%#llx: %.1f KiB in %llu " - "frames", i + 1, cls_name[off[i].cls], (unsigned long long)off[i].pool, off[i].size / 1048576.0, off[i].flags, - off[i].mirrored ? "mirrored" : "in place", (unsigned long long)off[i].first, off[i].bytes / 1024.0, + rt_log("sync-ahead check: offender %d: %s pool %#llx (%.1f MiB, flags %#x, %s) first at +%#llx: %.1f KiB in written " + "pages, %.1f KiB changed, in %llu frames", i + 1, cls_name[off[i].cls], (unsigned long long)off[i].pool, + off[i].size / 1048576.0, off[i].flags, off[i].mirrored ? "mirrored" : "in place", + (unsigned long long)off[i].first, off[i].bytes / 1024.0, off[i].changed / 1024.0, (unsigned long long)off[i].frames); } @@ -527,9 +626,10 @@ int vk_ahead_selftest(uint64_t a, uint64_t size) { vk_ahead_read(a + 0x10000, 0x8000, AH_MSYNC, f); vk_ahead_read(a + 0x20000, 0x1000, AH_VB, f); vk_ahead_read(a + 0x28000, 0x1000, AH_VB, f + 1); // a later frame's: stays in the list + vk_ahead_read(a + 0x12300, 0x100, AH_BOUND, f); // N binds 256 bytes of the synced range vk_ahead_gpu_write(a + 0x30000, 16, AHW_COUNTER, f); m[0x10010] = (uint8_t)(m[0x10010] + 1); // before the wait: the frame wrote it before the GPU read it - vk_ahead_frame_lag(f, f - 1); + vk_ahead_frame_lag(f, f - 1, 1); vk_ahead_frame_done(1); if (!armed || win.read[AH_MSYNC] != 0x8000 || win.read[AH_VB] != 0x1000 || win.read[AH_N] != 0x9000) bad = 2; else if (win.late[AHW_COUNTER] != 1) bad = 3; @@ -539,11 +639,12 @@ int vk_ahead_selftest(uint64_t a, uint64_t size) { m[0x3a000] = (uint8_t)(m[0x3a000] + 1); // after the wait, outside the set vk_ahead_frame_entry(); if (!bad && (armed || win.hit[AH_MSYNC] != 0x1000 || win.hit[AH_VB] || win.hit[AH_N] != 0x1000)) bad = 5; + if (!bad && (win.hitb[AH_MSYNC] != 1 || win.hitb[AH_N] != 1 || win.hit_mb != 1 || win.read_mb != 0x100)) bad = 9; plat_mutex_lock(&ah_lock); int left = nnotes; plat_mutex_unlock(&ah_lock); if (!bad && left != 1) bad = 6; - vk_ahead_frame_lag(f + 1, f + 1); // not lagged: nothing at risk + vk_ahead_frame_lag(f + 1, f + 1, 0); // not lagged: nothing at risk vk_ahead_frame_done(1); if (!bad && win.read[AH_N] != 0x9000) bad = 7; m[0x28010] = (uint8_t)(m[0x28010] + 1); diff --git a/runtime/vk_ahead.h b/runtime/vk_ahead.h index 48149cb0..8d4a1b9a 100644 --- a/runtime/vk_ahead.h +++ b/runtime/vk_ahead.h @@ -5,8 +5,9 @@ // Classes of GPU-timeline accesses of guest memory a submission makes after its recording: in-place bindings of an // imported pool without a mirror (vertex, index, uniform, storage, indirect, copy source), the VRAM mirror's sync copies, -// the GPU deswizzle of a texture's storage, and the mirror write-backs (plain copies, word merges). -enum { AH_VB, AH_IB, AH_UBO, AH_SSBO, AH_IND, AH_COPY, AH_MSYNC, AH_DESWZ, AH_WB, AH_WBM, AH_N }; +// the GPU deswizzle of a texture's storage, and the mirror write-backs (plain copies, word merges). BOUND: a binding of +// a mirrored pool, the bytes the GPU reads from the mirror; not a race of its own, it narrows MSYNC to what N reads. +enum { AH_VB, AH_IB, AH_UBO, AH_SSBO, AH_IND, AH_COPY, AH_MSYNC, AH_DESWZ, AH_WB, AH_WBM, AH_BOUND, AH_N }; // Kinds of GPU results the CPU may read back: buffer writes, counter reports, samples-passed reports, event signals. enum { AHW_WRITE, AHW_COUNTER, AHW_SAMPLES, AHW_EVENT, AHW_N }; @@ -21,10 +22,11 @@ void vk_ahead_gpu_write(uint64_t addr, uint64_t size, int kind, uint64_t fence); void vk_ahead_invalidate(uint64_t addr, uint64_t size); // nvnMemoryPoolInvalidateMappedRange void vk_ahead_small(uint64_t addr, uint64_t gpu_fence); // the fence write-back of a small target (vk_texture.c) // The game's per-frame wait (nvn_api_core.c nvnSyncWait at frame_wait_lr): entry checks the previous frame's read set; -// lag before cpu_synced gives the value a one-frame-late wait would have waited for (lagged == asked: no lag); done after -// it records this frame's read set. ok 0: the wait timed out. +// lag before cpu_synced gives the value a one-frame-late wait would have waited for (lagged == asked: no lag), counted +// whether or not the GPU had finished asked (behind: it had not, the late wait would have returned early on this +// machine); done after it records this frame's read set. ok 0: the wait timed out. void vk_ahead_frame_entry(void); -void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged); +void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged, int behind); void vk_ahead_frame_done(int ok); void vk_ahead_log_final(void); // at exit int vk_ahead_selftest(uint64_t addr, uint64_t size); // --probe-vkmem: 0 passed diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index cfc10d78..1e17ee7c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -985,7 +985,8 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // ---------------------------------------------------------------- TOTK_SYNC_AHEAD=check (vk_ahead.c) int vk_mem_span_inplace(const VkMemSpan *sp) { const PoolRec *r = sp->pool; - return r && !sp->alloc && r->kind == POOL_IMPORT && !r->smap && sp->buffer == r->buf; + if (!r || sp->alloc || r->kind != POOL_IMPORT || r->smap) return 0; + return sp->buffer == r->buf ? 1 : r->mbuf && sp->buffer == r->mbuf ? 2 : 0; } uint32_t vk_mem_ahead_observe(const uint64_t (*iv)[2], int n, uint8_t *untracked) { @@ -1002,7 +1003,8 @@ uint32_t vk_mem_ahead_observe(const uint64_t (*iv)[2], int n, uint8_t *untracked return g; } -uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, uint64_t *first) { +uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, void (*fn)(uint64_t, uint64_t, void *), + void *arg) { uint64_t bytes = 0; if (hi <= lo || hi > RT_ADDRESS_LIMIT) return 0; plat_mutex_lock_shared(&gen_lock); @@ -1015,7 +1017,7 @@ uint64_t vk_mem_ahead_written(uint64_t lo, uint64_t hi, uint32_t since, uint64_t for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) { if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] <= since) continue; uint64_t s = x > lo ? x : lo, e = x + (1ull << GEN_PAGE_SHIFT) < hi ? x + (1ull << GEN_PAGE_SHIFT) : hi; - if (!bytes && first) *first = s; + if (fn) fn(s, e, arg); bytes += e - s; } } diff --git a/tests/sync_ahead_check.py b/tests/sync_ahead_check.py index a8499799..192e6b56 100644 --- a/tests/sync_ahead_check.py +++ b/tests/sync_ahead_check.py @@ -42,10 +42,11 @@ static const char *cfg_value(const char *name) { } static int entry_calls, lag_calls, done_calls, done_ok; static uint64_t lag_asked, lag_value; +static int lag_behind; static void vk_ahead_frame_entry(void) { assert(!lag_calls && !done_calls); entry_calls++; } -static void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged) { +static void vk_ahead_frame_lag(uint64_t asked, uint64_t lagged, int behind) { assert(entry_calls == 1 && !sync_calls); // before cpu_synced: the small write-backs see the lag - lag_calls++, lag_asked = asked, lag_value = lagged; + lag_calls++, lag_asked = asked, lag_value = lagged, lag_behind = behind; } static void vk_ahead_frame_done(int ok) { assert(lag_calls == 1); done_calls++, done_ok = ok; } static int vk_mem_shadow_on(void) { return shadow; } @@ -93,6 +94,7 @@ int main(int argc, char **argv) { int lagcase = sync.fenced && timeout && lr == site && completed < target && target > 20 && target <= 40; relaxed = on && lagcase; int checked = check && sync.fenced && timeout && lr == site; + int checkcase = checked && target > 20 && target <= 40; // check: as if the GPU were behind relaxed_cases += relaxed; polls += timeout == 0; other_waits += lr != site; if (relaxed) target = 20; replay_calls = shadow_calls = sync_calls = wait_calls = 0; @@ -104,7 +106,8 @@ int main(int argc, char **argv) { assert(replay_calls == relaxed && shadow_calls == relaxed); assert(entry_calls == checked && lag_calls == checked && done_calls == checked); if (checked) { // exact wait, the late one's value told to the check - assert(lag_asked == sync.fence && lag_value == (lagcase ? 20 : sync.fence)); + assert(lag_asked == sync.fence && lag_value == (checkcase ? 20 : sync.fence)); + assert(lag_behind == (initial_done < sync.fence)); assert(done_ok == (initial_done >= target || succeed)); } assert(pa_guard == (relaxed ? sync.fence : 0));