diff --git a/gpu/vk/vk_coopmat.cpp b/gpu/vk/vk_coopmat.cpp index c649a336e..0b8df77f7 100644 --- a/gpu/vk/vk_coopmat.cpp +++ b/gpu/vk/vk_coopmat.cpp @@ -18,6 +18,7 @@ #include "vk_internal.h" #include "shader_recompiler/backend/spirv/mma_layout.h" #include +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { namespace { diff --git a/gpu/vk/vk_draw.cpp b/gpu/vk/vk_draw.cpp index 73d6550e6..db934c9ce 100644 --- a/gpu/vk/vk_draw.cpp +++ b/gpu/vk/vk_draw.cpp @@ -41,6 +41,7 @@ #include "shaders/taau_upscale_comp_spv.h" #include "shaders/movie_upscale_comp_spv.h" #include "shader_recompiler/backend/spirv/mma_layout.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { // (EvilOOT, RX_VK_DRAW_STATS: the pointer table's regions, vk_bank.cpp) @@ -49,6 +50,67 @@ const PointerEntry* pointer_table(const NvnState& s, int stage); uint32_t stage_cbuf_word(const NvnState& s, const Stage& st, uint32_t index, uint32_t offset); // (vk_shader.cpp) namespace { +// ---- (hookshot) RX_VK_PROF_FINE=1: where a draw's time goes, finer than the frame line's phases, and what each draw does ---- +// Off, each site is a test of one constant. On, a timed site reads the clock twice (~15 ns; the frame line's phases grow by the same, so +// compare a run with it on to another with it on) and the sums are logged with the frame statistics, "a frame (fine)": the times in ms a +// frame, the counts a draw. FT_*: the sections of the descriptor build (banks, c1, user uniform buffers, storage buffers, textures) and +// of the commands (hazards and the pass, pipeline bind, descriptor push, push constants, dynamic state, vertex and index binds, the draw) +enum FineT { FT_BANK, FT_C1, FT_UBO, FT_STORAGE, FT_TEX, FT_HAZARD, FT_BINDPL, FT_BINDDESC, FT_PUSHCALL, FT_PUSHC, FT_DYN, FT_VERTEX, + FT_DRAWCALL, FT_COUNT }; +const char* const kFineT[] = {"banks", "c1", "user uniform buffers", "storage buffers", "textures", "hazards+pass", "bind pipeline", + "bind descriptors", "of it the push call", "push constants", "dynamic state", "vertex+index binds", + "draw call"}; +enum FineC { FC_DRAWS, FC_BUILDS, FC_DESCS, FC_WRITES, FC_BANK_CALLS, FC_BANK_GEN, FC_BANK_FILLS, FC_BANK_FILL_BYTES, FC_BANK_SAME, + FC_BANK_NEW, FC_BANK_NEW_BYTES, FC_C1_CALLS, FC_UBO_CALLS, FC_UBO_OVERLAY, FC_UBO_STREAM, FC_PUSH_FULL, FC_PUSH_DELTA, + FC_PUSH_NONE, FC_PUSH_WRITES, FC_PUSH_DESCS, FC_BIND_PIPE, FC_DYN_FAST, FC_DYN_FULL, FC_DYN_CALLS, FC_VBIND, FC_IBIND, + FC_PUSHC, FC_PIPE_FAST, FC_PIPE_MAP, FC_POOL_SETS, FC_POOL_DESCS, FC_SSBO_DESCS, FC_COUNT }; +const char* const kFineC[] = {"draws", "descriptor builds", "descriptors", "descriptor writes", "bank calls", + "bank kept by its stage's generation", "bank fills", "bank bytes filled", "bank refills equal", + "bank copies into the stream", "bank bytes copied", "c1 calls", "user uniform buffer calls", + "uniform ranges copied with an update's overlay", "uniform ranges copied into the stream", + "full pushes", "pushes of changed writes only", "pushes of nothing (all as pushed)", + "writes pushed", "descriptors pushed", "pipeline binds", "dynamic state from the draw before", + "dynamic state worked out", "dynamic state commands", "vertex stream binds", "index buffer binds", + "push constants", "pipeline keys from the fast pair", "pipeline keys from the thread's map", + "sets allocated (over the push limit)", "descriptors in them", "storage buffer descriptors built"}; +static_assert(std::size(kFineT) == FT_COUNT && std::size(kFineC) == FC_COUNT, "every section and count is named"); +const bool g_fine = getenv("RX_VK_PROF_FINE") != nullptr; +int64_t fine_ns[FT_COUNT], fine_n[FC_COUNT]; +struct FineScope { + int ph; + int64_t t0; + explicit FineScope(int p) : ph(p), t0(g_fine ? prof_now() : 0) {} + ~FineScope() { if (g_fine) fine_ns[ph] += prof_now() - t0; } + FineScope(const FineScope&) = delete; + FineScope& operator=(const FineScope&) = delete; +}; +#define FINE_SCOPE(ph) FineScope fine_scope_##ph(ph) +#define FINE_N(c, n) do { if (g_fine) fine_n[c] += (n); } while (0) +#define FINE_X(c, n) (g_fine ? (void)(fine_n[c] += (n)) : (void)0) // (the same, as an expression) +/// The sums since the last report, as a frame's times and a draw's counts (the caller: every 120 frames) +void fine_report(double frames) { + if (!g_fine) return; + static int64_t last_ns[FT_COUNT], last_n[FC_COUNT]; + char line[900]; + int k = snprintf(line, sizeof line, "vk: a frame (fine): ms:"); + for (int i = 0; i < FT_COUNT; i++) { + k += snprintf(line + k, sizeof line - k, " %s %.2f%s", kFineT[i], (fine_ns[i] - last_ns[i]) / frames / 1e6, i + 1 < FT_COUNT ? "," : ""); + last_ns[i] = fine_ns[i]; + } + logf("%s", line); + const double draws = std::max(1.0, (double)(fine_n[FC_DRAWS] - last_n[FC_DRAWS])); + k = snprintf(line, sizeof line, "vk: a frame (fine): %.0f draws; a draw:", draws / frames); + for (int i = 1; i < FC_COUNT; i++) { + k += snprintf(line + k, sizeof line - k, " %s %.2f%s", kFineC[i], (fine_n[i] - last_n[i]) / draws, i + 1 < FC_COUNT ? "," : ""); + if (k > (int)sizeof line - 80) { + logf("%s", line); + k = snprintf(line, sizeof line, "vk: a frame (fine): a draw:"); + } + } + logf("%s", line); + for (int i = 0; i < FC_COUNT; i++) last_n[i] = fine_n[i]; +} + bool no_a2c() { // (RX_VK_NO_A2C, read once: it was a getenv on every alpha-to-coverage draw, the foliage's) static const bool v = getenv("RX_VK_NO_A2C") != nullptr; return v; @@ -267,23 +329,43 @@ thread_local uint64_t rec_value = 0; struct RecordingRes { VkDescriptorPool pool = VK_NULL_HANDLE; std::vector extra; // (pools added when the first ran out) + // (the stream in use: the first, or one a recording that outgrew it was moved to. Descriptors written before the move hold + // the old buffer's range, so the old ones stay until the slot's next recording, which the GPU's use of this one is done by) VkBuffer stream = VK_NULL_HANDLE; VmaAllocation stream_alloc = nullptr; uint8_t* mapped = nullptr; - uint64_t used = 0; - std::unordered_map c1_at; // (c1 of a stage: uploaded once a recording) + uint64_t used = 0, size = 0; + struct Stream { VkBuffer buffer; VmaAllocation alloc; uint8_t* mapped; uint64_t size; }; + std::vector streams; // (the first, then the ones made on demand: kept for the slot's next recordings) + size_t si = 0; // (the one in use) + uint64_t before = 0; // (what the recording used of the streams before this one) + uint32_t idle = 0; // (recordings of the slot that stayed in the first stream) + std::unordered_map> c1_at; // (c1 of a stage: uploaded once a recording) }; -constexpr uint64_t kStream = 24u << 20; // (a recording's: 24 of them, vk_replay.cpp kRecordings) +// (a recording's first stream: 24 of them, vk_replay.cpp recording_slots. RX_VK_STREAM_MB=: that size, 2 by default; a recording +// that needs more is given a bigger stream for the rest of it (RX_VK_STREAM_GROW_OFF=1: draws are skipped instead, and the default +// is 24 MB as before); the most a recording used is logged: "the recording streams") +bool stream_grow_off() { + static const bool off = getenv("RX_VK_STREAM_GROW_OFF") != nullptr; + return off; +} +const uint64_t kStream = (getenv("RX_VK_STREAM_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_STREAM_MB"))) : stream_grow_off() ? 24u : 2u) << 20; +std::atomic stream_peak{0}, stream_extra{0}, stream_grown{0}; std::deque res; // (a deque: a recording thread keeps its own in place while another is added) thread_local RecordingRes* cur = nullptr; VkDescriptorPool make_pool() { + // (each recording's pool: 8192 sets of up to 32768 descriptors a type, for the programs with more descriptors than the push limit, + // 24 or 48 pools at once, and the driver may keep host memory for all of that. A pool that runs out is followed by another of the same + // size (cur->extra), so a smaller one is safe, and costs a pool made and destroyed per recording in a scene that needs more. + // RX_VK_DESC_POOL_DIV= (1..32): a n-th of both) + static const uint32_t div = getenv("RX_VK_DESC_POOL_DIV") ? (uint32_t)std::clamp(atoi(getenv("RX_VK_DESC_POOL_DIV")), 1, 32) : 1; const VkDescriptorPoolSize sizes[] = { - {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 32768}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 16384}, - {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 32768}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4096}, - {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 2048}, {VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 2048}}; + {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 32768 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 16384 / div}, + {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 32768 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4096 / div}, + {VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 2048 / div}, {VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 2048 / div}}; VkDescriptorPoolCreateInfo ci{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - ci.maxSets = 8192; + ci.maxSets = 8192 / div; ci.poolSizeCount = (uint32_t)std::size(sizes); ci.pPoolSizes = sizes; VkDescriptorPool p = VK_NULL_HANDLE; @@ -291,14 +373,45 @@ VkDescriptorPool make_pool() { return p; } -/// Space in the recording's stream (UINT64_MAX when full) +constexpr VkBufferUsageFlags kStreamUse = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; +/// The recording goes on in its next stream (made when the slot has none yet, twice the last one's size up to 32 MB, or the size asked) +bool stream_next(uint64_t need) { + RecordingRes& r = *cur; + if (r.si + 1 < r.streams.size() && r.streams[r.si + 1].size >= need) { + r.si++; + } else { + if (stream_grow_off()) return false; + RecordingRes::Stream s{}; + s.size = std::max((need + (1u << 20) - 1) & ~((1ull << 20) - 1), std::min(r.size * 2, 32ull << 20)); + void* mapped = nullptr; + if (!create_buffer(s.size, kStreamUse, true, s.buffer, s.alloc, &mapped, true) || !mapped) return false; + s.mapped = (uint8_t*)mapped; + stream_extra += s.size, stream_grown++; + static int logged; + if (logged++ < 8) + logf("vk: a recording needed more than its %llu MB stream: one of %llu MB made (%llu MB of them in all)", + (unsigned long long)(r.size >> 20), (unsigned long long)(s.size >> 20), (unsigned long long)(stream_extra.load() >> 20)); + // (one made while the old ones stay: a bigger one that fits here goes before the smaller ones kept after it) + r.streams.insert(r.streams.begin() + (ptrdiff_t)++r.si, s); + } + r.before += r.used; + r.stream = r.streams[r.si].buffer, r.stream_alloc = r.streams[r.si].alloc, r.mapped = r.streams[r.si].mapped; + r.size = r.streams[r.si].size, r.used = 0; + return true; +} +/// Room for n bytes at align in the recording's stream, moving it to a bigger one when there is none (false: no room) +bool stream_reserve(uint64_t n, uint64_t align) { + if (((cur->used + align - 1) & ~(align - 1)) + n <= cur->size) return true; + if (stream_next(n + align)) return true; + static int logged; + if (logged++ < 4) logf("vk: the recording's stream is full (%llu MB)", (unsigned long long)(cur->size >> 20)); + return false; +} +/// Space in the recording's stream (UINT64_MAX when there is none; the buffer is cur->stream and cur->mapped after the call) uint64_t stream_alloc(uint64_t n, uint64_t align) { + if (!stream_reserve(n, align)) return UINT64_MAX; const uint64_t at = (cur->used + align - 1) & ~(align - 1); - if (at + n > kStream) { - static int logged; - if (logged++ < 4) logf("vk: the recording's stream is full (%llu MB)", (unsigned long long)(kStream >> 20)); - return UINT64_MAX; - } cur->used = at + n; return at; } @@ -648,6 +761,22 @@ uint32_t target_write_mask(const Program& p, const NvnState& s, int i) { return (s.chmask.mask >> (4 * i)) & 0xF & (all || !written ? 0xFu : (uint32_t)written); } thread_local const Program* cur_program = nullptr; // (the draw's, for its dynamic write masks) +/// (hookshot) What target_write_mask reads of a program, as one word: whether its fragment stage stores each output and which +/// components it writes of it (6 bits each). Two programs with the same word make the same masks from the same state +uint64_t program_wm_sig(const Program& p) { + if (const int64_t known = p.wm_sig.load(std::memory_order_relaxed); known >= 0) return (uint64_t)known; + const Stage* fs = nullptr; + for (const Stage& st : p.stages) + if (st.vk_stage == VK_SHADER_STAGE_FRAGMENT_BIT) fs = &st; + uint64_t sig = 0; + for (int i = 0; fs && i < 8; i++) + if (fs->info.stores_frag_color[i]) { + const uint32_t written = fs->info.frag_color_components[i]; + sig |= (uint64_t)(1u | (written ? 2u : 0u) | (written & 0xFu) << 2) << (6 * i); + } + p.wm_sig.store((int64_t)sig, std::memory_order_relaxed); // (replay threads may ask at once: the same word) + return sig; +} // ---- (hookshot) The precompile, as TotK Recomp's (runtime/vk_pipeline.c vk_pipeline_prewarm, keys in pipeline-keys.bin): each // pipeline the game asks for is noted in hookshot_pipelines.bin, by what built it; a release build, before the game starts, @@ -816,8 +945,10 @@ PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& pipe_same((uint64_t)(uintptr_t)&p, p.variant, topo_c, (uint32_t)at.samples, at.n, (uint32_t)at.depth_format, formats, s.nvattr, vattr, divs, blend_en, blend, chmask, poly, clamp, a2c, f.prog, f.variant, f.topo, f.samples, f.n, f.depth, f.formats, f.nvattr, f.vattr, f.div, f.blend_en, f.blend, f.chmask, f.poly, - f.clamp, f.a2c)) + f.clamp, f.a2c)) { + FINE_N(FC_PIPE_FAST, 1); return f.pe; + } auto remember = [&](PipeEntry* e) { if (!e) return e; if (!(pipe_fast[0].on && pipe_fast[0].pe == e)) pipe_fast[1] = pipe_fast[0]; @@ -856,7 +987,7 @@ PipeEntry* graphics_pipeline(const Program& p, const NvnState& s, const Targets& if (!dev.eds3_clamp) key = mix(key, (uint64_t)s.depth_clamp); if (!dev.eds3_a2c) key = mix(key, (uint64_t)s.ms.a2c); static thread_local std::unordered_map mine; // (entries live as long as the cache) - if (auto m = mine.find(key); m != mine.end()) return remember(m->second); + if (auto m = mine.find(key); m != mine.end()) return FINE_X(FC_PIPE_MAP, 1), remember(m->second); ParLock lk; auto it = pipelines.find(key); if (it != pipelines.end()) return remember(mine[key] = it->second.get()); @@ -1122,11 +1253,18 @@ VkSamplerAddressMode wrap(uint8_t w) { default: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; } } +// (hookshot) RX_VK_ANISO= (default 0: the game's own): the samplers that filter linearly between texels and between mip levels +// get at least n-times anisotropic filtering (the game's value if it asks for more; the device's limit if n is above it): surfaces seen +// at a slant keep their detail, for some GPU time. Part of every sampler's key below, so no cached sampler is one made for another n +uint32_t aniso_override() { + static const uint32_t n = getenv("RX_VK_ANISO") ? (uint32_t)std::clamp(atoi(getenv("RX_VK_ANISO")), 0, 16) : 0u; + return n; +} /// compare: the shader reads depth with a reference (a sampler without one cannot compare) VkSampler sampler_for(int id, bool compare) { nvn_samp_par p{}; if (!sampler_of(id, &p)) p.minf = NVN_MIN_FILTER_LINEAR, p.magf = NVN_MAG_FILTER_LINEAR, p.lod_max = 1000.0f; - uint64_t key = compare ? 0xC0 : 0xAB; + uint64_t key = mix(compare ? 0xC0 : 0xAB, aniso_override()); const uint8_t* b = reinterpret_cast(&p); for (size_t i = 0; i < sizeof p; i += 8) { uint64_t w = 0; @@ -1146,8 +1284,21 @@ VkSampler sampler_for(int id, bool compare) { ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST; ci.addressModeU = wrap(p.ws), ci.addressModeV = wrap(p.wt), ci.addressModeW = wrap(p.wr); ci.mipLodBias = std::clamp(p.lod_bias, -dev.props.limits.maxSamplerLodBias, dev.props.limits.maxSamplerLodBias); - ci.anisotropyEnable = p.aniso > 1.0f && dev.features.samplerAnisotropy; - ci.maxAnisotropy = std::clamp(p.aniso, 1.0f, dev.props.limits.maxSamplerAnisotropy); + float aniso = p.aniso; + { // (RX_VK_ANISO: what the game asks for is told once for each value, 16 at most: what it sets is not known otherwise) + static std::mutex am; + static std::unordered_set asked; + uint32_t bits; + std::memcpy(&bits, &p.aniso, sizeof bits); + std::lock_guard ag(am); + if (asked.size() < 16 && asked.insert(bits).second) + logf("vk: sampler anisotropy: the game asks for %g (min filter %u, mip filter %s; RX_VK_ANISO=%u)", (double)p.aniso, (unsigned)p.minf, + ci.mipmapMode == VK_SAMPLER_MIPMAP_MODE_LINEAR ? "linear" : "nearest", aniso_override()); + } + if (aniso_override() && ci.minFilter == VK_FILTER_LINEAR && ci.mipmapMode == VK_SAMPLER_MIPMAP_MODE_LINEAR) + aniso = std::max(aniso, (float)aniso_override()); + ci.anisotropyEnable = aniso > 1.0f && dev.features.samplerAnisotropy; + ci.maxAnisotropy = std::clamp(aniso, 1.0f, dev.props.limits.maxSamplerAnisotropy); ci.compareEnable = compare && p.cmp_mode != 0; ci.compareOp = compare_op(p.cmp_func); { // (amd-fix) A depth-reference sample through a sampler that does not compare is undefined in Vulkan: NVIDIA's driver @@ -1377,6 +1528,7 @@ struct BankCache { int stage = -1; uint32_t index = 0, size = 0; uint64_t at = 0; + VkBuffer buf = VK_NULL_HANDLE; // (the stream it is in: a recording may have moved to a bigger one since) uint64_t bind_gen = ~0ull; // (the bindings it was made for: the same and no new grid, the same bank) uint32_t grid[3]{}; std::vector bytes; @@ -1392,14 +1544,25 @@ VkDescriptorBufferInfo uniform_range(const NvnState& s, const Stage& st, uint32_ for (const auto& d : st.info.storage_buffers_descriptors) if (d.cbuf_index == index) used = std::max(used, d.cbuf_offset + d.count * 16u); if (index == 0 || index == 2) { // the driver's banks: only what the program reads of them + FINE_SCOPE(FT_BANK); + FINE_N(FC_BANK_CALLS, 1); const uint32_t n = std::clamp((used + 15) & ~15u, 16, 0xA00); BankCache& c = bank_cache[index == 2][std::clamp(st.nvn_stage, 0, kNvnStages - 1)]; const bool grid_same = st.nvn_stage != 5 || !std::memcmp(c.grid, s.grid, sizeof c.grid); static const bool no_bank_cache = getenv("RX_VK_NO_DESC_REUSE") != nullptr; // (A/B: filled every time) - if (!no_bank_cache && c.at != UINT64_MAX && c.bind_gen == s.bind_gen && grid_same && c.size >= n) - return {cur->stream, c.at, n}; + // (hookshot) a graphics stage's c2 is made from its textures, samplers and images, its c0 from its storage buffers: it stands while + // those do, whatever else was bound (a uniform buffer for each draw). RX_VK_BANK_GEN_OFF=1: while nothing at all was bound anew) + static const bool stage_gen_off = getenv("RX_VK_BANK_GEN_OFF") != nullptr; + const int bank_stage = std::clamp(st.nvn_stage, 0, kNvnStages - 1); + const uint64_t gen = stage_gen_off || bank_stage > 4 ? s.bind_gen : index == 2 ? s.tex_gen[bank_stage] : s.ssbo_gen[bank_stage]; + if (!no_bank_cache && c.at != UINT64_MAX && c.bind_gen == gen && grid_same && c.size >= n) { + FINE_N(FC_BANK_GEN, 1); + return {c.buf, c.at, n}; + } static thread_local std::vector tmp; tmp.resize(n); + FINE_N(FC_BANK_FILLS, 1); + FINE_N(FC_BANK_FILL_BYTES, n); driver_bank_fill(s, (int)index, st.nvn_stage, tmp.data(), n); static const bool check = getenv("RX_VK_CHECK_BANKS") != nullptr; // (the fill against the word path) if (check) @@ -1410,45 +1573,58 @@ VkDescriptorBufferInfo uniform_range(const NvnState& s, const Stage& st, uint32_ logf("vk: bank c%u stage %d word %#x: fill %#x, word path %#x", index, st.nvn_stage, off, *(const uint32_t*)(tmp.data() + off), w); } - c.bind_gen = s.bind_gen; + c.bind_gen = gen; std::memcpy(c.grid, s.grid, sizeof c.grid); if (c.at != UINT64_MAX && c.size >= n && c.bytes.size() >= n && !std::memcmp(c.bytes.data(), tmp.data(), n) && - c.prog != nullptr) - return {cur->stream, c.at, n}; + c.prog != nullptr) { + FINE_N(FC_BANK_SAME, 1); + return {c.buf, c.at, n}; + } const uint64_t at = stream_alloc(n, align); if (at == UINT64_MAX) return bi; + FINE_N(FC_BANK_NEW, 1); + FINE_N(FC_BANK_NEW_BYTES, n); std::memcpy(cur->mapped + at, tmp.data(), n); - c.prog = (const Program*)1, c.at = at, c.size = n, c.bytes = tmp; + c.prog = (const Program*)1, c.at = at, c.buf = cur->stream, c.size = n, c.bytes = tmp; return {cur->stream, at, n}; } if (index == 1) { + FINE_SCOPE(FT_C1); + FINE_N(FC_C1_CALLS, 1); const void* key = &st; auto it = cur->c1_at.find(key); const uint64_t n = std::max((st.c1.size() + 15) & ~15ull, 16); - if (it != cur->c1_at.end()) return {cur->stream, it->second, n}; + if (it != cur->c1_at.end()) return {it->second.first, it->second.second, n}; const uint64_t at = stream_alloc(n, align); if (at == UINT64_MAX) return bi; std::memset(cur->mapped + at, 0, n); std::memcpy(cur->mapped + at, st.c1.data(), st.c1.size()); - cur->c1_at[key] = at; + cur->c1_at[key] = {cur->stream, at}; return {cur->stream, at, n}; } if (index >= 3 && index < 3 + 16) { + FINE_SCOPE(FT_UBO); + FINE_N(FC_UBO_CALLS, 1); const nvn_p_bind_buf& u = s.ubo[std::clamp(st.nvn_stage, 0, kNvnStages - 1)][index - 3]; if (!u.addr) return bi; const uint64_t want = used ? std::max(used, 16) : std::max(u.size, 16); const uint64_t n = std::min(std::min(want, u.size ? std::max(u.size, 16) : want), dev.props.limits.maxUniformBufferRange); // (updates the stream made to this range in this submission or ones still in flight: a copy with them in) - if (const uint64_t at = ub_overlay(u.addr, n, cur->mapped, [&](uint64_t bytes) { return stream_alloc(bytes, align); }); - at != UINT64_MAX) - return {cur->stream, at, n}; + // (room first: the overlay copies into the stream's mapping, which a move to a bigger stream in the middle would change) + if (stream_reserve(n, align)) + if (const uint64_t at = ub_overlay(u.addr, n, cur->mapped, [&](uint64_t bytes) { return stream_alloc(bytes, align); }); + at != UINT64_MAX) { + FINE_N(FC_UBO_OVERLAY, 1); + return {cur->stream, at, n}; + } // (RX_VK_UB_SNAPSHOT=1: the contents as they are now, in the stream, not the pool the GPU reads later) static const bool snapshot = getenv("RX_VK_UB_SNAPSHOT") != nullptr; BufferRef b = snapshot ? BufferRef{} : gpu_buffer_bind(u.addr, n, false); if (b && b.offset % align == 0) return {b.buffer, b.offset, n}; const uint64_t at = stream_alloc(n, align); // (unaligned or outside the pools: copied into the stream) if (at == UINT64_MAX) return bi; + FINE_N(FC_UBO_STREAM, 1); // (as the submission had it, not as the game has rewritten it since: the captures' pages) memory_read_as_submitted(u.addr, cur->mapped + at, n); static int64_t copied; @@ -1613,6 +1789,7 @@ void build_descriptors(const NvnState& s, const Program& p) { for (const auto& d : st.info.image_descriptors) nimg += d.count; } dw.bufs.reserve(nbuf), dw.imgs.reserve(nimg), dw.views.reserve(nview); // (stable pointers below) + FINE_N(FC_DESCS, (int64_t)(nbuf + nimg + nview)); uint32_t binding = 0; auto write = [&](VkDescriptorType type, uint32_t count) { VkWriteDescriptorSet w{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; @@ -1637,6 +1814,8 @@ void build_descriptors(const NvnState& s, const Program& p) { dw.writes.push_back(w); } for (const auto& d : info.storage_buffers_descriptors) { + FINE_SCOPE(FT_STORAGE); + FINE_N(FC_SSBO_DESCS, d.count); VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, d.count); const size_t first = dw.bufs.size(); for (uint32_t k = 0; k < d.count; k++) { @@ -1875,6 +2054,7 @@ void build_descriptors(const NvnState& s, const Program& p) { texel_views(info.texture_buffer_descriptors, VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, false); texel_views(info.image_buffer_descriptors, VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, true); for (const auto& d : info.texture_descriptors) { + FINE_SCOPE(FT_TEX); VkWriteDescriptorSet w = write(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, d.count); const size_t first = dw.imgs.size(); const VkImageViewType vt = view_type(d.type); @@ -1963,11 +2143,16 @@ void build_descriptors(const NvnState& s, const Program& p) { struct Pushed { VkCommandBuffer cmd = VK_NULL_HANDLE; VkPipelineLayout layout = VK_NULL_HANDLE; - std::vector> writes; // (each write's descriptors, as pushed) + size_t n = 0; // (the writes as pushed) + std::vector> writes; // (each write's descriptors, as pushed; RX_VK_PUSH_STATE_OLD=1) + // (hookshot) the same in one vector of words and where each write's begin: a program change was a vector of vectors resized to the + // new program's writes, each new one grown word by word (3 allocations a write, and the ones cut off freed: a dozen of each a draw) + std::vector words; + std::vector at; }; thread_local Pushed pushed[2]; // [graphics, compute] } // namespace -void draw_skip_report(double frames) { skip_report_impl(frames); } +void draw_skip_report(double frames) { skip_report_impl(frames), fine_report(frames); } void descriptors_disturbed() { pushed[0].cmd = pushed[1].cmd = VK_NULL_HANDLE; } @@ -1994,29 +2179,107 @@ void write_words(const VkWriteDescriptorSet& w, std::vector& out) { } } +/// write_words' words appended to out (the words of the pushed state, a write after another) +void words_append(const VkWriteDescriptorSet& w, std::vector& out) { + out.push_back((uint64_t)w.descriptorType << 32 | w.dstBinding); + for (uint32_t k = 0; k < w.descriptorCount; k++) { + if (w.pBufferInfo) { + const VkDescriptorBufferInfo& b = w.pBufferInfo[k]; + out.push_back((uint64_t)b.buffer), out.push_back(b.offset), out.push_back(b.range); + } else if (w.pImageInfo) { + const VkDescriptorImageInfo& i = w.pImageInfo[k]; + out.push_back((uint64_t)i.sampler), out.push_back((uint64_t)i.imageView); + } else if (w.pTexelBufferView) { + out.push_back((uint64_t)w.pTexelBufferView[k]); + } + } +} +/// Whether the words in [o, end) are write_words' of w (1), or not (0: they are after the call). -1: w is not the write they were made for +/// (its words do not fill them exactly), and nothing was touched +int words_same_store(const VkWriteDescriptorSet& w, uint64_t* o, const uint64_t* end) { + const size_t per = w.pBufferInfo ? 3 : w.pImageInfo ? 2 : w.pTexelBufferView ? 1 : 0; + if ((size_t)(end - o) != 1 + (size_t)w.descriptorCount * per) return -1; + bool same = true; + auto put = [&](uint64_t v) { + same &= *o == v; + *o++ = v; + }; + put((uint64_t)w.descriptorType << 32 | w.dstBinding); + for (uint32_t k = 0; k < w.descriptorCount; k++) { + if (w.pBufferInfo) { + const VkDescriptorBufferInfo& b = w.pBufferInfo[k]; + put((uint64_t)b.buffer), put(b.offset), put(b.range); + } else if (w.pImageInfo) { + const VkDescriptorImageInfo& i = w.pImageInfo[k]; + put((uint64_t)i.sampler), put((uint64_t)i.imageView); + } else if (w.pTexelBufferView) { + put((uint64_t)w.pTexelBufferView[k]); + } + } + return same ? 1 : 0; +} + bool bind_descriptors(VkCommandBuffer cmd, VkPipelineBindPoint bp, const Program& p) { if (dw.writes.empty()) return true; Pushed& ps = pushed[bp == VK_PIPELINE_BIND_POINT_COMPUTE]; if (p.push) { static const bool whole = getenv("RX_VK_PUSH_ALL") != nullptr; // (A/B: every binding every time) - if (whole || ps.cmd != cmd || ps.layout != p.layout || ps.writes.size() != dw.writes.size()) { - vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)dw.writes.size(), dw.writes.data()); - ps.cmd = cmd, ps.layout = p.layout; - ps.writes.resize(dw.writes.size()); - for (size_t i = 0; i < dw.writes.size(); i++) write_words(dw.writes[i], ps.writes[i]); + static const bool old_state = getenv("RX_VK_PUSH_STATE_OLD") != nullptr; // (A/B: a vector of words a write, as before) + if (whole || ps.cmd != cmd || ps.layout != p.layout || ps.n != dw.writes.size()) { + if (g_fine) { + fine_n[FC_PUSH_FULL]++, fine_n[FC_PUSH_WRITES] += (int64_t)dw.writes.size(); + for (const VkWriteDescriptorSet& w : dw.writes) fine_n[FC_PUSH_DESCS] += w.descriptorCount; + } + { + FINE_SCOPE(FT_PUSHCALL); + vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)dw.writes.size(), dw.writes.data()); + } + ps.cmd = cmd, ps.layout = p.layout, ps.n = dw.writes.size(); + if (old_state) { + ps.writes.resize(dw.writes.size()); + for (size_t i = 0; i < dw.writes.size(); i++) write_words(dw.writes[i], ps.writes[i]); + } else { + ps.words.clear(), ps.at.clear(); + for (const VkWriteDescriptorSet& w : dw.writes) ps.at.push_back((uint32_t)ps.words.size()), words_append(w, ps.words); + } return true; } static thread_local std::vector changed; static thread_local std::vector words; changed.clear(); - for (size_t i = 0; i < dw.writes.size(); i++) { - write_words(dw.writes[i], words); - if (words != ps.writes[i]) changed.push_back(dw.writes[i]), ps.writes[i].swap(words); + if (old_state) { + for (size_t i = 0; i < dw.writes.size(); i++) { + write_words(dw.writes[i], words); + if (words != ps.writes[i]) changed.push_back(dw.writes[i]), ps.writes[i].swap(words); + } + } else { // (the same layout, so the same count of words in each write: compared and kept in place) + for (size_t i = 0; i < dw.writes.size(); i++) { + const uint64_t* end = ps.words.data() + (i + 1 < ps.at.size() ? ps.at[i + 1] : ps.words.size()); + const int same = words_same_store(dw.writes[i], ps.words.data() + ps.at[i], end); + if (same < 0) { // (not so: nothing is known of what is pushed; the whole set again) + ps.cmd = VK_NULL_HANDLE; + return bind_descriptors(cmd, bp, p); + } + if (!same) changed.push_back(dw.writes[i]); + } + } + if (!changed.empty()) { + if (g_fine) { + fine_n[FC_PUSH_DELTA]++, fine_n[FC_PUSH_WRITES] += (int64_t)changed.size(); + for (const VkWriteDescriptorSet& w : changed) fine_n[FC_PUSH_DESCS] += w.descriptorCount; + } + FINE_SCOPE(FT_PUSHCALL); + vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)changed.size(), changed.data()); + } else { + FINE_N(FC_PUSH_NONE, 1); } - if (!changed.empty()) vkCmdPushDescriptorSetKHR(cmd, bp, p.layout, 0, (uint32_t)changed.size(), changed.data()); return true; } ps.cmd = VK_NULL_HANDLE; // (a set bound: what was pushed is not known to stay) + if (g_fine) { + fine_n[FC_POOL_SETS]++; + for (const VkWriteDescriptorSet& w : dw.writes) fine_n[FC_POOL_DESCS] += w.descriptorCount; + } VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; ai.descriptorPool = cur->extra.empty() ? cur->pool : cur->extra.back(); ai.descriptorSetCount = 1; @@ -2056,6 +2319,7 @@ thread_local uint64_t fallback_payload_address = 0, fallback_records_address = 0 void push_constants(VkCommandBuffer cmd, const Program& p, const Targets* at) { checkpoint(cmd, p, at ? 'd' : 'c'); if (!p.uses_push) return; // (most programs read none: a push a draw was ~5% of the replay's commands) + FINE_N(FC_PUSHC, 1); uint8_t data[kPushSize]{}; const float one = 1.0f; std::memcpy(data + Shader::Backend::SPIRV::RESCALING_LAYOUT_DOWN_FACTOR_OFFSET, &one, 4); @@ -2280,7 +2544,7 @@ void begin_pass(VkCommandBuffer cmd, const Targets& at) { for (int i = 0; i < at.n && same; i++) same = pass.color[i] == at.color_view[i]; if (same) return; end_pass(cmd); - full_barrier(cmd); // (what the pass before wrote, and the transfers before it) + full_barrier(cmd, "barrier (pass start)"); // (what the pass before wrote, and the transfers before it) work_since_barrier = true; VkRenderingAttachmentInfo ca[8]{}; bool marks = !texture_marked(at.depth); @@ -2351,7 +2615,7 @@ struct DynState { }; thread_local DynState last_dyn; thread_local bool last_dyn_valid = false; -#define DYN_CHANGED(f) (!last_dyn_valid || std::memcmp(&d.f, &last_dyn.f, sizeof d.f)) +#define DYN_CHANGED(f) ((!last_dyn_valid || std::memcmp(&d.f, &last_dyn.f, sizeof d.f)) && (FINE_X(FC_DYN_CALLS, 1), true)) /// The inputs a draw's dynamic state is built from. Most draws repeat the draw before them. bool dyn_same(const float* fp, int nf, const uint32_t* up, int nu, const uint8_t* raw, int nb, const float* cfp, @@ -2398,7 +2662,10 @@ int dyn_pack(const NvnState& s, const Targets& at, VkPrimitiveTopology topo, boo atb += (int)sizeof s.ds; std::memcpy(raw + atb, s.blend, sizeof s.blend); atb += (int)sizeof s.blend; - const uint64_t prog = (uint64_t)(uintptr_t)cur_program; + // (hookshot) what the program makes of the write masks, not the program: draws change program far more often than they change that, + // and each program change worked the whole state out again. RX_VK_DYN_PROG_KEY=1: the program, as before) + static const bool by_program = getenv("RX_VK_DYN_PROG_KEY") != nullptr; + const uint64_t prog = by_program || !cur_program ? (uint64_t)(uintptr_t)cur_program : program_wm_sig(*cur_program); std::memcpy(raw + atb, &prog, 8); return atb + 8; } @@ -2411,8 +2678,10 @@ void dynamic_state(VkCommandBuffer cmd, const NvnState& s, const Targets& at, Vk if (!dump_on() && last_dyn_valid && dyn_fast.on && dyn_fast.nb == nb && dyn_same(df, kDynF, du, kDynU, db, nb, dyn_fast.fp, dyn_fast.up, dyn_fast.raw)) { pass.wm_or |= dyn_fast.wm0, pass.wm_n = dyn_fast.wm_n; // (the pass trace still sees the masks) + FINE_N(FC_DYN_FAST, 1); return; } + FINE_N(FC_DYN_FULL, 1); DynState d; std::memset(&d, 0, sizeof d); // (padding compared too) // (RX_VK_FOOTPRINT=,...: those programs' draws write depth wherever they rasterize, both faces, whatever is @@ -2574,11 +2843,11 @@ void dynamic_state(VkCommandBuffer cmd, const NvnState& s, const Targets& at, Vk if (d.n) { const bool same_n = last_dyn_valid && last_dyn.n == d.n; if (!same_n || std::memcmp(d.en, last_dyn.en, sizeof(VkBool32) * d.n)) - vkCmdSetColorBlendEnableEXT(cmd, 0, (uint32_t)d.n, d.en); + FINE_X(FC_DYN_CALLS, 1), vkCmdSetColorBlendEnableEXT(cmd, 0, (uint32_t)d.n, d.en); if (!same_n || std::memcmp(d.eq, last_dyn.eq, sizeof(VkColorBlendEquationEXT) * d.n)) - vkCmdSetColorBlendEquationEXT(cmd, 0, (uint32_t)d.n, d.eq); + FINE_X(FC_DYN_CALLS, 1), vkCmdSetColorBlendEquationEXT(cmd, 0, (uint32_t)d.n, d.eq); if (!same_n || std::memcmp(d.wm, last_dyn.wm, sizeof(VkColorComponentFlags) * d.n)) - vkCmdSetColorWriteMaskEXT(cmd, 0, (uint32_t)d.n, d.wm); + FINE_X(FC_DYN_CALLS, 1), vkCmdSetColorWriteMaskEXT(cmd, 0, (uint32_t)d.n, d.wm); } last_dyn = d; last_dyn_valid = true; @@ -2840,17 +3109,40 @@ void draw_begin_recording(int slot, VkCommandBuffer cmd, uint64_t value) { if (!r.pool) { r.pool = make_pool(); void* mapped = nullptr; - create_buffer(kStream, VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | - VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | - VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - true, r.stream, r.stream_alloc, &mapped); - r.mapped = (uint8_t*)mapped; + RecordingRes::Stream s{}; + s.size = kStream; + create_buffer(s.size, kStreamUse, true, s.buffer, s.alloc, &mapped, true); + s.mapped = (uint8_t*)mapped; + r.streams.push_back(s); } else { vkResetDescriptorPool(dev.device, r.pool, 0); for (VkDescriptorPool p : r.extra) vkDestroyDescriptorPool(dev.device, p, nullptr); r.extra.clear(); + // (the streams the last recording of the slot grew into: given back after 120 recordings that stayed in the first, the + // slot's last one being done (recording_wait). RX_VK_STREAM_KEEP=1: kept) + static const bool keep = getenv("RX_VK_STREAM_KEEP") != nullptr; + if (r.streams.size() > 1 && !keep) { + if (r.si > 0) r.idle = 0; + else if (++r.idle >= 120) { + for (size_t i = 1; i < r.streams.size(); i++) { + stream_extra -= r.streams[i].size; + destroy_buffer(r.streams[i].buffer, r.streams[i].alloc); + } + r.streams.resize(1); + } + } } - r.used = 0; + { + const RecordingRes::Stream& s = r.streams[0]; + r.si = 0, r.stream = s.buffer, r.stream_alloc = s.alloc, r.mapped = s.mapped, r.size = s.size; + } + if (r.before + r.used > stream_peak.load(std::memory_order_relaxed)) stream_peak.store(r.before + r.used, std::memory_order_relaxed); + static int64_t begun; + if (++begun % 3000 == 0) + logf("vk: the recording streams: %zu of %llu MB, the most one held %llu KB; %llu made on demand (%llu MB kept now)", res.size(), + (unsigned long long)(kStream >> 20), (unsigned long long)(stream_peak.load(std::memory_order_relaxed) >> 10), + (unsigned long long)stream_grown.load(), (unsigned long long)(stream_extra.load() >> 20)); + r.used = 0, r.before = 0; r.c1_at.clear(); cur = &r; pass = Pass{}; @@ -3127,11 +3419,11 @@ void end_pass(VkCommandBuffer cmd) { (unsigned long long)pass.first_prog, (unsigned long long)pass.prog, sampled); } if (gprof_on()) { - char label[128]; - snprintf(label, sizeof label, "pass %ux%u f%d %dc%s, %s draws, last prog %016llx", pass.width, pass.height, - (int)pass.format0, pass.n, pass.depth ? "+d" : "", pass.draws > 100 ? ">100" : pass.draws > 10 ? "11-100" : std::to_string(pass.draws).c_str(), - (unsigned long long)pass.prog); - gprof_end(cmd, label); + char label[160]; + snprintf(label, sizeof label, "pass %ux%u f%d %dc%s, %s draws, first prog %016llx, last prog %016llx", pass.width, pass.height, + (int)pass.format0, pass.n, pass.depth ? "+d" : "", pass.draws > 256 ? ">256" : pass.draws > 32 ? "33-256" : pass.draws > 4 ? "5-32" : "0-4", + (unsigned long long)pass.first_prog, (unsigned long long)pass.prog); + gprof_end(cmd, label, pass.draws); } if (dump_on()) for (int i = 0; i < pass.n; i++) @@ -4138,6 +4430,7 @@ bool fallback_record(VkCommandBuffer cmd, Program& p, const DrawArgs& d) { void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { PROF(PROF_DRAW); + FINE_N(FC_DRAWS, 1); stats_tick(); mesh_indirect_convert(cmd, d); // (EvilOOT: before the pass opens; sets mesh_conv_buf for the mesh draw below) VkPrimitiveTopology topo = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; @@ -4294,8 +4587,10 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { last_desc_reg == reg_gen && last_desc_ub == ub_gen; if (!same_desc) { PROF(PROF_DESC); + FINE_N(FC_BUILDS, 1); trace_tex_hit = nullptr, trace_tex_mask = 0; build_descriptors(s, p); // (textures first used here are uploaded by the recording's upload command buffer) + FINE_N(FC_WRITES, (int64_t)dw.writes.size()); static std::string last_logged; if (dump_on()) { char head[256]; @@ -4316,7 +4611,11 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { if (!same_desc) gfx_sampled = draw_sampled; static const bool barrier_every = getenv("RX_VK_BARRIER_EVERY_DRAW") != nullptr; // (an experiment: a pass each) if (barrier_every) end_pass(cmd); - const uint16_t writes = draw_hazards(cmd, p, s, at); + uint16_t writes; + { + FINE_SCOPE(FT_HAZARD); + writes = draw_hazards(cmd, p, s, at); + } BufferRef mesh_args{}; uint32_t mesh_args_at = 0; // (RX_VK_MESH_ARGS=1: rewritten {count, 1, 1, first}. Off: it did not bring the canopies back, and it ends the pass at each draw) @@ -4382,8 +4681,12 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { } } if (p.fallback && d.mesh && !fallback_record(cmd, p, d)) SKIP_HERE(); - begin_pass(cmd, at); + { + FINE_SCOPE(FT_HAZARD); + begin_pass(cmd, at); + } probe_draw(s, p, pass.draws); // (hookshot: RX_VK_PROBE) + gprof_run(cmd, p.key); // (RX_VK_PASS_TIMING=2) pass.prog = p.key; if (!pass.first_prog) pass.first_prog = p.key; for (const Texture* t : gfx_sampled) { @@ -4396,15 +4699,23 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { if (slot >= 0) pass.sampled_targets |= (uint16_t)(1u << slot); } pass.written |= writes; - if (pl != last_pipeline) vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pl), last_pipeline = pl; + if (pl != last_pipeline) { + FINE_SCOPE(FT_BINDPL); + FINE_N(FC_BIND_PIPE, 1); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pl), last_pipeline = pl; + } if (!same_desc) { + FINE_SCOPE(FT_BINDDESC); if (!bind_descriptors(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, p)) SKIP_HERE(); last_desc_valid = true, last_desc_prog = &p, last_desc_bind = s.bind_gen, last_desc_reg = reg_gen; last_desc_ub = ub_gen; } else { st_desc_reused++; } - push_constants(cmd, p, &at); + { + FINE_SCOPE(FT_PUSHC); + push_constants(cmd, p, &at); + } if (watched(p.key)) { // (RX_VK_WATCH_PROG: the draw's state, 80 lines) static std::atomic n{0}; if (n++ < 80) { @@ -4422,7 +4733,10 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { logf("%s", line); } } - dynamic_state(cmd, s, at, topo, d.mesh && !p.fallback); + { + FINE_SCOPE(FT_DYN); + dynamic_state(cmd, s, at, topo, d.mesh && !p.fallback); + } if (trace_tex_hit && texture_frame_no() % 30 == 0) { static std::atomic told{0}; if (told++ < 600) { @@ -4577,6 +4891,7 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { // vertex streams (from their pools, strides dynamic) and the zero stream // (the same streams are bound for draw after draw: the pool lookup, which was ~8% of a race under the lock, is // skipped until the submission or the mirror changes) + const int64_t fine_vertex_t0 = g_fine ? prof_now() : 0; VkBuffer vbufs[16]; VkDeviceSize voffs[16], vsizes[16], vstrides[16]; for (uint32_t b = 0; b < 16; b++) { @@ -4597,11 +4912,13 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { } if (!last_vb_valid || std::memcmp(vbufs, last_vb.b, sizeof vbufs) || std::memcmp(voffs, last_vb.o, sizeof voffs) || std::memcmp(vsizes, last_vb.z, sizeof vsizes) || std::memcmp(vstrides, last_vb.s, sizeof vstrides)) { + FINE_N(FC_VBIND, 1); vkCmdBindVertexBuffers2(cmd, 0, 16, vbufs, voffs, vsizes, vstrides); std::memcpy(last_vb.b, vbufs, sizeof vbufs), std::memcpy(last_vb.o, voffs, sizeof voffs); std::memcpy(last_vb.z, vsizes, sizeof vsizes), std::memcpy(last_vb.s, vstrides, sizeof vstrides); last_vb_valid = true; } + if (g_fine) fine_ns[FT_VERTEX] += prof_now() - fine_vertex_t0; const VkIndexType itype = d.itype == 0 ? VK_INDEX_TYPE_UINT8_EXT : d.itype == 1 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32; if (d.indexed && d.itype == 0 && !dev.index_uint8) SKIP_HERE(); if (d.prim == NVN_DRAW_PRIMITIVE_QUADS) { // (quads as two triangles each: indices made in the stream) @@ -4649,11 +4966,18 @@ void draw(VkCommandBuffer cmd, NvnState& s, const DrawArgs& d) { } } else if (d.indexed) { const uint32_t unit = d.itype == 0 ? 1 : d.itype == 1 ? 2 : 4; - BufferRef ib = gpu_buffer_bind(d.ibuf, (uint64_t)std::max(d.count, 1) * unit, false); - if (!ib) SKIP_HERE(); - vkCmdBindIndexBuffer(cmd, ib.buffer, ib.offset, itype); + BufferRef ib; + { + FINE_SCOPE(FT_VERTEX); + ib = gpu_buffer_bind(d.ibuf, (uint64_t)std::max(d.count, 1) * unit, false); + if (!ib) SKIP_HERE(); + FINE_N(FC_IBIND, 1); + vkCmdBindIndexBuffer(cmd, ib.buffer, ib.offset, itype); + } + FINE_SCOPE(FT_DRAWCALL); vkCmdDrawIndexed(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), 0, d.base_vertex, (uint32_t)d.base_instance); } else { + FINE_SCOPE(FT_DRAWCALL); vkCmdDraw(cmd, (uint32_t)d.count, (uint32_t)std::max(1, d.instances), (uint32_t)d.first, (uint32_t)d.base_instance); } st_draws++, pass.draws++; @@ -4964,8 +5288,20 @@ static bool typed_upscale(VkCommandBuffer cmd, VkImageView input_view, VkImageVi // (off by default: bounding a pixel by the last frame's smeared everything that moves (the user: blurry, warping) // and only dimmed the flashes; RX_VK_TAAU_CLAMP=1 turns it on) static const bool no_clamp = getenv("RX_VK_TAAU_CLAMP") == nullptr; + // (the ones a new size replaced: given back 120 presents later, when no work in flight can use them. They were left, the + // output's size changing with the dynamic resolution. RX_VK_TAAU_HIST_LEAK=1: as before) + struct OldHist { VkImage image; VmaAllocation mem; VkImageView view; uint64_t frame; }; + static std::vector old_hist; + static const bool leak = getenv("RX_VK_TAAU_HIST_LEAK") != nullptr; + for (size_t i = 0; i < old_hist.size();) { + if (texture_frame_no() - old_hist[i].frame < 120) { i++; continue; } + if (old_hist[i].view) vkDestroyImageView(dev.device, old_hist[i].view, nullptr); + destroy_image(old_hist[i].image, old_hist[i].mem); + old_hist.erase(old_hist.begin() + (ptrdiff_t)i); + } if (hist_w != size[0] || hist_h != size[1]) { - // (the one before is kept: work in flight may still use it, and the output's size seldom changes) + // (the one before is kept for the while: work in flight may still use it) + if (hist && !leak) old_hist.push_back({hist, hist_mem, hist_view, texture_frame_no()}); hist = VK_NULL_HANDLE, hist_view = VK_NULL_HANDLE; VkImageCreateInfo ic{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; ic.imageType = VK_IMAGE_TYPE_2D, ic.format = VK_FORMAT_B10G11R11_UFLOAT_PACK32; @@ -5468,7 +5804,7 @@ void dispatch(VkCommandBuffer cmd, NvnState& s, uint32_t x, uint32_t y, uint32_t // (dispatches after other work wait for it: the WebGPU renderer's dispatches were ordered the same way by // wgpu's own tracking, and MKW's compute chains read what the one before wrote) static const bool no_order = getenv("RX_VK_NO_DISPATCH_ORDER") != nullptr; - if (work_since_barrier && !no_order) full_barrier(cmd); + if (work_since_barrier && !no_order) full_barrier(cmd, "barrier (dispatch)"); work_since_barrier = true; if (!p.stages.empty() && p.stages[0].info.uses_cooperative_matrix) { // (the game's networks, once each: what they read and write) static std::mutex mu; diff --git a/gpu/vk/vk_dump.cpp b/gpu/vk/vk_dump.cpp index 943a2768c..afc56b096 100644 --- a/gpu/vk/vk_dump.cpp +++ b/gpu/vk/vk_dump.cpp @@ -11,6 +11,7 @@ #include #include "vk_dump.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { namespace { diff --git a/gpu/vk/vk_framegen.cpp b/gpu/vk/vk_framegen.cpp index 0d0931e6f..bf2fd9820 100644 --- a/gpu/vk/vk_framegen.cpp +++ b/gpu/vk/vk_framegen.cpp @@ -26,6 +26,7 @@ #include "shaders/fg_median_comp_spv.h" #include "shaders/fg_scale_comp_spv.h" #include "shaders/fg_synth_comp_spv.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { extern std::atomic present_kind; // (vk_replay.cpp: the frame log's) diff --git a/gpu/vk/vk_gprof.cpp b/gpu/vk/vk_gprof.cpp index a2222e3a7..c5531f9e3 100644 --- a/gpu/vk/vk_gprof.cpp +++ b/gpu/vk/vk_gprof.cpp @@ -1,7 +1,8 @@ -// GPU profile (vk_gprof.h): a query pool of timestamp pairs per recording slot. +// GPU profile (vk_gprof.h): a query pool of timestamps per recording slot and command buffer (the recording's, and its upload one). #include #include #include +#include #include #include @@ -11,89 +12,241 @@ namespace mkwvk { namespace { constexpr int kSlots = 48; // (vk_replay.cpp's kRecordings) -constexpr uint32_t kPairs = 1024; // spans a recording can time +constexpr uint32_t kQ = 2048; // timestamps a command buffer can take in a recording VkQueryPool pool = VK_NULL_HANDLE; +struct Span { + std::string label; + uint32_t qb = 0, qe = 0; // (its timestamps: a run of a pass shares them with its neighbours) + int items = 1; + uint64_t bytes = 0; + bool nested = false; // (a program's run inside a pass: counted by itself, not in the command buffer's total) +}; +struct Stream { + VkCommandBuffer cmd = VK_NULL_HANDLE; + uint32_t base = 0; // (its first query) + std::vector spans; + uint32_t nq = 0; // (timestamps taken) + uint32_t dirty = kQ; // (queries that may hold a result: the next use resets them; all of them before the first) + int outer = -1; // (the open span) + int run = -1; // (the open program run of the open pass, RX_VK_PASS_TIMING=2) + uint64_t run_key = 0; +}; struct Slot { - std::vector labels; // one a closed span - bool open = false; - bool used = false; + Stream s[2]; // (0: the recording's command buffer, 1: its upload command buffer) + bool used = false, up_sent = false; }; Slot slots[kSlots]; int cur = -1; -std::unordered_map> totals; // label -> (ns, spans) +struct Agg { + double ns = 0; + int64_t n = 0, items = 0; + uint64_t bytes = 0; +}; +std::unordered_map totals[2]; // label -> what its spans took, per command buffer kind +double active_ns[2], sum_ns[2]; // (the command buffers' first timestamp to last, and what their spans took) +int64_t recs[2], dropped; +int acc_frames = 0; + +Stream* find(VkCommandBuffer cmd) { + if (!pool || cur < 0 || !cmd) return nullptr; + for (Stream& s : slots[cur].s) + if (s.cmd == cmd) return &s; + return nullptr; +} +uint32_t stamp(Stream& st, VkCommandBuffer cmd) { + vkCmdWriteTimestamp2(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, pool, st.base + st.nq); + return st.nq++; +} } // namespace bool gprof_on() { - static const bool on = getenv("RX_VK_GPU_PROFILE") && atoi(getenv("RX_VK_GPU_PROFILE")) != 0; + static const bool on = [] { + const char* a = getenv("RX_VK_PASS_TIMING"); + const char* b = getenv("RX_VK_GPU_PROFILE"); + return (a && atoi(a) != 0) || (b && atoi(b) != 0); + }(); return on; } +static int level() { + static const int v = [] { + const char* a = getenv("RX_VK_PASS_TIMING"); + return a ? atoi(a) : 1; + }(); + return v; +} -void gprof_begin_recording(int slot, VkCommandBuffer cmd) { +void gprof_begin_recording(int slot, VkCommandBuffer cmd, VkCommandBuffer upload) { if (!gprof_on() || slot < 0 || slot >= kSlots) return; if (!pool) { VkQueryPoolCreateInfo qi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qi.queryType = VK_QUERY_TYPE_TIMESTAMP; - qi.queryCount = kSlots * kPairs * 2; + qi.queryCount = kSlots * 2 * kQ; if (vkCreateQueryPool(dev.device, &qi, nullptr, &pool) != VK_SUCCESS) return; - logf("vk: GPU profile on (RX_VK_GPU_PROFILE)"); + logf("vk: GPU pass timing on (RX_VK_PASS_TIMING=%d): every 600 presents the spans of GPU work are logged", level()); } cur = slot; Slot& s = slots[slot]; - s.labels.clear(); - s.open = false; - s.used = true; - vkCmdResetQueryPool(cmd, pool, slot * kPairs * 2, kPairs * 2); + s.used = true, s.up_sent = false; + for (int k = 0; k < 2; k++) { + Stream& st = s.s[k]; + st.cmd = k ? upload : cmd; + st.base = (uint32_t)(slot * 2 + k) * kQ; + st.spans.clear(); + st.outer = st.run = -1; + if (st.dirty) vkCmdResetQueryPool(st.cmd, pool, st.base, st.dirty); // (an upload command buffer that is not sent resets nothing: its dirty stands) + st.nq = 0; + } } -void gprof_begin(VkCommandBuffer cmd) { - if (!pool || cur < 0) return; - Slot& s = slots[cur]; - if (s.open || s.labels.size() >= kPairs) return; - vkCmdWriteTimestamp2(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, pool, (uint32_t)(cur * kPairs + s.labels.size()) * 2); - s.open = true; +void gprof_submitted(int slot, bool upload_sent) { + if (!pool || slot < 0 || slot >= kSlots) return; + slots[slot].up_sent = upload_sent; } -void gprof_end(VkCommandBuffer cmd, const std::string& label) { - if (!pool || cur < 0) return; - Slot& s = slots[cur]; - if (!s.open) return; - vkCmdWriteTimestamp2(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, pool, - (uint32_t)(cur * kPairs + s.labels.size()) * 2 + 1); - s.labels.push_back(label); - s.open = false; +bool gprof_begin(VkCommandBuffer cmd) { + Stream* st = find(cmd); + if (!st || st->outer >= 0 || st->nq + 4 > kQ) { + if (st && st->outer < 0) dropped++; + return false; + } + Span sp; + sp.qb = stamp(*st, cmd); + st->spans.push_back(std::move(sp)); + st->outer = (int)st->spans.size() - 1; + return true; +} + +void gprof_end(VkCommandBuffer cmd, const std::string& label, int items, uint64_t bytes) { + Stream* st = find(cmd); + if (!st || st->outer < 0) return; + const uint32_t q = stamp(*st, cmd); + if (st->run >= 0) st->spans[st->run].qe = q, st->run = -1; // (the pass's last run ends with it) + Span& sp = st->spans[st->outer]; + sp.qe = q, sp.label = label, sp.items = items, sp.bytes = bytes; + st->outer = -1; +} + +void gprof_run(VkCommandBuffer cmd, uint64_t key) { + if (level() < 2) return; + Stream* st = find(cmd); + if (!st || st->outer < 0) return; + if (st->run >= 0 && st->run_key == key) { + st->spans[st->run].items++; + return; + } + if (st->nq + 3 > kQ) return; + const uint32_t q = stamp(*st, cmd); + if (st->run >= 0) st->spans[st->run].qe = q; + Span sp; + char l[32]; + snprintf(l, sizeof l, "prog %016llx", (unsigned long long)key); + sp.label = l, sp.qb = q, sp.nested = true; + st->spans.push_back(std::move(sp)); + st->run = (int)st->spans.size() - 1, st->run_key = key; } void gprof_collect(int slot) { if (!pool || slot < 0 || slot >= kSlots) return; Slot& s = slots[slot]; - if (!s.used || s.labels.empty()) return; - std::vector t(s.labels.size() * 2); - if (vkGetQueryPoolResults(dev.device, pool, slot * kPairs * 2, (uint32_t)t.size(), t.size() * 8, t.data(), 8, - VK_QUERY_RESULT_64_BIT) != VK_SUCCESS) - return; + if (!s.used) return; + s.used = false; const double period = dev.props.limits.timestampPeriod; - for (size_t i = 0; i < s.labels.size(); i++) { - if (t[2 * i + 1] <= t[2 * i]) continue; - auto& e = totals[s.labels[i]]; - e.first += (double)(t[2 * i + 1] - t[2 * i]) * period; - e.second++; + for (int k = 0; k < 2; k++) { + Stream& st = s.s[k]; + if (k == 0 || s.up_sent) st.dirty = st.nq; // (the next use resets what this one wrote) + if (st.spans.empty() || st.nq == 0 || (k && !s.up_sent)) { + st.spans.clear(); + continue; + } + std::vector t(st.nq); + if (vkGetQueryPoolResults(dev.device, pool, st.base, st.nq, t.size() * 8, t.data(), 8, VK_QUERY_RESULT_64_BIT) != VK_SUCCESS) { + st.spans.clear(); + continue; + } + uint64_t first = UINT64_MAX, last = 0; + double sum = 0; + for (const Span& sp : st.spans) { + if (sp.label.empty() || sp.qb >= st.nq || sp.qe >= st.nq || t[sp.qe] <= t[sp.qb]) continue; + const double ns = (double)(t[sp.qe] - t[sp.qb]) * period; + Agg& a = totals[k][sp.label]; + a.ns += ns, a.n++, a.items += sp.items, a.bytes += sp.bytes; + if (sp.nested) continue; + sum += ns; + first = std::min(first, t[sp.qb]), last = std::max(last, t[sp.qe]); + } + if (last > first) { + active_ns[k] += (double)(last - first) * period; + sum_ns[k] += sum; + recs[k]++; + } + st.spans.clear(); } - s.labels.clear(); - s.used = false; } void gprof_report(int frames) { - if (!pool || totals.empty()) return; - std::vector>> v(totals.begin(), totals.end()); - std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.second.first > b.second.first; }); - double all = 0; - for (const auto& e : v) all += e.second.first; - logf("vk: GPU profile: %.2f ms a frame in %zu kinds of span; the most:", all / frames / 1e6, v.size()); - for (size_t i = 0; i < v.size() && i < 25; i++) - logf("vk: %6.3f ms %5.1f a frame %s", v[i].second.first / frames / 1e6, (double)v[i].second.second / frames, - v[i].first.c_str()); - totals.clear(); + if (!pool) return; + acc_frames += frames; + if (acc_frames < 600) return; + const double f = acc_frames, mb = 1048576.0; + logf("vk: GPU pass timing, %d presents, per frame (each timestamp waits for the GPU to finish what came before it: spans add up to " + "more than an unprofiled frame; read the shares):", acc_frames); + logf("vk: %.2f ms a frame in all (recordings' command buffers + upload command buffers; the present's blit to the window is not timed; " + "the 'GPU time' line of an unprofiled run counts the first kind only)", (active_ns[0] + active_ns[1]) / f / 1e6); + logf("vk: recordings' command buffers: %.2f ms from the first timestamp to the last in %.1f a frame (%.2f ms in spans, %.2f ms " + "between them: glue the spans do not name)", active_ns[0] / f / 1e6, recs[0] / f, sum_ns[0] / f / 1e6, + (active_ns[0] - sum_ns[0]) / f / 1e6); + logf("vk: upload command buffers: %.2f ms in %.1f a frame (%.2f ms in spans, %.2f ms between them: texture uploads and " + "decodes, the page table, glue)", active_ns[1] / f / 1e6, recs[1] / f, sum_ns[1] / f / 1e6, + (active_ns[1] - sum_ns[1]) / f / 1e6); + if (dropped) logf("vk: %lld spans not timed (a command buffer's timestamps ran out)", (long long)dropped); + struct Row { + std::string name; + int k; + Agg a; + }; + std::vector rows, cats, progs; + std::map by_cat[2]; + for (int k = 0; k < 2; k++) + for (const auto& [label, a] : totals[k]) { + if (label.compare(0, 5, "prog ") == 0) { + progs.push_back({label, k, a}); + continue; + } + rows.push_back({label, k, a}); + Agg& c = by_cat[k][label.substr(0, label.find(' '))]; + c.ns += a.ns, c.n += a.n, c.items += a.items, c.bytes += a.bytes; + } + for (int k = 0; k < 2; k++) + for (const auto& [name, a] : by_cat[k]) cats.push_back({name, k, a}); + const auto by_ns = [](const Row& a, const Row& b) { return a.a.ns > b.a.ns; }; + std::sort(cats.begin(), cats.end(), by_ns); + std::sort(rows.begin(), rows.end(), by_ns); + std::sort(progs.begin(), progs.end(), by_ns); + logf("vk: by kind (cb: m = recordings' command buffers, u = upload ones; items: draws of a pass, copies of a copy batch):"); + for (const Row& c : cats) + logf("vk: %c %-9s %7.3f ms %5.1f%% %7.1f spans %8.1f items %7.2f MB %7.1f us a span", c.k ? 'u' : 'm', c.name.c_str(), + c.a.ns / f / 1e6, 100.0 * c.a.ns / std::max(active_ns[c.k], 1.0), c.a.n / f, c.a.items / f, c.a.bytes / f / mb, + c.a.ns / std::max(c.a.n, 1) / 1e3); + logf("vk: the 25 spans that took the most (ms a frame, spans a frame, us a span, items a frame, MB a frame):"); + for (size_t i = 0; i < rows.size() && i < 25; i++) { + const Row& r = rows[i]; + logf("vk: %c %7.3f ms %6.1f x %7.1f us %8.1f items %6.2f MB %s", r.k ? 'u' : 'm', r.a.ns / f / 1e6, r.a.n / f, + r.a.ns / std::max(r.a.n, 1) / 1e3, r.a.items / f, r.a.bytes / f / mb, r.name.c_str()); + } + if (!progs.empty()) { + logf("vk: the 20 programs whose runs of draws took the most (RX_VK_PASS_TIMING=2: a timestamp at each change of program):"); + for (size_t i = 0; i < progs.size() && i < 20; i++) { + const Row& r = progs[i]; + logf("vk: %7.3f ms %6.1f runs %7.1f us a run %8.1f draws %6.2f us a draw %s", r.a.ns / f / 1e6, r.a.n / f, + r.a.ns / std::max(r.a.n, 1) / 1e3, r.a.items / f, r.a.ns / std::max(r.a.items, 1) / 1e3, + r.name.c_str()); + } + } + for (auto& t : totals) t.clear(); + active_ns[0] = active_ns[1] = sum_ns[0] = sum_ns[1] = 0; + recs[0] = recs[1] = dropped = 0; + acc_frames = 0; } } // namespace mkwvk diff --git a/gpu/vk/vk_gprof.h b/gpu/vk/vk_gprof.h index 6d44fc9b1..360a85522 100644 --- a/gpu/vk/vk_gprof.h +++ b/gpu/vk/vk_gprof.h @@ -1,6 +1,10 @@ -// GPU profile (diagnostics): RX_VK_GPU_PROFILE=1 times every pass and dispatch on the GPU (a timestamp after all the -// work before it and one after its own), labelled by what it renders or dispatches, and every 120 presents logs the -// labels that took the most GPU time a frame. +// GPU profile (diagnostics): RX_VK_PASS_TIMING=1 (RX_VK_GPU_PROFILE=1 is the same) times the GPU's work with timestamps: every render +// pass, compute dispatch, barrier, texture/buffer copy, mirror sync, write-back and uniform write of the recordings' command buffers, +// and of their upload command buffers, each a pair of timestamps labelled by what it was. Every 600 presents it logs, per frame: the +// kinds of span with their totals, and the labels that took the most GPU time. +// RX_VK_PASS_TIMING=2: also the runs of consecutive draws of one program inside the passes ("prog" lines: GPU ms by program key). +// A timestamp waits for the GPU to finish what came before it, so profiled spans add up to more than an unprofiled frame (the +// passes are ordered by full barriers anyway; the overlap that narrower barriers would allow is not measured here). #pragma once #include @@ -10,14 +14,37 @@ namespace mkwvk { bool gprof_on(); -/// A recording begins in slot (its earlier work is done: its timings are collected first) -void gprof_begin_recording(int slot, VkCommandBuffer cmd); -/// A timed span begins (outside a pass, or right after it began) / ends (its label: what it was) -void gprof_begin(VkCommandBuffer cmd); -void gprof_end(VkCommandBuffer cmd, const std::string& label); +/// A recording begins in slot (its earlier work is done: its timings are collected first); the upload command buffer is the +/// recording's other one (submitted ahead of cmd, only when it has work) +void gprof_begin_recording(int slot, VkCommandBuffer cmd, VkCommandBuffer upload); +/// The recording is going to the GPU (upload_sent: with its upload command buffer) +void gprof_submitted(int slot, bool upload_sent); +/// A timed span begins in cmd (one at a time a command buffer: false when one is open already, or cmd is not a profiled one); +/// gprof_end closes it (its label: what it was; items: draws, copies...; bytes: copied) +bool gprof_begin(VkCommandBuffer cmd); +void gprof_end(VkCommandBuffer cmd, const std::string& label, int items = 1, uint64_t bytes = 0); +/// A draw of program key into the open pass (RX_VK_PASS_TIMING=2): the run of draws of one program is timed +void gprof_run(VkCommandBuffer cmd, uint64_t key); /// The slot's work completed: its spans are added to the totals void gprof_collect(int slot); -/// Every 120 presents: the top labels a frame +/// Every 120 presents (frames of them): every 600, the report void gprof_report(int frames); +/// A span of a scope: closed by the destructor with its label (items and bytes set while it is open) +struct GprofSpan { + VkCommandBuffer cmd; + const char* label; + int items = 1; + uint64_t bytes = 0; + bool on = false; + GprofSpan(VkCommandBuffer c, const char* l) : cmd(c), label(l) { + if (gprof_on()) on = gprof_begin(c); + } + ~GprofSpan() { + if (on) gprof_end(cmd, label, items, bytes); + } + GprofSpan(const GprofSpan&) = delete; + GprofSpan& operator=(const GprofSpan&) = delete; +}; + } // namespace mkwvk diff --git a/gpu/vk/vk_internal.h b/gpu/vk/vk_internal.h index fee075928..86ab6bea9 100644 --- a/gpu/vk/vk_internal.h +++ b/gpu/vk/vk_internal.h @@ -160,8 +160,9 @@ uint32_t memory_page_stamp(uint64_t gpu); // (that address's page only) uint64_t gpu_pool_end(uint64_t gpu); // (hookshot) where the memory pool holding that address ends, 0: in none (vk_memory.cpp) uint64_t gpu_pool_start(uint64_t gpu); // (EvilOOT) and where it begins bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, - void** mapped); // (vk_vma.cpp: cached host memory, mapped) + void** mapped, bool dedicated = false); // (vk_vma.cpp: cached host memory, mapped) void flush_mapped(VmaAllocation allocation, uint64_t size); +void invalidate_mapped(VmaAllocation allocation, uint64_t size); // (the CPU is to read what the GPU wrote: a no-op in coherent memory) void memory_stamp_gpu(uint64_t guest, uint64_t size); // (a GPU write: memory_stamp, and the pages noted) void memory_protect_gpu(uint64_t guest, uint64_t size); // (RX_VK_PROTECT_BARE=1: pages a shader writes through a table pointer) /// (vk_replay.cpp) guest memory as the stream's place sees it: as submitted, the stream's uniform updates before it over it @@ -200,13 +201,17 @@ bool has_stencil(VkFormat f); // device memory (vk_vma.cpp, Vulkan Memory Allocator) bool vma_init(PFN_vkGetInstanceProcAddr gipa); -bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation); +bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated = false); void destroy_image(VkImage image, VmaAllocation allocation); +/// What an allocation really takes (VkMemoryRequirements.size: padding, alignment, the driver's metadata), 0 for none +uint64_t allocation_size(VmaAllocation allocation); +/// (dedicated: its own memory object, not a share of a 64 MB block: for the 16 to 32 MB ones that fit two to a block and leave +/// the rest of it free; RX_VK_BLOCK_BUFFERS=1: as before) bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, VkBuffer& buffer, - VmaAllocation& allocation, void** mapped); + VmaAllocation& allocation, void** mapped, bool dedicated = false); void destroy_buffer(VkBuffer buffer, VmaAllocation allocation); /// A buffer in VRAM only (within the driver's budget), or false -bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation); +bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, bool dedicated = false); /// Device-local memory this process uses and what the driver allows it now (VK_EXT_memory_budget), summed over the /// device-local heaps; false when unknown bool vram_usage(uint64_t* used, uint64_t* budget, uint64_t* block_free = nullptr); diff --git a/gpu/vk/vk_memory.cpp b/gpu/vk/vk_memory.cpp index 689807b2b..4030b0bd1 100644 --- a/gpu/vk/vk_memory.cpp +++ b/gpu/vk/vk_memory.cpp @@ -27,7 +27,9 @@ #include "vk_internal.h" #include "vk_texture.h" // (upload_command_buffer) +#include "vk_gprof.h" // (RX_VK_PASS_TIMING) #include "import_hold.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { namespace { @@ -260,9 +262,11 @@ bool memory_table_flush() { std::stable_sort(pages.begin(), pages.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); std::vector run; VkCommandBuffer cmd = upload_command_buffer(); + GprofSpan gp(cmd, "ptable update (guest page table)"); + gp.items = 0; uint64_t run_first = 0; const auto emit = [&] { - if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()); + if (!run.empty()) vkCmdUpdateBuffer(cmd, guest_pages_buffer, run_first * 8, run.size() * 8, run.data()), gp.items++, gp.bytes += run.size() * 8; run.clear(); }; for (size_t i = 0; i < pages.size(); i++) { @@ -287,7 +291,7 @@ uint64_t memory_guest_pages() { void* mapped = nullptr; const uint64_t bytes = ((8ull << 30 >> 12) + (4ull << 30 >> 12)) * 8; if (!create_buffer(bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, - true, b, a, &mapped) || !mapped) { + true, b, a, &mapped, true) || !mapped) { logf("vk: no page table of guest memory: pointers in no bound buffer read as 0"); return 0; } @@ -588,16 +592,40 @@ bool sync_live() { return v; } std::mutex cap_free_mu; // (the submitting threads take buffers from cap_free, the replay gives them back) +uint64_t cap_free_bytes = 0; // (cap_free_mu) +// (the spare buffers: 12 and, on a card of 6 GB or less, 32 MB of them; RX_VK_CAP_SPARE_MB=: that many, 0: as before. +// After a load the retired ones, 40-60 MB each, stayed as spares for good) +uint64_t cap_spare_max() { + static const uint64_t v = [] { + if (const char* e = getenv("RX_VK_CAP_SPARE_MB")) return e[0] && atoll(e) ? (uint64_t)atoll(e) << 20 : ~0ull; + uint64_t used = 0, allowed = 0; + return vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 32ull << 20 : ~0ull; + }(); + return v; +} void cap_buf_free(VkBuffer buffer, VmaAllocation alloc, void* map, uint64_t size) { std::lock_guard lk(cap_free_mu); - if (cap_free.size() < 12 && size <= (64ull << 20)) cap_free.push_back({buffer, alloc, map, size}); - else destroy_buffer(buffer, alloc); + if (cap_free.size() < 12 && size <= (64ull << 20) && cap_free_bytes + size <= cap_spare_max()) { + cap_free.push_back({buffer, alloc, map, size}); + cap_free_bytes += size; + } else { + destroy_buffer(buffer, alloc); + } } void cap_buf_retire(std::map::iterator it) { cap_bytes -= it->second.size; cap_buf_free(it->second.buffer, it->second.alloc, it->second.map, it->second.size); cap_bufs.erase(it); } +// (a capture's buffer by its size class: 2, 4, 8, then 16 MB and up to the MB. It was never under 16 MB, for captures of ~1 MB: +// 25 kept ones were 413 MB of host-cached system memory, all resident, and three to a 64 MB block left 14 MB of each unused. +// RX_VK_CAP_MIN_MB=: the smallest (2), 16: as before. From 16 MB each has a memory object of its own: RX_VK_CAP_BLOCKS=1: not) +uint64_t cap_buf_size(uint64_t need) { + static const uint64_t min = (getenv("RX_VK_CAP_MIN_MB") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_CAP_MIN_MB"))) : 2u) << 20; + uint64_t s = min; + while (s < need && s < (16ull << 20)) s <<= 1; + return s >= need ? s : (need + (1u << 20) - 1) & ~((1ull << 20) - 1); +} bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& map, uint64_t& size) { std::lock_guard lk(cap_free_mu); size_t best = cap_free.size(); @@ -605,12 +633,14 @@ bool cap_buf_get(uint64_t need, VkBuffer& buffer, VmaAllocation& alloc, void*& m if (cap_free[i].size >= need && (best == cap_free.size() || cap_free[i].size < cap_free[best].size)) best = i; if (best < cap_free.size()) { buffer = cap_free[best].buffer, alloc = cap_free[best].alloc, map = cap_free[best].map, size = cap_free[best].size; + cap_free_bytes -= size; cap_free.erase(cap_free.begin() + (ptrdiff_t)best); return map != nullptr; } - size = std::max((need + (1u << 20) - 1) & ~((1ull << 20) - 1), 16ull << 20); + size = cap_buf_size(need); map = nullptr; - return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map) && map; + return create_buffer_cached(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, buffer, alloc, &map, + size >= (16ull << 20) && !getenv("RX_VK_CAP_BLOCKS")) && map; } std::vector> cap_ranges; // (the mirrored pools the captures take, as of the last one) std::mutex cap_ranges_mu; @@ -1197,6 +1227,17 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed if (!c->pages.empty() && tex_capture_on()) { // (the texture storage it took: the shadow, as of this submit) std::lock_guard sk(shadow_mu); shadow_seq++; + // (the copies of pages no capture took in the last 256 submits go, over 32 MB of them, every 256 captures: such a page is what guest + // memory holds (a write drops its copy), and the cap's sweep below came only at 256 MB. RX_VK_SHADOW_SWEEP_OFF=1: the cap's alone) + static const bool sweep_off = getenv("RX_VK_SHADOW_SWEEP_OFF") != nullptr; + if (!sweep_off && shadow_seq % 256 == 0 && shadow_n.load(std::memory_order_relaxed) > 8192) + for (auto it = shadow.begin(); it != shadow.end();) { + if (it->second.seq + 256 >= shadow_seq) { ++it; continue; } + auto r = shadow_regions.find(it->first >> 9); + if (r != shadow_regions.end() && --r->second == 0) shadow_regions.erase(r); + it = shadow.erase(it); + shadow_n--; + } if (!tex_merged.empty()) { // (every page written since the last capture: its old copy is stale; the ones copied now replace it) for (uint32_t p : c->pages) @@ -1250,8 +1291,9 @@ bool take_capture(std::unique_ptr c, uint64_t value, uint64_t completed st_tk_all += std::chrono::duration(tk2 - tk0).count(); if (++n % 600 == 0) { logf("vk: a capture taken (ms): all %.3f; retires and stamps %.3f, bytes copied %.3f; %zu kept (%llu MB), %zu " - "spare buffers", st_tk_all / 600, st_tk_stamp / 600, st_tk_copy / 600, cap_bufs.size(), - (unsigned long long)(cap_bytes >> 20), cap_free.size()); + "spare buffers, %zu texture pages copied (%llu MB)", st_tk_all / 600, st_tk_stamp / 600, st_tk_copy / 600, cap_bufs.size(), + (unsigned long long)(cap_bytes >> 20), cap_free.size(), shadow_n.load(std::memory_order_relaxed), + (unsigned long long)(shadow_n.load(std::memory_order_relaxed) >> 8)); st_tk_all = st_tk_stamp = st_tk_copy = 0; } } @@ -1658,17 +1700,11 @@ int64_t st_sync_bytes, st_sync_copies, st_wb_bytes, st_mirrors; int64_t st_sync_live_pages, st_sync_cap_pages, st_sync_old_pages; // (mirror syncs' dirty pages: live / this submit's capture / an earlier one's) std::unordered_map live_by_pool; std::unordered_map> direct_by_pool; // (bindings the GPU reads in guest memory: count, state) +std::unordered_map direct_bytes_by_pool; // (and the bytes of those bindings: what the GPU may read over PCIe, a binding's whole range) int64_t live_seen = 0; // (pool -> pages its syncs read live, since the last report) void transfer_barrier(VkCommandBuffer cmd) { - VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; - b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; - b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT; - b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT; - VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; - d.memoryBarrierCount = 1; - d.pMemoryBarriers = &b; - vkCmdPipelineBarrier2(cmd, &d); + full_barrier(cmd, "barrier (write-back or sync)"); // (the same barrier; one right after another is left out) } /// The pages of the pool the CPU wrote since its mirror's sync, copied in (the upload command buffer) @@ -1742,14 +1778,18 @@ void mirror_sync(Pool& p) { } if (run_hi) runs.push_back({run_lo, run_lo, run_hi - run_lo}); for (const auto& [buf, rs] : cap_runs) { + GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages from a capture -> mirror)"); + gp.items = (int)rs.size(); vkCmdCopyBuffer(upload_command_buffer(), buf, p.mirror, (uint32_t)rs.size(), rs.data()); - for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size; + for (const VkBufferCopy& r : rs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size; st_sync_copies += (int64_t)rs.size(); } if (runs.empty()) return; checkpoint_mark(upload_command_buffer(), p.lo, 'm'); + GprofSpan gp(upload_command_buffer(), "msync copies (CPU pages live from guest memory -> mirror)"); + gp.items = (int)runs.size(); vkCmdCopyBuffer(upload_command_buffer(), p.buffer, p.mirror, (uint32_t)runs.size(), runs.data()); - for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size; + for (const VkBufferCopy& r : runs) st_sync_bytes += (int64_t)r.size, gp.bytes += r.size; st_sync_copies += (int64_t)runs.size(); } @@ -1791,9 +1831,12 @@ bool memory_rb_refresh(VkCommandBuffer up) { } if (by_pool.empty()) return false; full_barrier(up); // (after the older submissions' write-backs into guest memory and this one's other uploads) + GprofSpan gp(up, "rbrefresh copies (readback pages guest -> mirror)"); + gp.items = 0; for (auto& [pp, rs] : by_pool) { vkCmdCopyBuffer(up, pp->buffer, pp->mirror, (uint32_t)rs.size(), rs.data()); st_rb_refresh += (int64_t)rs.size(); + gp.items += (int)rs.size(), gp.bytes += rs.size() * 4096; } static std::atomic n{0}; if (++n % 3000 == 1) logf("vk: readback pages refreshed from guest memory: %lld pages so far", (long long)st_rb_refresh.load()); @@ -1978,7 +2021,7 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (p->last_value < mem_rec_value) p->last_value = mem_rec_value; const uint64_t off = guest - p->lo; if (!mirror_on() || !mirror_upload || p->mstate == Pool::MIRROR_REFUSED) { - if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; } + if (mirror_on()) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = mirror_upload ? (int)p->mstate : 9; direct_bytes_by_pool[p->lo] += size; } return remember({p->buffer, off, size, 0}); } // (a pool's mirror is made at its first binding, in this part's uploads: bound in guest memory until the next @@ -2001,6 +2044,7 @@ BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written) { if (p->mstate != Pool::MIRROR_ON) { auto& e = direct_by_pool[p->lo]; e.first++, e.second = (int)p->mstate; + direct_bytes_by_pool[p->lo] += size; if (written) p->direct_write_epoch = mirror_epoch; if (written && size <= (1u << 20)) // (EvilBoyce: GPU results written in place; small ones: the readbacks are) wb_pages_note(p->lo + off, p->lo + std::min(off + size, p->hi - p->lo), mem_rec_value); @@ -2202,7 +2246,11 @@ bool mirror_make(Pool& pool) { (unsigned long long)ma, (unsigned long long)(ma + n)); } const VkBufferCopy all{0, 0, n}; - vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation) + { + GprofSpan gp(upload_command_buffer(), "mmake copy (a whole pool -> its new mirror)"); + gp.bytes = n; + vkCmdCopyBuffer(upload_command_buffer(), p->buffer, p->mirror, 1, &all); // (all of it, at this generation) + } { // (EvilOOT) then the mirrors of the pools it took in, over their part: what the GPU wrote there and only there // (retire_pool). The captures' pages go over it after, as before: the CPU's newer writes win bool barrier_done = false; @@ -2313,13 +2361,19 @@ void memory_end_submission(VkCommandBuffer cmd) { for (size_t i = 0; i < dtop.size() && i < 8; i++) { const auto it = imported.find(dtop[i].second); char b[160]; - snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld;", (unsigned long long)dtop[i].second, + snprintf(b, sizeof b, " %#llx (%llu MB, state %d): %lld binds, %.0f MB;", (unsigned long long)dtop[i].second, it != imported.end() ? (unsigned long long)((it->second.hi - it->second.lo) >> 20) : 0ull, - direct_by_pool[dtop[i].second].second, (long long)dtop[i].first); + direct_by_pool[dtop[i].second].second, (long long)dtop[i].first, + direct_bytes_by_pool[dtop[i].second] / 1048576.0); ds += b; } - if (!dtop.empty()) logf("vk: bound in guest memory, by pool:%s", ds.c_str()); - direct_by_pool.clear(); + if (!dtop.empty()) { + double all = 0; + for (const auto& [lo, bytes] : direct_bytes_by_pool) all += (double)bytes; + logf("vk: bound in guest memory, by pool (the GPU reads these over PCIe; binds and the bytes of their ranges over these 600 submits, " + "%.1f MB in all, a range counted once a submit): %s", all / 1048576.0, ds.c_str()); + } + direct_by_pool.clear(), direct_bytes_by_pool.clear(); } } static const bool no_writeback = getenv("RX_VK_NO_WRITEBACK") != nullptr; // (an experiment: none) @@ -2329,6 +2383,8 @@ void memory_end_submission(VkCommandBuffer cmd) { std::sort(writebacks.begin(), writebacks.end(), [](const WriteBack& a, const WriteBack& b) { return a.pool != b.pool ? a.pool < b.pool : a.lo < b.lo; }); + GprofSpan gp(cmd, "wb copies (mirror -> guest memory, with its two barriers)"); // (RX_VK_PASS_TIMING) + gp.items = 0; transfer_barrier(cmd); std::vector runs; for (size_t i = 0; i < writebacks.size();) { @@ -2345,7 +2401,8 @@ void memory_end_submission(VkCommandBuffer cmd) { if (runs.empty()) continue; checkpoint_mark(cmd, p->lo, 'b'); vkCmdCopyBuffer(cmd, p->mirror, p->buffer, (uint32_t)runs.size(), runs.data()); - for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size; + for (const VkBufferCopy& r : runs) st_wb_bytes += (int64_t)r.size, gp.bytes += r.size; + gp.items += (int)runs.size(); for (const VkBufferCopy& r : runs) wb_pages_note(p->lo + r.dstOffset, p->lo + r.dstOffset + r.size, mem_rec_value); // (EvilBoyce) { // (hookshot) RX_VK_LOG_WRITEBACK=1: the pools written back the most, every 600 write-back submissions static const bool log_wb = getenv("RX_VK_LOG_WRITEBACK") != nullptr; diff --git a/gpu/vk/vk_menu.cpp b/gpu/vk/vk_menu.cpp index 3bfd0baf9..ed5569d94 100644 --- a/gpu/vk/vk_menu.cpp +++ b/gpu/vk/vk_menu.cpp @@ -14,6 +14,7 @@ #include "vk_paths.h" #include "vk_menu.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) #ifndef HOOKSHOT_VERSION // (nx2/build.py gives it to this file: nx2/version.py) #define HOOKSHOT_VERSION "unknown" #endif diff --git a/gpu/vk/vk_replay.cpp b/gpu/vk/vk_replay.cpp index af0079fe7..07d9f5f4e 100644 --- a/gpu/vk/vk_replay.cpp +++ b/gpu/vk/vk_replay.cpp @@ -38,6 +38,7 @@ #include "vk_present_rect.h" #include "vk_internal.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { extern std::atomic movie_networks_slow; // (vk_draw.cpp: the movie networks left out, issue 109) @@ -143,6 +144,17 @@ bool batch_on() { }(); return on && batch_tl != VK_NULL_HANDLE; } +int par_threads(); +/// (hookshot) How many of the recording slots the ring runs through: 24 (what it was before the 48) unless submits are batched +/// (they wait in the batch for a slot's turn) or a parallel replay takes a slot for each segment. Each slot holds a 24 MB stream +/// the draw path writes through the BAR, in VRAM on a card with resizable BAR (vk_draw.cpp: VMA puts a host-visible buffer the +/// shaders read in DEVICE_LOCAL | HOST_VISIBLE), and a descriptor pool: 48 streams were 1152 MB of a 6 GB card, 1536 MB with the +/// 16 MB VMA left free in each 64 MB block around two of them; the game waits for the last frame's GPU work before it submits the +/// next, so no more than ~18 slots are ever in flight. RX_VK_RECORDINGS_48=1: all 48, as before +int recording_slots() { + static const int n = (getenv("RX_VK_RECORDINGS_48") || batch_on() || par_threads() > 1) ? kRecordings : kRecordings / 2; + return n; +} } // namespace void queue_flush_locked() { // (dev.present_mu held) if (pend.cbs.empty()) return; @@ -259,7 +271,7 @@ void gpu_time_end(Recording& r) { /// The next recording slot, begun, for the submission that will signal value Recording& begin_recording(uint64_t value) { Recording& r = recordings[recording_index]; - recording_index = (recording_index + 1) % kRecordings; + recording_index = (recording_index + 1) % recording_slots(); if (r.value) { PROF(PROF_SLOTWAIT); const auto t0 = std::chrono::steady_clock::now(); @@ -275,7 +287,7 @@ Recording& begin_recording(uint64_t value) { vkBeginCommandBuffer(r.cmd, &bi); vkBeginCommandBuffer(r.upload, &bi); gpu_time_begin(r); - gprof_begin_recording(r.slot, r.cmd); + gprof_begin_recording(r.slot, r.cmd, r.upload); r.value = value; memory_recording_value(value, completed()); texture_begin_recording(r.upload, value); @@ -284,6 +296,7 @@ Recording& begin_recording(uint64_t value) { } void submit_recording(Recording& r, bool signal_value = true, uint64_t signal_at = 0) { const bool uploads = texture_uploads_recorded(); + gprof_submitted(r.slot, uploads); gpu_time_end(r); vkEndCommandBuffer(r.upload); vkEndCommandBuffer(r.cmd); @@ -362,6 +375,7 @@ void use_queue_state(uint64_t queue) { static uint64_t gen_base = 0; gen_base += 1ull << 32; s->bind_gen = gen_base, s->rt_serial = gen_base; + for (int k = 0; k < kNvnStages; k++) s->tex_gen[k] = gen_base, s->ssbo_gen[k] = gen_base; } template @@ -372,6 +386,8 @@ T payload(const uint8_t* a, uint32_t bytes) { } int64_t st_barriers, st_ub_updates, st_ub_overlays, st_submits; +std::map st_barrier_flags; // (the game's nvnCommandBufferBarrier bits: how many of each combination, the frame log) +std::mutex st_barrier_flags_mu; // (a parallel replay's threads count their barriers too) // ---- events in replay order: the values the stream's signals and the CPU's set, for the GPU's waits on them. The // game's three queues order their work with events (a command buffer waits for another queue's to reach a stage); // replayed on one queue in the order they were submitted, a waiter submitted before its signaller ran first ---- @@ -630,9 +646,11 @@ namespace { /// their overlay copies) void flush_ub_updates(VkCommandBuffer cmd) { if (ub_first_of_recording >= ub_updates.size()) return; + GprofSpan gp(cmd, "ubupd (the stream's uniform updates -> guest memory, with its barriers)"); + gp.items = (int)(ub_updates.size() - ub_first_of_recording); full_barrier(cmd); for (size_t i = ub_first_of_recording; i < ub_updates.size(); i++) - gpu_write(cmd, ub_updates[i].dst, ub_updates[i].bytes.data(), ub_updates[i].bytes.size()); + gpu_write(cmd, ub_updates[i].dst, ub_updates[i].bytes.data(), ub_updates[i].bytes.size()), gp.bytes += ub_updates[i].bytes.size(); full_barrier(cmd); } @@ -688,6 +706,7 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* v = old + e.value; } end_pass(cmd); + GprofSpan gp(cmd, "event signal (barrier, write, barrier)"); full_barrier(cmd); gpu_write(cmd, e.gpu, &v, 4); break; @@ -772,6 +791,8 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* dst ? "bound" : "NOT bound (on the CPU)"); } end_pass(cmd); + GprofSpan gp(cmd, "copy buffer->buffer (with its barriers)"); + gp.bytes = c.size; full_barrier(cmd); if (src && dst) { const VkBufferCopy r{src.offset, dst.offset, c.size}; @@ -816,6 +837,8 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* (unsigned long long)c.size, (unsigned)c.value, dst ? "on the GPU" : "NOT bound"); } end_pass(cmd); + GprofSpan gp(cmd, "clear buffer (with its barriers)"); + gp.bytes = c.size; full_barrier(cmd); if (dst && !(dst.offset & 3) && !(c.size & 3)) { vkCmdFillBuffer(cmd, dst.buffer, dst.offset, c.size, c.value); @@ -956,8 +979,10 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* (unsigned long long)b.r[i].address, (unsigned long long)b.r[i].size); } nvn_p_bind_buf& u = (b.ssbo ? s.ssbo : s.ubo)[b.stage][index]; - if (!buf_unchanged(u.stage, u.index, u.addr, u.size, b.stage, index, b.r[i].address, b.r[i].size)) + if (!buf_unchanged(u.stage, u.index, u.addr, u.size, b.stage, index, b.r[i].address, b.r[i].size)) { s.bind_gen++; + if (b.ssbo) s.ssbo_gen[b.stage] = s.bind_gen; + } u.stage = b.stage, u.index = index, u.addr = b.r[i].address, u.size = b.r[i].size; } break; @@ -974,7 +999,10 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* } if (b.stage >= 0 && b.stage < kNvnStages && b.index >= 0 && b.index < 16) { nvn_p_bind_buf& u = (h.op == NOP_BIND_UBO ? s.ubo : s.ssbo)[b.stage][b.index]; - if (!buf_unchanged(u.stage, u.index, u.addr, u.size, b.stage, b.index, b.addr, b.size)) s.bind_gen++; + if (!buf_unchanged(u.stage, u.index, u.addr, u.size, b.stage, b.index, b.addr, b.size)) { + s.bind_gen++; + if (h.op == NOP_BIND_SSBO) s.ssbo_gen[b.stage] = s.bind_gen; + } u = b; } break; @@ -994,7 +1022,10 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* const int lim = h.op == NOP_BIND_IMG ? 32 : 128; int n = 0; while (n < b.n && b.first + n < lim) n++; - if (!handles_unchanged(to + b.first, b.h, n)) s.bind_gen++; // (the same handles: the draw keeps its descriptors) + if (!handles_unchanged(to + b.first, b.h, n)) { // (the same handles: the draw keeps its descriptors) + s.bind_gen++; + s.tex_gen[b.stage] = s.bind_gen; + } for (int k = 0; k < n; k++) to[b.first + k] = b.h[k]; break; } @@ -1051,7 +1082,10 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* s.render_enabled ? "on" : "off"); break; } - case NOP_SET_TEX_POOL: case NOP_SET_SAMP_POOL: s.bind_gen++; break; + case NOP_SET_TEX_POOL: case NOP_SET_SAMP_POOL: + s.bind_gen++; + for (int k = 0; k < kNvnStages; k++) s.tex_gen[k] = s.bind_gen; + break; case NOP_COPY_STRIDE: s.copy_stride = payload(a, avail).stride; break; case NOP_PUSH_CONST: { // (hookshot) compute push constants, kept in the state (vk_bank.cpp serves them from c0) const auto u = payload(a, avail); @@ -1066,8 +1100,13 @@ void replay(VkCommandBuffer& cmd, NvnState& s, const uint8_t* p, const uint8_t* } case NOP_BARRIER: if (dry) break; + { + const int32_t bits = payload(a, avail); + std::lock_guard lk(st_barrier_flags_mu); + st_barrier_flags[bits]++; + } end_pass(cmd); - full_barrier(cmd); + full_barrier(cmd, "barrier (the game's nvnCommandBufferBarrier)"); st_barriers++; break; // ---- work ---- @@ -1183,12 +1222,12 @@ void maybe_split(VkCommandBuffer& cmd) { end_pass(cmd); flush_ub_updates(cmd); memory_end_submission(cmd); - full_barrier(cmd); + full_barrier(cmd, "barrier (recording end)"); submit_recording(*rec_cur, false); ub_first_of_recording = ub_updates.size(); Recording& r = begin_recording(rec_cur_value); memory_begin_submission(); - full_barrier(r.cmd); + full_barrier(r.cmd, "barrier (recording start)"); rec_cur = &r; cmd = r.cmd; part_work = 0; @@ -1232,7 +1271,7 @@ void par_worker_main() { /// A recording slot for a segment a worker records (its command buffer only: uploads go to the shared one) Recording& begin_segment_recording(uint64_t value) { Recording& r = recordings[recording_index]; - recording_index = (recording_index + 1) % kRecordings; + recording_index = (recording_index + 1) % recording_slots(); if (r.value) { PROF(PROF_SLOTWAIT); recording_wait(r); @@ -1700,7 +1739,7 @@ void finish_recording(uint64_t value) { end_pass(r.cmd); flush_ub_updates(r.cmd); memory_end_submission(r.cmd); - full_barrier(r.cmd); + full_barrier(r.cmd, "barrier (recording end)"); const uint64_t oldest = oldest_unfinished(); const uint64_t at = oldest == UINT64_MAX ? done_max : std::min(done_max, oldest - 1); const bool signal = at > signaled; @@ -1723,7 +1762,7 @@ bool run_submit(const uint64_t* handles, int n, uint64_t value, uint64_t queue, } if (burst_open && rec_cur) { // (the burst's recording goes on: what the piece before did, done first) end_pass(rec_cur->cmd); - full_barrier(rec_cur->cmd); + full_barrier(rec_cur->cmd, "barrier (a burst's next submit start)"); use_queue_state(queue); } else { const uint64_t tag = burst_open ? std::max(burst_value, value) : value; @@ -1733,7 +1772,7 @@ bool run_submit(const uint64_t* handles, int n, uint64_t value, uint64_t queue, ub_first_of_recording = ub_updates.size(); Recording& first = begin_recording(tag); memory_begin_submission(); - full_barrier(first.cmd); + full_barrier(first.cmd, "barrier (recording start)"); rec_cur = &first, rec_cur_value = tag, rec_parts = 0, part_work = 0; } VkCommandBuffer cmd = rec_cur->cmd; @@ -2395,6 +2434,23 @@ void log_frame_stats() { (st_ub_overlays - l_ubo) / 120.0, (st_wait_ns - l_wait) / 120.0 / 1e6, (long long)pl, pms, (long long)tr, (long long)fl, texture_count(), (unsigned long long)(texture_bytes() >> 20)); draw_skip_report(120.0); + { // (hookshot: the full barriers recorded and the ones left out as the second of two in a row, RX_VK_BARRIER_DEDUPE_OFF=1) + static int64_t l_made, l_left; + int64_t made = 0, left = 0; + barrier_stats(&made, &left); + logf("vk: a frame: %.1f full barriers recorded, %.1f left out (nothing recorded since the one before)", (made - l_made) / 120.0, + (left - l_left) / 120.0); + l_made = made, l_left = left; + std::string fl; // (NVN's barrier bits: ORDER_PRIMITIVES 1, ORDER_FRAGMENTS 2, ORDER_FRAGMENTS_TILED 4, ORDER_INDIRECT_DATA 8, INVALIDATE_TEXTURE 0x10, _SHADER 0x20, _TEXTURE_DESCRIPTOR 0x40, _ZCULL 0x80) + std::lock_guard lk(st_barrier_flags_mu); + for (const auto& [bits, n] : st_barrier_flags) { + char b[48]; + snprintf(b, sizeof b, " %#x: %.1f;", (unsigned)bits, n / 120.0); + fl += b; + } + logf("vk: a frame: the game's barriers by their NVN flags:%s", fl.c_str()); + st_barrier_flags.clear(); + } static int64_t l_gpu; static uint64_t l_made; logf("vk: GPU time %.1f ms a frame (the longest submission %.1f ms); %.1f textures made a frame", diff --git a/gpu/vk/vk_shader.cpp b/gpu/vk/vk_shader.cpp index 800862d72..5f00eb541 100644 --- a/gpu/vk/vk_shader.cpp +++ b/gpu/vk/vk_shader.cpp @@ -1606,6 +1606,27 @@ uint64_t prog_stage_sig(const nvn_prog& p) { for (int i = 0; i < 6; i++) h = h * 1099511628211ull ^ p.sd[i].data ^ (p.sd[i].control << 1); return h ^ p.serial; } +// (hookshot) The stages whose bindings a graphics program's answers are read from, one bit a stage, when each answer is a handle in c2 of the +// stage itself (a handle program reads uniform memory, another bank other state): then the pick stands while those stages' textures, +// samplers and images do (NvnState::tex_gen), whatever else was bound. ~0: not so, the pick stands while nothing at all was bound anew +uint32_t answer_stage_mask(const Program& p) { + uint32_t mask = 0; + for (const Program::Answer& a : p.answers) { + const Stage& st = p.stages[a.stage]; + if (st.nvn_stage < 0 || st.nvn_stage > 4) return ~0u; + bool c2; + if (a.image) { + const auto& d = st.info.image_descriptors[a.desc]; + c2 = d.handle_program == ~0U && d.cbuf_index == 2; + } else { + const auto& d = st.info.texture_descriptors[a.desc]; + c2 = d.handle_program == ~0U && d.cbuf_index == 2 && (!d.has_secondary || d.secondary_cbuf_index == 2); + } + if (!c2) return ~0u; + mask |= 1u << st.nvn_stage; + } + return mask; +} struct ProgFast { bool on = false; int kind = 0; @@ -1614,6 +1635,8 @@ struct ProgFast { uint32_t vattr[16]{}; uint8_t ct[8]{}; const Program* p = nullptr; + uint32_t amask = ~0u; // (answer_stage_mask of p, ~0 for a compute program) + uint64_t tex_gen[kNvnStages]{}; // (the stages' tex_gen it was kept at) }; thread_local ProgFast prog_fast[2]; @@ -1621,17 +1644,32 @@ const Program* program_for(const NvnState& s, ProgramKind kind, const uint8_t co if (!s.has_prog) return nullptr; if (kind == ProgramKind::Mesh) return mesh_program_for(s, color_types); const bool compute = kind == ProgramKind::Compute; - static int64_t st_hit, st_none, st_differ, st_calls; // (logged now and then: how the cache does) + static int64_t st_hit, st_none, st_differ, st_calls, st_by_tex; // (logged now and then: how the cache does) if (++st_calls % 200000 == 0) - logf("vk: program picks: %lld kept, %lld new, %lld with other textures (of %lld)", (long long)st_hit, - (long long)st_none, (long long)st_differ, (long long)st_calls); + logf("vk: program picks: %lld kept (%lld by the last picks' textures, a uniform buffer bound since), %lld new, %lld with other textures (of %lld)", + (long long)st_hit, (long long)st_by_tex, (long long)st_none, (long long)st_differ, (long long)st_calls); const uint64_t stages = prog_stage_sig(s.prog); const uint64_t bind = s.bind_gen, reg = registrations_generation(), ub = ub_generation(); - for (const ProgFast& f : prog_fast) // (the draw before this one, and the one that took turns with it) - if (f.on && f.p && f.p->ready.load(std::memory_order_acquire) && !f.p->failed && - prog_same((int)kind, s.prog.serial, stages, bind, reg, ub, s.nvattr, s.vattr, color_types, f.kind, f.serial, - f.stages, f.bind, f.reg, f.ub, f.nvattr, f.vattr, f.ct)) + // (hookshot) the pair of the last picks stood only while nothing at all was bound anew, and OoT binds a uniform buffer for nearly every + // draw: a draw of the same program with the same textures found its program through the full pick (the key, the answers' handles, a + // compare). It stands now while the textures its answers are read from do (answer_stage_mask). RX_VK_PICK_BIND_GEN=1: as before + static const bool by_bind = getenv("RX_VK_PICK_BIND_GEN") != nullptr; + for (const ProgFast& f : prog_fast) { // (the draw before this one, and the one that took turns with it) + if (!(f.on && f.p && f.p->ready.load(std::memory_order_acquire) && !f.p->failed)) continue; + uint64_t fbind = f.bind; + bool by_tex = false; + if (fbind != bind && !by_bind && !compute && f.amask != ~0u) { + by_tex = true; + for (int k = 0; k < kNvnStages; k++) + if ((f.amask >> k & 1) && f.tex_gen[k] != s.tex_gen[k]) by_tex = false; + if (by_tex) fbind = bind; + } + if (prog_same((int)kind, s.prog.serial, stages, bind, reg, ub, s.nvattr, s.vattr, color_types, f.kind, f.serial, + f.stages, fbind, f.reg, f.ub, f.nvattr, f.vattr, f.ct)) { + st_by_tex += by_tex && f.bind != bind; return st_hit++, f.p; + } + } const uint64_t key = (compute ? 0x5EED : pick_inputs(s, color_types)) * 1099511628211ull ^ s.prog.serial; Pick& pk = picks[key]; auto keep = [&](const Program* p) { @@ -1640,6 +1678,8 @@ const Program* program_for(const NvnState& s, ProgramKind kind, const uint8_t co ProgFast& f = prog_fast[0]; f.on = true, f.kind = (int)kind, f.serial = s.prog.serial, f.stages = stages, f.bind = bind, f.reg = reg, f.ub = ub; f.nvattr = s.nvattr, f.p = p; + f.amask = compute ? ~0u : answer_stage_mask(*p); + std::memcpy(f.tex_gen, s.tex_gen, sizeof f.tex_gen); std::memcpy(f.vattr, s.vattr, sizeof f.vattr); std::memcpy(f.ct, color_types, sizeof f.ct); return p; diff --git a/gpu/vk/vk_shader.h b/gpu/vk/vk_shader.h index 589e80fd4..7e4aa7730 100644 --- a/gpu/vk/vk_shader.h +++ b/gpu/vk/vk_shader.h @@ -69,6 +69,7 @@ struct Program { }; std::vector answers; uint64_t variant = 0; // (a hash of the answers: key alone names all the variants of these stages) + mutable std::atomic wm_sig{-1}; // (hookshot: what its write masks are made from, vk_draw.cpp program_wm_sig; -1: not asked yet) }; enum class ProgramKind { Graphics, Compute, Mesh }; diff --git a/gpu/vk/vk_state.h b/gpu/vk/vk_state.h index c1e84d064..d61430437 100644 --- a/gpu/vk/vk_state.h +++ b/gpu/vk/vk_state.h @@ -29,6 +29,9 @@ struct NvnState { nvn_p_set_rt rt{}; uint64_t rt_serial = 0; // bumped at each NOP_SET_RT (the draw path resolves the targets once per change) uint64_t bind_gen = 0; // bumped when a binding the descriptors read changes (a rebind of the same values does not) + // (hookshot) what bind_gen was when a stage's textures, samplers or images (tex_gen) or its storage buffers (ssbo_gen) last changed: the + // driver banks c2 and c0 of a graphics stage are made from those alone, so a draw after another uniform buffer's bind keeps its banks + uint64_t tex_gen[kNvnStages] = {}, ssbo_gen[kNvnStages] = {}; float viewport[16][4]{}; float depth_near = 0.0f, depth_far = 1.0f; int32_t scissor[4]{0, 0, 0, 0}; diff --git a/gpu/vk/vk_texture.cpp b/gpu/vk/vk_texture.cpp index 77c514952..359c312f4 100644 --- a/gpu/vk/vk_texture.cpp +++ b/gpu/vk/vk_texture.cpp @@ -16,6 +16,9 @@ #endif #include #include +#if defined(__linux__) +#include +#endif #include #include #include @@ -30,7 +33,9 @@ #include "vk_texture.h" #include "vk_menu.h" #include "vk_draw.h" +#include "vk_gprof.h" #include "vk_dump.h" +#include "vk_jobs.h" #include @@ -40,6 +45,7 @@ #include "host_shaders/astc_decoder_comp_spv.h" #include "host_shaders/bc_encoder_comp_spv.h" #include "host_shaders/block_linear_unswizzle_2d_comp_spv.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { namespace { @@ -181,6 +187,8 @@ struct Dead { VmaAllocation allocation; std::vector views; std::vector memory; // (a sparse image's) + VkBuffer buffer = VK_NULL_HANDLE; // (the BC3 scratch's output buffer: given back with its image) + VmaAllocation buffer_alloc = nullptr; }; std::deque dead; @@ -206,6 +214,7 @@ struct Arena { VmaAllocation allocation = nullptr; uint8_t* mapped = nullptr; uint64_t size = 0, used = 0, value = 0; // value: the last submission that reads it + std::chrono::steady_clock::time_point used_at; // (when it last staged something) }; std::vector arenas; constexpr uint64_t kArena = 64ull << 20; @@ -228,6 +237,7 @@ Staging staging(uint64_t n, uint64_t align = 256) { const uint64_t at = (a.used + align - 1) & ~(align - 1); if (at + n <= a.size) { a.used = at + n; + a.used_at = std::chrono::steady_clock::now(); return {a.buffer, at, a.mapped + at}; } } @@ -235,6 +245,7 @@ Staging staging(uint64_t n, uint64_t align = 256) { for (Arena& a : arenas) { if (a.value != rec_value && a.value <= completed_value && a.size >= n) { a.value = rec_value, a.used = n; + a.used_at = std::chrono::steady_clock::now(); return {a.buffer, 0, a.mapped}; } } @@ -246,6 +257,7 @@ Staging staging(uint64_t n, uint64_t align = 256) { return {}; a.mapped = (uint8_t*)m; a.value = rec_value, a.used = n; + a.used_at = std::chrono::steady_clock::now(); arenas.push_back(a); static int logged; if (logged++ < 4 || arenas.size() % 16 == 0) @@ -336,6 +348,47 @@ VkImageView bc_scratch_view = VK_NULL_HANDLE; VkBuffer bc_out = VK_NULL_HANDLE; VmaAllocation bc_out_alloc = nullptr; bool bc_scratch_fresh = false; // (its layout still to be set, in the next upload command buffer) +// (the scratch and its buffer, 160 MB, are made when an upload first needs them and given back after RX_VK_BC_IDLE_S seconds +// (20) without one: a title or a load decodes hundreds of textures through them, steady gameplay few (the "given back" line says +// whether it did). +// 0: kept for good once made. They have memory objects of their own, so that freeing gives the VRAM back: VMA keeps the empty +// block of its image pool for the next images. RX_VK_BC_POOL=1: from the pool as before) +std::mutex bc_mu; +std::chrono::steady_clock::time_point bc_used_at; +uint64_t bc_last_value = 0; // (the recording that last used them: they go once it is done) +std::atomic bc_have{false}; +uint32_t bc_idle_s() { + static const uint32_t v = getenv("RX_VK_BC_IDLE_S") ? (uint32_t)atoi(getenv("RX_VK_BC_IDLE_S")) : 20; + return v; +} +bool bc_scratch_make() { // (bc_mu held) + VkImageCreateInfo ii{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + ii.imageType = VK_IMAGE_TYPE_2D; + ii.format = VK_FORMAT_R8G8B8A8_UNORM; + ii.extent = {kBcDim, kBcDim, 1}; + ii.mipLevels = 1; + ii.arrayLayers = kBcLayers; + ii.samples = VK_SAMPLE_COUNT_1_BIT; + ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + static const bool pool = getenv("RX_VK_BC_POOL") != nullptr; + if (!create_image(ii, bc_scratch, bc_scratch_alloc, !pool)) return false; + VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vi.image = bc_scratch; + vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; + vi.format = VK_FORMAT_R8G8B8A8_UNORM; + vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; + const uint64_t out = (uint64_t)(kBcDim / 4) * (kBcDim / 4) * kBcLayers * 16; + if (vkCreateImageView(dev.device, &vi, nullptr, &bc_scratch_view) != VK_SUCCESS || + !create_device_buffer(out, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, bc_out, bc_out_alloc, !pool)) { + if (bc_scratch_view) vkDestroyImageView(dev.device, bc_scratch_view, nullptr); + destroy_image(bc_scratch, bc_scratch_alloc); + bc_scratch = VK_NULL_HANDLE, bc_scratch_view = VK_NULL_HANDLE; + return false; + } + bc_scratch_fresh = true; + bc_have = true; + return true; +} bool bc_ready() { static int state = 0; @@ -374,26 +427,6 @@ bool bc_ready() { const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &bc_pipe); vkDestroyShaderModule(dev.device, m, nullptr); if (r != VK_SUCCESS) return false; - VkImageCreateInfo ii{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; - ii.imageType = VK_IMAGE_TYPE_2D; - ii.format = VK_FORMAT_R8G8B8A8_UNORM; - ii.extent = {kBcDim, kBcDim, 1}; - ii.mipLevels = 1; - ii.arrayLayers = kBcLayers; - ii.samples = VK_SAMPLE_COUNT_1_BIT; - ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; - if (!create_image(ii, bc_scratch, bc_scratch_alloc)) return false; - VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; - vi.image = bc_scratch; - vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; - vi.format = VK_FORMAT_R8G8B8A8_UNORM; - vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers}; - if (vkCreateImageView(dev.device, &vi, nullptr, &bc_scratch_view) != VK_SUCCESS) return false; - const uint64_t out = (uint64_t)(kBcDim / 4) * (kBcDim / 4) * kBcLayers * 16; - if (!create_device_buffer(out, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, bc_out, - bc_out_alloc)) - return false; - bc_scratch_fresh = true; state = 1; logf("vk: big ASTC textures are kept as BC3 (re-encoded on the GPU; RX_VK_ASTC_RGBA8=1: as RGBA8)"); return true; @@ -422,8 +455,15 @@ void bc_encode_into(VkCommandBuffer cmd, uint32_t w, uint32_t h, uint32_t n, Tex full_barrier(cmd); // (the scratch and the buffer are the next part's) } -void bc_scratch_layout(VkCommandBuffer cmd) { - if (!bc_scratch_fresh) return; +/// The scratch for the upload recording now: made if it is not there, and its layout set (false: it could not be made) +bool bc_scratch_layout(VkCommandBuffer cmd) { + { + std::lock_guard lk(bc_mu); + if (!bc_have && !bc_scratch_make()) return false; + bc_used_at = std::chrono::steady_clock::now(); + bc_last_value = rec_value; + } + if (!bc_scratch_fresh) return true; bc_scratch_fresh = false; VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2}; b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; @@ -437,6 +477,19 @@ void bc_scratch_layout(VkCommandBuffer cmd) { di.imageMemoryBarrierCount = 1; di.pImageMemoryBarriers = &b; vkCmdPipelineBarrier2(cmd, &di); + return true; +} +/// (every present: the scratch unused for RX_VK_BC_IDLE_S seconds goes with the images the GPU is done with) +void bc_scratch_idle() { + if (!bc_have.load(std::memory_order_relaxed) || !bc_idle_s()) return; + std::lock_guard lk(bc_mu); + if (!bc_have || std::chrono::steady_clock::now() - bc_used_at < std::chrono::seconds(bc_idle_s())) return; + Dead d{bc_last_value, bc_scratch, bc_scratch_alloc, {bc_scratch_view}, {}}; + d.buffer = bc_out, d.buffer_alloc = bc_out_alloc; + dead.push_back(std::move(d)); + bc_scratch = VK_NULL_HANDLE, bc_scratch_alloc = nullptr, bc_scratch_view = VK_NULL_HANDLE, bc_out = VK_NULL_HANDLE, bc_out_alloc = nullptr; + bc_have = false, bc_scratch_fresh = false; + logf("vk: the BC3 scratch (160 MB) unused for %u s: given back", bc_idle_s()); } /// The storage view of one level (every layer) the ASTC decoder writes @@ -570,7 +623,7 @@ bool upload_astc_gpu(Texture& tex, uint64_t total, const Ranges* dirty) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_pipe); const uint64_t align = std::max(dev.props.limits.minStorageBufferOffsetAlignment, 4); if (tex.bc) { // (decoded into the scratch a run of consecutive layers at a time, re-encoded, copied in) - bc_scratch_layout(cmd); + if (!bc_scratch_layout(cmd)) return false; for (uint32_t level = 0; level < tex.levels; level++) { const LevelGeo g = level_geo(tex, level); const uint64_t off = g.offset - tail0; @@ -1261,7 +1314,7 @@ void destroy_later(Texture& tex) { if (kv.second) d.views.push_back(kv.second); d.memory = tex.sparse_chunks; dead.push_back(std::move(d)); - total_bytes -= std::min(total_bytes, tex.bytes); + total_bytes -= std::min(total_bytes, tex.bytes + tex.pad); } void vw_forget(const Texture& t); // (below: a destroyed texture leaves the list of GPU-written ones) @@ -1457,7 +1510,11 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l) * std::max(1u, x.depth >> l); const uint32_t texel = x.bc || is_bc(x.format) ? 1 : astc ? 4 : std::max(1, from ? (from == 3 ? 4 : from == 6 ? 8 : 16) : bpb); x.bytes = x.sparse ? 0 : px * x.layers * texel * x.samples; // (a sparse image's: as its layers are bound) - total_bytes += x.bytes; + // (the budget counts what the allocator gave it: the texels' 8% short of that over a scene's 2000 textures, 3228 MB counted for 3504 + // MB held, and a budget of 4035 MB took 4300. RX_VK_COUNT_BY_ESTIMATE=1: the texels) + static const bool by_estimate = getenv("RX_VK_COUNT_BY_ESTIMATE") != nullptr; + if (!x.sparse && !by_estimate) x.pad = std::max(x.bytes, allocation_size(x.allocation)) - x.bytes; + total_bytes += x.bytes + x.pad; // into GENERAL, and cleared (a target the game renders into before writing it all reads zeros, not garbage) VkCommandBuffer cmd = upload_cmd(); VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2}; @@ -1540,7 +1597,29 @@ Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw } // namespace -void full_barrier(VkCommandBuffer cmd) { +std::atomic work_epoch{0}; +namespace { +// (each thread's own: the display thread's present records barriers too) +thread_local VkCommandBuffer last_barrier_cmd = VK_NULL_HANDLE; // (the command buffer this thread's last full barrier went into, and the work count then) +thread_local uint64_t last_barrier_epoch = 0; +std::atomic st_barriers_made{0}, st_barriers_left_out{0}; +} // namespace +void barrier_stats(int64_t* made, int64_t* left_out) { + *made = st_barriers_made.load(std::memory_order_relaxed), *left_out = st_barriers_left_out.load(std::memory_order_relaxed); +} + +// (hookshot) A full barrier right after another one on the same command buffer, no command that does or orders GPU work in between +// (vk_work.h counts them), orders nothing the first did not: left out. The game's own barrier (end of a pass), then the next pass's +// start barrier, then a dispatch's, were two or three in a row. RX_VK_BARRIER_DEDUPE_OFF=1: all recorded, as before. +void full_barrier(VkCommandBuffer cmd, const char* why) { + static const bool off = getenv("RX_VK_BARRIER_DEDUPE_OFF") != nullptr; + const bool dedupe = !off && !par_active.load(std::memory_order_relaxed); // (a parallel replay's threads share command buffers and the count) + if (dedupe && cmd == last_barrier_cmd && work_epoch.load(std::memory_order_relaxed) == last_barrier_epoch) { + st_barriers_left_out.fetch_add(1, std::memory_order_relaxed); + return; + } + st_barriers_made.fetch_add(1, std::memory_order_relaxed); + GprofSpan gp(cmd, why); // (RX_VK_PASS_TIMING: what a barrier costs the GPU that has nothing left to wait for) VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT; b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT; @@ -1549,9 +1628,11 @@ void full_barrier(VkCommandBuffer cmd) { d.memoryBarrierCount = 1; d.pMemoryBarriers = &b; vkCmdPipelineBarrier2(cmd, &d); + last_barrier_cmd = cmd, last_barrier_epoch = work_epoch.load(std::memory_order_relaxed); } void texture_begin_recording(VkCommandBuffer upload, uint64_t value) { + last_barrier_cmd = VK_NULL_HANDLE; // (a new recording: its command buffers start afresh, whatever their handles held) up_cmd = upload; if (value != rec_value) epoch++; // (a submission's parts check a texture's memory once: nothing is collected between) rec_value = value; @@ -1966,7 +2047,7 @@ bool texture_copy_into_bc(VkCommandBuffer cmd, Texture* src, uint32_t src_level, uint32_t layers) { if (!src || !dst || !dst->bc || !src->astc || src->bc || !bc_ready() || w > kBcDim || h > kBcDim || (dx & 3) || (dy & 3)) return false; - bc_scratch_layout(cmd); + if (!bc_scratch_layout(cmd)) return false; for (uint32_t l0 = 0; l0 < layers; l0 += kBcLayers) { const uint32_t n = std::min(kBcLayers, layers - l0); full_barrier(cmd); @@ -2087,7 +2168,12 @@ bool flush_texture(Texture& t, uint64_t clip_lo, uint64_t clip_hi) { VkBuffer buf = VK_NULL_HANDLE; VmaAllocation alloc = nullptr; void* map = nullptr; - if (!create_buffer(total, VK_BUFFER_USAGE_TRANSFER_DST_BIT, true, buf, alloc, &map) || !map) return false; + // (cached host memory, which the CPU reads: the sequential-write kind is write-combined, or VRAM behind the BAR, and every read + // of it by the unswizzle below goes over PCIe uncached. RX_VK_FLUSH_UNCACHED=1: as before) + static const bool uncached = getenv("RX_VK_FLUSH_UNCACHED") != nullptr; + if (!(uncached ? create_buffer(total, VK_BUFFER_USAGE_TRANSFER_DST_BIT, true, buf, alloc, &map) + : create_buffer_cached(total, VK_BUFFER_USAGE_TRANSFER_DST_BIT, buf, alloc, &map)) || !map) + return false; bool ok = readback_begin(); if (ok) { full_barrier(rb_cmd); // (after what the submissions before wrote) @@ -2103,6 +2189,7 @@ bool flush_texture(Texture& t, uint64_t clip_lo, uint64_t clip_hi) { } vkCmdCopyImageToBuffer(rb_cmd, t.image, VK_IMAGE_LAYOUT_GENERAL, buf, (uint32_t)c.size(), c.data()); ok = readback_submit_wait(); + if (ok && !uncached) invalidate_mapped(alloc, total); } if (ok && total <= 64) { // (RX_VK_LOG_TINY_FLUSH=1: the values of tiny ones, 400 lines: a 1 x 1 luminance) static const bool log_tiny = getenv("RX_VK_LOG_TINY_FLUSH") != nullptr; @@ -2569,11 +2656,22 @@ void texture_frame(uint64_t completed) { frame_no++; { // (staging arenas the GPU is done with, beyond 256 MB of them, and any over 64 MB: freed. A scene change stages // hundreds of MB to over a GB (the course maps, the terrain's ASTC arrays), and that host memory was kept) - static const uint64_t keep = (uint64_t)(getenv("RX_VK_STAGING_KEEP_MB") ? atoi(getenv("RX_VK_STAGING_KEEP_MB")) : 256) << 20; + // (hookshot: 64 on a card of 6 GB or less. On one with resizable BAR the arenas are VRAM: VMA puts a host-visible buffer the GPU + // reads (STORAGE: the ASTC decoder) in DEVICE_LOCAL | HOST_VISIBLE. The RTX 4050 Laptop held 4 idle ones, 256 MB of 5.7 GB, through + // all of Kokiri gameplay, which stages ~nothing. A scene change makes the ones it needs again. RX_VK_STAGING_KEEP_MB=256: as before) + static const uint64_t keep = []() -> uint64_t { + if (const char* e = getenv("RX_VK_STAGING_KEEP_MB")) return (uint64_t)atoi(e) << 20; + uint64_t used = 0, allowed = 0; + return (vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 64ull : 256ull) << 20; + }(); uint64_t kept = 0; for (auto it = arenas.begin(); it != arenas.end();) { const bool idle = it->value <= completed && it->value != rec_value; - if (idle && (it->size > kArena || kept + it->size > keep)) { + // (and one nothing has staged into for 30 s: Kokiri Forest's gameplay stages next to nothing, and the 64 MB it kept are VRAM + // behind the BAR. A load makes one again. RX_VK_STAGING_IDLE_S=: that long, 0: kept) + static const int64_t idle_s = getenv("RX_VK_STAGING_IDLE_S") ? atoll(getenv("RX_VK_STAGING_IDLE_S")) : 30; + const bool stale = idle_s && std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_s); + if (idle && (it->size > kArena || kept + it->size > keep || stale)) { destroy_buffer(it->buffer, it->allocation); it = arenas.erase(it); continue; @@ -2588,8 +2686,40 @@ void texture_frame(uint64_t completed) { logf("vk: memory: working set %llu MB (peak %llu), private %llu MB, staging %llu MB in %zu arenas", (unsigned long long)(pm.WorkingSetSize >> 20), (unsigned long long)(pm.PeakWorkingSetSize >> 20), (unsigned long long)(pm.PrivateUsage >> 20), (unsigned long long)(kept >> 20), arenas.size()); +#if defined(__linux__) && defined(__GLIBC__) + // (glibc's heap: what its arenas hold free. The shader translation and the loads' decoding leave their garbage spread through + // the threads' arenas (26 of them, 1.2 GB resident in Kokiri Forest, the heap's growth in a scene change never going back), + // and a free chunk's pages are the process's till malloc_trim gives them to the system. Asked on a worker, when 192 MB or + // more are free: RX_VK_MALLOC_TRIM_MB=: that many, 0: never; the "vk: malloc:" line says what the arenas hold) + // (every 1800 presents: mallinfo2 walks every free chunk of every arena with its lock held) + static std::atomic busy{false}; + if (frame_no % 1800 == 0 && !busy.exchange(true)) + jobs_push([] { + static const uint64_t trim_mb = getenv("RX_VK_MALLOC_TRIM_MB") ? (uint64_t)atoll(getenv("RX_VK_MALLOC_TRIM_MB")) : 192; + const struct mallinfo2 a = mallinfo2(); + PROCESS_MEMORY_COUNTERS_EX pm0{}; + pm0.cb = sizeof pm0; + K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm0, sizeof pm0); + logf("vk: malloc: %llu MB in arenas (%llu MB in use, %llu MB free, %llu MB of it at the tops), %llu MB mapped", + (unsigned long long)(a.arena >> 20), (unsigned long long)(a.uordblks >> 20), (unsigned long long)(a.fordblks >> 20), + (unsigned long long)(a.keepcost >> 20), (unsigned long long)(a.hblkhd >> 20)); + if (trim_mb && (a.fordblks >> 20) >= trim_mb) { + const auto t0 = std::chrono::steady_clock::now(); + malloc_trim(0); + PROCESS_MEMORY_COUNTERS_EX pm1{}; + pm1.cb = sizeof pm1; + K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm1, sizeof pm1); + logf("vk: malloc_trim: %llu MB free in the arenas now, the process's private memory %llu -> %llu MB (%.0f ms)", + (unsigned long long)(mallinfo2().fordblks >> 20), (unsigned long long)(pm0.PrivateUsage >> 20), + (unsigned long long)(pm1.PrivateUsage >> 20), + std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); + } + busy = false; + }); +#endif } } + bc_scratch_idle(); for (auto it = dead.begin(); it != dead.end();) { if (it->value > completed) { ++it; @@ -2597,6 +2727,7 @@ void texture_frame(uint64_t completed) { } for (VkImageView v : it->views) vkDestroyImageView(dev.device, v, nullptr); destroy_image(it->image, it->allocation); + if (it->buffer) destroy_buffer(it->buffer, it->buffer_alloc); for (VkDeviceMemory m : it->memory) vkFreeMemory(dev.device, m, nullptr); it = dead.erase(it); } diff --git a/gpu/vk/vk_texture.h b/gpu/vk/vk_texture.h index 2d399e163..67f994a5e 100644 --- a/gpu/vk/vk_texture.h +++ b/gpu/vk/vk_texture.h @@ -72,7 +72,8 @@ struct Texture { uint64_t last_frame = 0; // the present it was last used in (the cache's age) uint64_t first_render_frame = 0; // the present it was first rendered into in (0: never) uint64_t last_value = 0; // the timeline value of the last submission that used it - uint64_t bytes = 0; // device memory it holds + uint64_t bytes = 0; // device memory it holds (the texels', or a sparse image's chunks) + uint64_t pad = 0; // (what the allocator gave it beyond that: padding and alignment; counted in the budget with bytes) uint32_t id = 0; // (its creation number: diagnostics tell images at one address apart) uint64_t first_seen = 0; // (the present it was made at) uint64_t owned_frame = 0; // (the present its storage was last noted as render targets' (gpu_owned)) @@ -164,7 +165,10 @@ uint64_t texture_made(); void texture_unswizzle_stats(int64_t* bytes, int64_t* ns); uint64_t texture_bytes(); /// A full memory barrier (all commands, all accesses): what separates dependent work for now -void full_barrier(VkCommandBuffer cmd); +/// (why: its label in RX_VK_PASS_TIMING's barrier lines) +void full_barrier(VkCommandBuffer cmd, const char* why = "barrier (other)"); +/// Full barriers recorded and left out (RX_VK_BARRIER_DEDUPE_OFF=1: none), since the start +void barrier_stats(int64_t* made, int64_t* left_out); void texture_stats(int64_t* uploads, int64_t* astc_gpu, int64_t* bytes_uploaded); /// RX_VK_DUMP_TEX: the textures uploaded next written to that folder as PPM (diagnostics) void texture_read_texels(const nvn_tex* t, unsigned level, unsigned layer0, const NVNcopyRegion* r, uint8_t* dst, diff --git a/gpu/vk/vk_vma.cpp b/gpu/vk/vk_vma.cpp index 5c5db7225..7e770b0f0 100644 --- a/gpu/vk/vk_vma.cpp +++ b/gpu/vk/vk_vma.cpp @@ -35,7 +35,9 @@ bool vma_init(PFN_vkGetInstanceProcAddr gipa) { // memory type the pool's is not, are allocated as before. VmaPool image_pool = VK_NULL_HANDLE; bool image_pool_tried = false; -constexpr uint64_t kImageBlock = 128ull << 20; +// (RX_VK_IMAGE_BLOCK_MB=, 16 and up: the block's size. VMA keeps one block empty for the next image, 128 MB of VRAM that a 64 MB block +// would halve, at twice the memory objects: the driver allows ~4096, the imported pools take ~2400) +const uint64_t kImageBlock = (getenv("RX_VK_IMAGE_BLOCK_MB") ? (uint64_t)std::max(16, atoi(getenv("RX_VK_IMAGE_BLOCK_MB"))) : 128u) << 20; VmaPool images_pool() { if (image_pool_tried) return image_pool; @@ -61,13 +63,18 @@ VmaPool images_pool() { return image_pool; } -bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation) { +bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated) { VmaAllocationCreateInfo ai{}; ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; // (in VRAM or not at all: an image VMA puts in system memory when VRAM is full is read over PCIe every frame, // and takes the RAM the game and everything else on the machine need) ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + if (dedicated) { // (a memory object of its own, which freeing gives back whole: a pool's block is kept by VMA when it empties) + VmaAllocationCreateInfo dai = ai; + dai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; + if (vmaCreateImage(vma, &ci, &dai, &image, &allocation, nullptr) == VK_SUCCESS) return true; + } static const bool no_pool = getenv("RX_VK_NO_IMAGE_POOL") != nullptr; if (VmaPool pool = no_pool ? VK_NULL_HANDLE : images_pool()) { VmaAllocationCreateInfo pai = ai; @@ -89,8 +96,13 @@ bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& al void destroy_image(VkImage image, VmaAllocation allocation) { vmaDestroyImage(vma, image, allocation); } +uint64_t allocation_size(VmaAllocation allocation) { + VmaAllocationInfo info{}; + if (allocation) vmaGetAllocationInfo(vma, allocation, &info); + return info.size; +} bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, VkBuffer& buffer, - VmaAllocation& allocation, void** mapped) { + VmaAllocation& allocation, void** mapped, bool dedicated) { VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.size = size; bi.usage = usage; @@ -99,6 +111,8 @@ bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, V if (host_visible) ai.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT; // (a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT gets VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT from VMA: // vma_init sets VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT) + static const bool blocks = getenv("RX_VK_BLOCK_BUFFERS") != nullptr; + if (dedicated && !blocks) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; VmaAllocationInfo info{}; if (vmaCreateBuffer(vma, &bi, &ai, &buffer, &allocation, &info) != VK_SUCCESS) return false; if (mapped) *mapped = info.pMappedData; @@ -107,13 +121,14 @@ bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, V /// A buffer in cached host memory, mapped: written on one thread, read back on the CPU too (the captures: the game's /// threads copy pages into it, the replay reads texture pages out of it and the GPU copies the rest from it) bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, - void** mapped) { + void** mapped, bool dedicated) { VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.size = size; bi.usage = usage; VmaAllocationCreateInfo ai{}; ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; ai.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT; + if (dedicated) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; // (its own memory object: no part of a 64 MB block left unused) VmaAllocationInfo info{}; if (vmaCreateBuffer(vma, &bi, &ai, &buffer, &allocation, &info) != VK_SUCCESS) return false; if (mapped) *mapped = info.pMappedData; @@ -122,10 +137,13 @@ bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf void flush_mapped(VmaAllocation allocation, uint64_t size) { if (size) vmaFlushAllocation(vma, allocation, 0, size); } +void invalidate_mapped(VmaAllocation allocation, uint64_t size) { + if (size) vmaInvalidateAllocation(vma, allocation, 0, size); +} void destroy_buffer(VkBuffer buffer, VmaAllocation allocation) { vmaDestroyBuffer(vma, buffer, allocation); } -bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation) { +bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, bool dedicated) { VkBufferCreateInfo bi{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.size = size; bi.usage = usage; @@ -133,6 +151,7 @@ bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buf ai.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE; ai.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; if (dev.memory_budget) ai.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + if (dedicated) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; return vmaCreateBuffer(vma, &bi, &ai, &buffer, &allocation, nullptr) == VK_SUCCESS; } /// (every heap: what VMA has in blocks and allocations there, what the process uses of it, its budget; one line) diff --git a/gpu/vk/vk_work.h b/gpu/vk/vk_work.h new file mode 100644 index 000000000..215efd7c6 --- /dev/null +++ b/gpu/vk/vk_work.h @@ -0,0 +1,46 @@ +// (hookshot) Every Vulkan command that does GPU work, or orders it, counts in work_epoch. full_barrier (vk_texture.cpp) leaves out a +// barrier recorded right after another one on the same command buffer with nothing in between: the second orders nothing the first +// did not. Include this header AFTER the file's other includes: it turns the calls below into a count and the call (a header +// included after it that names one of them with a '(' after it would be changed too). +#pragma once + +#include + +#include "vk_loader.h" + +namespace mkwvk { +extern std::atomic work_epoch; // (vk_texture.cpp) +inline void note_work() { + // (an add, not a load and a store: a thread preempted between the two would store a stale count over another thread's, which could + // make the count equal the one a barrier was recorded at with work recorded since, and the next barrier is left out for it) + work_epoch.fetch_add(1, std::memory_order_relaxed); +} +} // namespace mkwvk + +#define vkCmdPipelineBarrier2(...) (::mkwvk::note_work(), ::vkx::vkCmdPipelineBarrier2(__VA_ARGS__)) +#define vkCmdBeginRendering(...) (::mkwvk::note_work(), ::vkx::vkCmdBeginRendering(__VA_ARGS__)) +#define vkCmdEndRendering(...) (::mkwvk::note_work(), ::vkx::vkCmdEndRendering(__VA_ARGS__)) +#define vkCmdDraw(...) (::mkwvk::note_work(), ::vkx::vkCmdDraw(__VA_ARGS__)) +#define vkCmdDrawIndexed(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawIndexed(__VA_ARGS__)) +#define vkCmdDrawIndirect(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawIndirect(__VA_ARGS__)) +#define vkCmdDrawIndexedIndirect(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawIndexedIndirect(__VA_ARGS__)) +#define vkCmdDrawIndirectCount(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawIndirectCount(__VA_ARGS__)) +#define vkCmdDrawIndexedIndirectCount(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawIndexedIndirectCount(__VA_ARGS__)) +#define vkCmdDrawMeshTasksEXT(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawMeshTasksEXT(__VA_ARGS__)) +#define vkCmdDrawMeshTasksIndirectEXT(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawMeshTasksIndirectEXT(__VA_ARGS__)) +#define vkCmdDrawMeshTasksIndirectCountEXT(...) (::mkwvk::note_work(), ::vkx::vkCmdDrawMeshTasksIndirectCountEXT(__VA_ARGS__)) +#define vkCmdDispatch(...) (::mkwvk::note_work(), ::vkx::vkCmdDispatch(__VA_ARGS__)) +#define vkCmdDispatchIndirect(...) (::mkwvk::note_work(), ::vkx::vkCmdDispatchIndirect(__VA_ARGS__)) +#define vkCmdCopyBuffer(...) (::mkwvk::note_work(), ::vkx::vkCmdCopyBuffer(__VA_ARGS__)) +#define vkCmdCopyBufferToImage(...) (::mkwvk::note_work(), ::vkx::vkCmdCopyBufferToImage(__VA_ARGS__)) +#define vkCmdCopyImageToBuffer(...) (::mkwvk::note_work(), ::vkx::vkCmdCopyImageToBuffer(__VA_ARGS__)) +#define vkCmdCopyImage(...) (::mkwvk::note_work(), ::vkx::vkCmdCopyImage(__VA_ARGS__)) +#define vkCmdBlitImage(...) (::mkwvk::note_work(), ::vkx::vkCmdBlitImage(__VA_ARGS__)) +#define vkCmdResolveImage(...) (::mkwvk::note_work(), ::vkx::vkCmdResolveImage(__VA_ARGS__)) +#define vkCmdClearColorImage(...) (::mkwvk::note_work(), ::vkx::vkCmdClearColorImage(__VA_ARGS__)) +#define vkCmdClearDepthStencilImage(...) (::mkwvk::note_work(), ::vkx::vkCmdClearDepthStencilImage(__VA_ARGS__)) +#define vkCmdClearAttachments(...) (::mkwvk::note_work(), ::vkx::vkCmdClearAttachments(__VA_ARGS__)) +#define vkCmdFillBuffer(...) (::mkwvk::note_work(), ::vkx::vkCmdFillBuffer(__VA_ARGS__)) +#define vkCmdUpdateBuffer(...) (::mkwvk::note_work(), ::vkx::vkCmdUpdateBuffer(__VA_ARGS__)) +#define vkCmdBeginQuery(...) (::mkwvk::note_work(), ::vkx::vkCmdBeginQuery(__VA_ARGS__)) +#define vkCmdEndQuery(...) (::mkwvk::note_work(), ::vkx::vkCmdEndQuery(__VA_ARGS__)) diff --git a/gpu/vk/vk_xfer.cpp b/gpu/vk/vk_xfer.cpp index 6d7ca9d5b..372c9ade9 100644 --- a/gpu/vk/vk_xfer.cpp +++ b/gpu/vk/vk_xfer.cpp @@ -9,6 +9,8 @@ #include "vk_xfer.h" #include "vk_draw.h" // (end_pass: a pass cannot stay open across a transfer) #include "vk_texture.h" +#include "vk_gprof.h" +#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier) namespace mkwvk { namespace { @@ -201,6 +203,8 @@ void copy_b2t(VkCommandBuffer cmd, NvnState& st, const nvn_p_copy_bt& p, bool to return; } end_pass(cmd); + GprofSpan gp(cmd, to_image ? "xfer copy buffer->texture (with its barriers)" : "xfer copy texture->buffer (with its barriers)"); + gp.bytes = size; full_barrier(cmd); if (to_image) { vkCmdCopyBufferToImage(cmd, span.buffer, s.tex->image, VK_IMAGE_LAYOUT_GENERAL, 1, &c); @@ -240,6 +244,7 @@ void copy_t2t(VkCommandBuffer cmd, NvnState& st, const nvn_p_copy_tt& p) { } s.layers = d.layers = std::min(s.layers, d.layers); end_pass(cmd); + GprofSpan gp(cmd, "xfer copy texture->texture (with its barriers)"); full_barrier(cmd); const bool same_size = s.w == d.w && s.h == d.h && s.d == d.d; if (s.tex->bc != d.tex->bc || (s.tex->bc && !same_size)) { // (big ASTC kept as BC3 on one side: vk_texture.cpp) @@ -297,6 +302,7 @@ void clear_tex(VkCommandBuffer cmd, NvnState& st, const nvn_p_clear_tex& p) { const bool whole = !s.x && !s.y && s.w == lw && s.h == lh; if ((p.mask & 0xf) != 0xf && (s.tex->aspect & VK_IMAGE_ASPECT_COLOR_BIT)) warn(8, "texture clear with a channel mask: cleared whole"); end_pass(cmd); + GprofSpan gp(cmd, "xfer clear texture (with its barriers)"); full_barrier(cmd); if (whole && s.tex->type != VK_IMAGE_TYPE_3D) { VkImageSubresourceRange rng{s.tex->aspect, s.level, 1, s.layer, s.layers}; diff --git a/host/hos_main.c b/host/hos_main.c index c8dc33dc7..93c3563c7 100644 --- a/host/hos_main.c +++ b/host/hos_main.c @@ -23,6 +23,9 @@ #else #include /* (linux: host/posix's) */ #include +#ifdef __linux__ +#include +#endif #endif /* (linux) the separator of the paths the executable's folder is joined with */ #ifdef _WIN32 @@ -137,6 +140,11 @@ static int reserve_guest(void) { collapse khugepaged makes in them changes their mappings, which the kernel answers by pinning the pages again at the next submission: ~250 ms a submit on RADV while the GPU sat idle. HOS_THP=1: as before */ if (!getenv("HOS_THP")) madvise((void *)A64_HOST_BASE, GUEST_SPACE, MADV_NOHUGEPAGE); +#ifdef __linux__ + /* (HOS_NO_THP=1, experiment: none outside the window either. With THP "always" 400-450 MB of the host heap's and the binary's bss are huge + pages, each resident whole when any of its 2 MB is touched) */ + if (getenv("HOS_NO_THP")) prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0); +#endif return 1; #endif } diff --git a/tests/test_bank_gen.py b/tests/test_bank_gen.py new file mode 100644 index 000000000..06d7597c4 --- /dev/null +++ b/tests/test_bank_gen.py @@ -0,0 +1,60 @@ +"""A graphics stage's driver banks are kept by the generation of the bindings they are made from. + +c2 of a graphics stage holds its texture, sampler and image handles and c0 its storage buffer table, so a uniform buffer's bind (nearly every +draw) does not make the bank stale: tex_gen[stage] and ssbo_gen[stage] are what bind_gen was when those last changed. Compute, task and +mesh stages (c0 holds a grid, push constants, the pointer table) stay on bind_gen. +python -I -B tests/test_bank_gen.py +""" +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + vk = ROOT / "gpu" / "vk" + bank = (vk / "vk_bank.cpp").read_text(encoding="utf-8") + replay = (vk / "vk_replay.cpp").read_text(encoding="utf-8") + draw = (vk / "vk_draw.cpp").read_text(encoding="utf-8") + fill = block(bank, "void driver_bank_fill(") + c2 = block(fill, "if (bank == 2) {") + if set(re.findall(r"\bs\.(\w+)", c2)) - {"tex", "img", "samp"}: + raise SystemExit("c2 is made from more than tex_gen covers: " + str(set(re.findall(r"\bs\.(\w+)", c2)))) + tail = fill[fill.index("static constexpr uint32_t kSsboTable"):] + if set(re.findall(r"\bs\.(\w+)", tail)) - {"ssbo"} or "pointer_table(s, stage)" not in block(tail, "if ((stage == 6 || stage == 7)"): + raise SystemExit("a graphics stage's c0 is made from more than ssbo_gen covers") + if "stage == 5" not in fill[: fill.index("static constexpr uint32_t kSsboTable")]: + raise SystemExit("compute's c0 is no longer apart from the graphics stages'") + # (every place the handles or storage buffers change moves their generation with bind_gen) + for marker, field in (("case NOP_BIND_TEX:", "tex_gen"), ("case NOP_SET_TEX_POOL:", "tex_gen")): + at = replay.index(marker) + if field not in replay[at: at + 1400]: + raise SystemExit(f"{marker} does not set {field}") + if replay.count("ssbo_gen[b.stage] = s.bind_gen") != 2: + raise SystemExit("a storage buffer bind does not set ssbo_gen (NOP_BIND_BUFS and NOP_BIND_SSBO)") + if "s->tex_gen[k] = gen_base, s->ssbo_gen[k] = gen_base" not in block(replay, "void use_queue_state("): + raise SystemExit("another state's generations are not made afresh with bind_gen") + for name in ("s.tex[", "s.samp[", "s.img[", "s.ssbo["): + stores = [m for m in re.finditer(re.escape(name) + r"[^;\n]*\]\s*=[^=]", replay)] + if stores: + raise SystemExit(f"{name} is stored to outside the bind handlers' arrays") + uniform = block(draw, "VkDescriptorBufferInfo uniform_range(") + if "bank_stage > 4 ? s.bind_gen" not in uniform or "RX_VK_BANK_GEN_OFF" not in uniform: + raise SystemExit("compute, task and mesh banks are not kept by bind_gen, or the switch is missing") + if "c.bind_gen = gen;" not in uniform or "c.bind_gen == gen" not in uniform: + raise SystemExit("the bank cache is not kept and checked by the same generation") + print("PASS: a graphics stage's banks are kept by the generation of the bindings they are made from") + + +if __name__ == "__main__": + main() diff --git a/tests/test_mem_trim.py b/tests/test_mem_trim.py new file mode 100644 index 000000000..67fa98f34 --- /dev/null +++ b/tests/test_mem_trim.py @@ -0,0 +1,60 @@ +"""The memory trims keep what they must. + +A recording's stream may move to a bigger buffer in the middle of a recording: what was kept as an offset into "the stream" (the +bank cache, the c1 uploads, the uniform overlay's mapping) has to name its buffer or reserve its room first. The BC3 scratch is made +when an upload needs it: every user asks for it first and stops when it cannot be made. A capture's buffer is sized by class, with +a floor, and the spare ones are counted out as well as in. +python -I -B tests/test_mem_trim.py +""" +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def need(cond, why): + if not cond: + raise SystemExit(why) + + +def main(): + draw = (ROOT / "gpu/vk/vk_draw.cpp").read_text(encoding="utf-8") + uniform = block(draw, "VkDescriptorBufferInfo uniform_range(") + need("cur->c1_at[key] = {cur->stream, at};" in uniform, "a c1 upload forgets which stream it is in") + need("return {it->second.first, it->second.second, n};" in uniform, "a c1 upload is served from the current stream") + need("c.buf = cur->stream" in uniform and uniform.count("return {c.buf, c.at, n};") == 2, + "a bank-cache hit names the stream it was kept in") + overlay = uniform.index("ub_overlay(u.addr") + need("stream_reserve(n, align)" in uniform[:overlay], "the overlay takes its mapping before its room is reserved") + nxt = block(draw, "bool stream_next(") + need("r.streams[r.si].buffer" in nxt and "r.before += r.used" in nxt, "a stream move does not switch the buffer") + begin = block(draw, "void draw_begin_recording(") + need("r.si = 0" in begin and "r.used = 0, r.before = 0" in begin, "a recording begins in a stream that is not its first") + need(begin.index("r.streams.resize(1)") < begin.index("r.si = 0"), "the streams given back are used after") + + tex = (ROOT / "gpu/vk/vk_texture.cpp").read_text(encoding="utf-8") + need(tex.count("if (!bc_scratch_layout(cmd)) return false;") == 2, "a BC3 upload uses a scratch it did not ask for") + need("bc_have = false" in block(tex, "void bc_scratch_idle("), "the scratch given back is still taken for made") + need("dead.push_back" in block(tex, "void bc_scratch_idle("), "the scratch is freed with no wait for the GPU") + need("total_bytes -= std::min(total_bytes, tex.bytes + tex.pad)" in tex, "a texture leaves the budget by its texels only") + need("x.pad = " in tex and "total_bytes += x.bytes + x.pad" in tex, "a texture enters the budget by its texels only") + + mem = (ROOT / "gpu/vk/vk_memory.cpp").read_text(encoding="utf-8") + need("cap_free_bytes -= size;" in block(mem, "bool cap_buf_get("), "a spare buffer taken is still counted") + need("cap_free_bytes += size;" in block(mem, "void cap_buf_free("), "a spare buffer kept is not counted") + size = block(mem, "uint64_t cap_buf_size(") + need("s < need" in size and "(need + (1u << 20) - 1)" in size, "a capture buffer is smaller than its capture") + print("ok") + + +if __name__ == "__main__": + main() diff --git a/tests/test_pick_tex_gen.py b/tests/test_pick_tex_gen.py new file mode 100644 index 000000000..6f3e755a2 --- /dev/null +++ b/tests/test_pick_tex_gen.py @@ -0,0 +1,47 @@ +"""The last picks' program stands across a uniform buffer's bind when its textures are the same. + +A graphics program's answers (the types its translation was told of the bound textures) are read from c2 of its stages: nothing else +in the bindings can change them, so the pair of last picks (prog_fast) need not wait for bind_gen, which every uniform buffer moves. A +handle program (it reads uniform memory), another bank, a secondary handle in another bank, a compute or a task / mesh program keeps +the old condition. +python -I -B tests/test_pick_tex_gen.py +""" +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + shader = (ROOT / "gpu" / "vk" / "vk_shader.cpp").read_text(encoding="utf-8") + mask = block(shader, "uint32_t answer_stage_mask(") + for needle in ("handle_program == ~0U", "cbuf_index == 2", "secondary_cbuf_index == 2", "nvn_stage > 4) return ~0u"): + if needle not in mask: + raise SystemExit(f"an answer may come from somewhere tex_gen does not cover: {needle} missing") + if mask.count("return ~0u") != 2 or "image_descriptors" not in mask or "texture_descriptors" not in mask: + raise SystemExit("answer_stage_mask does not refuse every other source") + pick = block(shader, "const Program* program_for(") + if "RX_VK_PICK_BIND_GEN" not in pick or "!compute && f.amask != ~0u" not in pick: + raise SystemExit("the switch is missing, or a compute program is picked by textures") + if "f.tex_gen[k] != s.tex_gen[k]" not in pick or "f.amask >> k & 1" not in pick: + raise SystemExit("the stages' textures are not compared") + keep = pick[pick.index("auto keep = "):] + if "f.amask = compute ? ~0u : answer_stage_mask(*p)" not in keep or "f.tex_gen, s.tex_gen" not in keep: + raise SystemExit("a kept pick does not remember its stages' textures") + for field in ("serial", "stages", "reg", "ub", "nvattr", "vattr", "ct"): + if f"f.{field}" not in pick[: pick.index("auto keep = ")]: + raise SystemExit(f"the last pick is taken without comparing its {field}") + print("PASS: the last picks stand while their textures do") + + +if __name__ == "__main__": + main() diff --git a/tests/test_push_state.py b/tests/test_push_state.py new file mode 100644 index 000000000..686845f24 --- /dev/null +++ b/tests/test_push_state.py @@ -0,0 +1,96 @@ +"""The pushed descriptors are remembered as one vector of words, compared and kept in place. + +A program change pushes every write, and the next draws of it push the writes that changed. The old bookkeeping was a vector of +vectors resized to each program's writes, so a program with more writes than the one before it grew new vectors word by word. +python -I -B tests/test_push_state.py [--cxx g++] +""" +import argparse +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def block(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--cxx", default="g++") + a = p.parse_args() + source = (ROOT / "gpu" / "vk" / "vk_draw.cpp").read_text(encoding="utf-8") + old = block(source, "void write_words(") + app = block(source, "void words_append(") + same = block(source, "int words_same_store(") + bind = block(source, "bool bind_descriptors(") + if "RX_VK_PUSH_STATE_OLD" not in bind: + raise SystemExit("the old bookkeeping has no switch") + full = bind[: bind.index("static thread_local std::vector changed")] + if "ps.words.clear(), ps.at.clear()" not in full or "ps.n = dw.writes.size()" not in full: + raise SystemExit("a full push does not remember its words") + fixture = f'''#include +#include +#include +#include +#define CHECK(x) do {{ if (!(x)) {{ std::fprintf(stderr, "FAIL:%d %s\\n", __LINE__, #x); std::exit(1); }} }} while (0) +typedef uint64_t VkBuffer; typedef uint64_t VkSampler; typedef uint64_t VkImageView; typedef uint64_t VkBufferView; +typedef uint64_t VkDeviceSize; +struct VkDescriptorBufferInfo {{ VkBuffer buffer; VkDeviceSize offset, range; }}; +struct VkDescriptorImageInfo {{ VkSampler sampler; VkImageView imageView; int imageLayout; }}; +struct VkWriteDescriptorSet {{ + uint32_t dstBinding, descriptorCount, descriptorType; + const VkDescriptorImageInfo* pImageInfo; const VkDescriptorBufferInfo* pBufferInfo; const VkBufferView* pTexelBufferView; +}}; +{old} +{app} +{same} +int main() {{ + VkDescriptorBufferInfo b[2] = {{{{1, 2, 3}}, {{4, 5, 6}}}}; + VkDescriptorImageInfo i[1] = {{{{7, 8, 1}}}}; + VkBufferView t[1] = {{9}}; + VkWriteDescriptorSet w[3] = {{}}; + w[0].dstBinding = 0, w[0].descriptorCount = 2, w[0].descriptorType = 6, w[0].pBufferInfo = b; + w[1].dstBinding = 1, w[1].descriptorCount = 1, w[1].descriptorType = 1, w[1].pImageInfo = i; + w[2].dstBinding = 2, w[2].descriptorCount = 1, w[2].descriptorType = 4, w[2].pTexelBufferView = t; + std::vector words, old_words, one; + std::vector at; + for (auto& x : w) at.push_back((uint32_t)words.size()), words_append(x, words); + for (auto& x : w) write_words(x, one), old_words.insert(old_words.end(), one.begin(), one.end()); + CHECK(words == old_words); // (the same words as the vector of vectors held) + CHECK(at.size() == 3 && at[1] == 7 && at[2] == 10 && words.size() == 12); + const uint64_t* e = words.data(); + auto same = [&](int k) {{ return words_same_store(w[k], words.data() + at[k], e + (k < 2 ? at[k + 1] : words.size())); }}; + for (int k = 0; k < 3; k++) CHECK(same(k) == 1); + b[1].range = 60; // (a buffer range changed: that write only) + CHECK(same(0) == 0); + CHECK(same(0) == 1); // (and kept: the same again) + CHECK(same(1) == 1 && same(2) == 1); + i[0].imageView = 80; + CHECK(same(1) == 0 && same(1) == 1); + t[0] = 90; + CHECK(same(2) == 0 && same(2) == 1); + CHECK(words[at[0] + 6] == 60 && words[at[1] + 2] == 80 && words[at[2] + 1] == 90 && words.size() == 12); + w[0].descriptorCount = 1; // (a write that is not the one the words were made for: refused, nothing touched) + const std::vector before = words; + CHECK(same(0) == -1 && words == before); + std::printf("PASS: pushed descriptors are compared and kept in one vector\\n"); +}} +''' + with tempfile.TemporaryDirectory(prefix="push-state-") as tmp: + cpp = Path(tmp) / "fixture.cpp" + exe = Path(tmp) / "fixture" + cpp.write_text(fixture, encoding="utf-8") + subprocess.run([a.cxx, "-std=c++20", "-Wall", "-Wextra", "-Werror", str(cpp), "-o", str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == "__main__": + main() diff --git a/tests/test_same_dyn.py b/tests/test_same_dyn.py index 15e2d7c67..06cc70d07 100644 --- a/tests/test_same_dyn.py +++ b/tests/test_same_dyn.py @@ -3,6 +3,7 @@ python -I -B tests/test_same_dyn.py [--cxx g++] """ import argparse +import re import subprocess import tempfile from pathlib import Path @@ -30,6 +31,13 @@ def main(): for needle in ("s.viewport", "s.scissor", "s.chmask", "s.blend", "s.ds", "cur_program", "s.poly"): if needle not in pack: raise SystemExit(f"dynamic state is remembered without {needle}") + # (the program in the key is what its write masks are made from: a draw of another program with the same masks is the same state) + if "program_wm_sig(*cur_program)" not in pack or "RX_VK_DYN_PROG_KEY" not in pack: + raise SystemExit("dynamic state is remembered by program, not by what the program makes of the write masks") + reads = lambda text: set(re.findall(r"info\.(\w+)", text)) + mask, sig = block(source, "uint32_t target_write_mask("), block(source, "uint64_t program_wm_sig(") + if reads(mask) != reads(sig): + raise SystemExit(f"target_write_mask reads {sorted(reads(mask))} but the program's write mask word is made from {sorted(reads(sig))}") body = block(source, "void dynamic_state(") if body.index("dyn_same(") > body.index("std::memset(&d, 0"): raise SystemExit("dynamic state is built before checking the draw before it")