diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 67e3076..2710e3c 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -5,6 +5,9 @@ on: - main pull_request: +permissions: + contents: read + jobs: format: name: Check code formatting @@ -14,12 +17,12 @@ jobs: run: | sudo apt-get update sudo apt-get install -y clang-format clang-format-18 - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Check format run: | - git-clang-format --binary clang-format-18 --diff origin/main --extensions cpp,hpp,c,cl > output.txt + git-clang-format --binary clang-format-18 --diff origin/main --extensions cpp,hpp,c,cl > output.txt || true cat output.txt grep -E "(no modified files to format|clang-format did not modify any files)" output.txt build-android: @@ -27,7 +30,7 @@ jobs: needs: format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Install dependencies @@ -38,12 +41,23 @@ jobs: run: | git clone --depth 1 --branch v51.2 https://github.com/google/perfetto third_party/perfetto git clone --depth 1 --branch v2026.05.29 https://github.com/KhronosGroup/OpenCL-Headers.git third_party/opencl-headers + git clone --depth 1 --branch v2026.05.29 https://github.com/KhronosGroup/OpenCL-ICD-Loader.git third_party/opencl-icd-loader - name: Download and extract Android NDK run: | wget https://dl.google.com/android/repository/android-ndk-r27d-linux.zip -O android-ndk.zip unzip android-ndk.zip -d . rm android-ndk.zip ls -l + - name: Build OpenCL-ICD-Loader + shell: bash + run: | + cmake -B third_party/opencl-icd-loader/build -S third_party/opencl-icd-loader -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="$(pwd)/android-ndk-r27d/build/cmake/android.toolchain.cmake" \ + -DANDROID_ABI="arm64-v8a" \ + -DANDROID_PLATFORM="35" \ + -DOPENCL_ICD_LOADER_HEADERS_DIR="$(pwd)/third_party/opencl-headers" \ + -DCMAKE_BUILD_TYPE=Release + cmake --build third_party/opencl-icd-loader/build - name: Build shell: bash run: | @@ -53,6 +67,7 @@ jobs: -DANDROID_PLATFORM="35" \ -DPERFETTO_SDK_PATH="third_party/perfetto/sdk" \ -DOPENCL_HEADER_PATH="third_party/opencl-headers" \ + -DOPENCL_LIBRARY_PATH="$(pwd)/third_party/opencl-icd-loader/build" \ -DCMAKE_BUILD_TYPE=Release cmake --build build build-macos: @@ -60,7 +75,7 @@ jobs: needs: format runs-on: macos-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Clone dependencies @@ -80,7 +95,7 @@ jobs: needs: format runs-on: windows-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Clone dependencies @@ -102,7 +117,7 @@ jobs: needs: format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Install dependencies @@ -110,7 +125,7 @@ jobs: sudo apt-get update sudo apt-get install ninja-build clang-18 - name: Setup ccache - uses: hendrikmuhs/ccache-action@v1.2 + uses: hendrikmuhs/ccache-action@5ebbd400eff9e74630f759d94ddd7b6c26299639 # v1.2 with: variant: ccache max-size: 2G @@ -161,6 +176,7 @@ jobs: cmake -B build_debug -S . -G Ninja \ -DPERFETTO_SDK_PATH="${{ github.workspace }}/third_party/perfetto/sdk/" \ -DOPENCL_HEADER_PATH="$(pwd)/third_party/opencl-headers" \ + -DOPENCL_LIBRARY_PATH="$(pwd)/third_party/opencl-icd-loader/build" \ -DCMAKE_CXX_COMPILER="$(which clang++)" \ -DCMAKE_BUILD_TYPE=Debug cmake --build build_debug @@ -169,6 +185,7 @@ jobs: cmake -B build_release -S . -G Ninja \ -DPERFETTO_SDK_PATH="${{ github.workspace }}/third_party/perfetto/sdk/" \ -DOPENCL_HEADER_PATH="$(pwd)/third_party/opencl-headers" \ + -DOPENCL_LIBRARY_PATH="$(pwd)/third_party/opencl-icd-loader/build" \ -DCMAKE_CXX_COMPILER="$(which clang++)" \ -DCMAKE_C_COMPILER="$(which clang)" \ -DCLKP_BUILD_TESTS=1 \ diff --git a/Android.bp.example b/Android.bp.example index 280baaf..b14fa4c 100644 --- a/Android.bp.example +++ b/Android.bp.example @@ -26,13 +26,37 @@ license { cc_library_shared { name: "opencl-kernel-profiler", - srcs: ["src/opencl-kernel-profiler.cpp"], + srcs: ["layer/layer.cpp"], + local_include_dirs: ["include"], shared_libs: ["liblog"], static_libs: ["libperfetto_client_experimental"], header_libs: ["OpenCL-Headers"], soc_specific: true, } +cc_binary { + name: "clkp-extractor", + host_supported: true, + srcs: ["extractor/extractor.cpp"], + local_include_dirs: ["include"], + shared_libs: ["liblog"], + static_libs: ["libperfetto_client_experimental"], + soc_specific: true, +} + +cc_binary { + name: "clkp-runner", + srcs: ["runner/runner.cpp"], + local_include_dirs: ["include"], + shared_libs: [ + "//external/OpenCL-ICD-Loader:libOpenCL", + "liblog", + ], + static_libs: ["libperfetto_client_experimental"], + header_libs: ["OpenCL-Headers"], + soc_specific: true, +} + cc_genrule { name: "opencl-kernel-profiler-path", out: ["opencl-kernel-profiler.lay"], diff --git a/CMakeLists.txt b/CMakeLists.txt index 4425dcd..a18bb76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,8 +44,8 @@ if (SPIRV_DISASSEMBLY) endif() endif() -add_library (opencl-kernel-profiler SHARED src/opencl-kernel-profiler.cpp) -target_include_directories(opencl-kernel-profiler PUBLIC ${PERFETTO_SDK_PATH}) +add_library (opencl-kernel-profiler SHARED layer/layer.cpp) +target_include_directories(opencl-kernel-profiler PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${PERFETTO_SDK_PATH}) if (OPENCL_HEADER_PATH) target_include_directories(opencl-kernel-profiler PUBLIC ${OPENCL_HEADER_PATH}) endif() @@ -89,6 +89,33 @@ install(TARGETS opencl-kernel-profiler LIBRARY DESTINATION lib RUNTIME DESTINATION bin) +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR ANDROID) + add_executable(clkp-extractor extractor/extractor.cpp) + target_include_directories(clkp-extractor PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${PERFETTO_SDK_PATH}) + target_link_libraries(clkp-extractor ${PERFETTO_LIBRARY}) + target_compile_options(clkp-extractor PUBLIC "-DCLKP_PERFETTO_AMALGAMATED") + if (ANDROID) + target_link_libraries(clkp-extractor log) + endif() + + add_executable(clkp-runner runner/runner.cpp) + target_include_directories(clkp-runner PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${PERFETTO_SDK_PATH}) + if (OPENCL_HEADER_PATH) + target_include_directories(clkp-runner PUBLIC ${OPENCL_HEADER_PATH}) + endif() + if (OPENCL_LIBRARY_PATH) + target_link_directories(clkp-runner PUBLIC ${OPENCL_LIBRARY_PATH}) + endif() + target_link_libraries(clkp-runner OpenCL ${PERFETTO_LIBRARY}) + target_compile_options(clkp-runner PUBLIC "-DCLKP_PERFETTO_AMALGAMATED") + if (ANDROID) + target_link_libraries(clkp-runner log) + endif() + + install(TARGETS clkp-extractor clkp-runner + RUNTIME DESTINATION bin) +endif() + if (CLKP_BUILD_TESTS) add_subdirectory(tests) endif () diff --git a/README.md b/README.md index 7ec830a..7825a8f 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,10 @@ Then run the application using `opencl-kernel-profiler.sh`. This script will tak # Wait for the device to reboot, then: adb root adb remount + adb shell mkdir -p /vendor/etc/Khronos/OpenCL/layers/ adb push $OUT/vendor/lib64/opencl-kernel-profiler.so /vendor/lib64/ adb push $OUT/vendor/etc/Khronos/OpenCL/layers/opencl-kernel-profiler.lay /vendor/etc/Khronos/OpenCL/layers/ + adb shell chcon u:object_r:same_process_hal_file:s0 /vendor/lib64/opencl-kernel-profiler.so ``` Any application using the `OpenCL-ICD-Loader` will go through the `opencl-kernel-profiler`. diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp new file mode 100644 index 0000000..6560c63 --- /dev/null +++ b/extractor/extractor.cpp @@ -0,0 +1,657 @@ +// Copyright 2026 The OpenCL Kernel Profiler authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "clkp_common.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using TrackDescriptorMap = std::map>; +using SequenceTrackKey = std::pair; + +struct SliceContext { + bool is_clkp = false; + uint64_t call_id = 0; + uint64_t dispatch_id = 0; + uint64_t cmdbuf_id = 0; + uint64_t begin_timestamp = 0; + std::string name; + size_t begin_event_index = 0; +}; + +struct DispatchSummary { + uint64_t dispatch_id = 0; + uint64_t call_id = 0; + std::string api_name; + std::string program; + std::string kernel_name; + uint64_t cmdbuf_id = 0; + std::string gws; + std::string lws; + uint64_t gpu_duration_ns = 0; + bool has_gpu_duration = false; +}; + +struct ExtractorOptions { + std::string input_path; + std::string output_path; + bool list_dispatches = false; + bool has_start_dispatch = false; + bool has_end_dispatch = false; + uint64_t start_dispatch = 0; + uint64_t end_dispatch = UINT64_MAX; + bool has_start_call = false; + bool has_end_call = false; + uint64_t start_call = 0; + uint64_t end_call = UINT64_MAX; + + bool has_window() const { return has_start_dispatch || has_end_dispatch || has_start_call || has_end_call; } +}; + +void print_usage(const char *prog) +{ + std::cout << "Usage: " << prog << " [options] -i [-o ]\n" + << "\nOptions:\n" + << " -i, --input Input Perfetto trace file\n" + << " -o, --output Output filtered Perfetto trace file\n" + << " -l, --list-dispatches List all OpenCL dispatches and command buffer executions\n" + << " --start-dispatch Start extraction window at dispatch_id (inclusive)\n" + << " --end-dispatch End extraction window at dispatch_id (inclusive)\n" + << " --start-call Start extraction window at call_id (inclusive)\n" + << " --end-call End extraction window at call_id (inclusive)\n" + << " -h, --help Show this help message\n"; +} + +bool parse_args(int argc, char **argv, ExtractorOptions &opts) +{ + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + print_usage(argv[0]); + std::exit(0); + } else if ((arg == "-i" || arg == "--input") && i + 1 < argc) { + opts.input_path = argv[++i]; + } else if ((arg == "-o" || arg == "--output") && i + 1 < argc) { + opts.output_path = argv[++i]; + } else if (arg == "-l" || arg == "--list-dispatches") { + opts.list_dispatches = true; + } else if (arg == "--start-dispatch" && i + 1 < argc) { + opts.has_start_dispatch = true; + opts.start_dispatch = std::strtoull(argv[++i], nullptr, 10); + } else if (arg == "--end-dispatch" && i + 1 < argc) { + opts.has_end_dispatch = true; + opts.end_dispatch = std::strtoull(argv[++i], nullptr, 10); + } else if (arg == "--start-call" && i + 1 < argc) { + opts.has_start_call = true; + opts.start_call = std::strtoull(argv[++i], nullptr, 10); + } else if (arg == "--end-call" && i + 1 < argc) { + opts.has_end_call = true; + opts.end_call = std::strtoull(argv[++i], nullptr, 10); + } else if (arg[0] != '-' && opts.input_path.empty()) { + opts.input_path = arg; + } else { + std::cerr << "Unknown or incomplete option: " << arg << "\n"; + return false; + } + } + if (opts.input_path.empty()) { + std::cerr << "Error: missing input trace file (-i ).\n"; + return false; + } + if (!opts.list_dispatches && opts.output_path.empty()) { + std::cerr << "Error: missing output trace file (-o ) or --list-dispatches.\n"; + return false; + } + return true; +} + +bool is_creation_or_setup_event(const std::string &name) +{ + static const std::unordered_set kSetupEvents = { + "clCreateContext", + "clCreateContextFromType", + "clCreateCommandQueue", + "clCreateCommandQueueWithProperties", + "clCreateCommandQueue-properties", + "clCreateCommandQueue-properties-not-found", + "clCreateProgramWithSource", + "clCreateProgramWithSource-args", + "clCreateProgramWithIL", + "clCreateProgramWithIL-args", + "clCreateProgramWithBinary", + "clCreateProgramWithBinary-args", + "clBuildProgram", + "clCompileProgram", + "clLinkProgram", + "clSetProgramSpecializationConstant", + "clCreateKernel", + "clCreateKernelsInProgram", + "clCreateKernelsInProgram-kernel", + "clCloneKernel", + "clSetKernelArg", + "clCreateBuffer", + "clCreateBufferWithProperties", + "clCreateSubBuffer", + "clCreateImage", + "clCreateImageWithProperties", + "clCreateImage2D", + "clCreateImage3D", + "clCreateSampler", + "clCreateSamplerWithProperties", + "clCreateCommandBufferKHR", + "clFinalizeCommandBufferKHR", + "clCommandBarrierWithWaitListKHR", + "clCommandCopyBufferKHR", + "clCommandCopyBufferRectKHR", + "clCommandCopyBufferToImageKHR", + "clCommandCopyImageKHR", + "clCommandCopyImageToBufferKHR", + "clCommandFillBufferKHR", + "clCommandFillImageKHR", + "clCommandNDRangeKernelKHR", + "clUpdateMutableCommandsKHR", + "clUpdateMutableCommandsKHR-config", + "clUpdateMutableCommandsKHR-arg", + }; + return kSetupEvents.count(name) > 0 || name.rfind("clkp-queue_", 0) == 0; +} + +void add_uint_annotation(clkp::ParsedEvent &ev, const char *name, uint64_t val) +{ + clkp::ParsedAnnotation ann; + ann.name = name; + ann.type = clkp::ParsedAnnotation::ValueType::kUint; + ann.uint_val = val; + ev.annotations.push_back(std::move(ann)); +} + +void handle_slice_begin( + clkp::ParsedEvent &&ev, std::vector &stack, std::vector &clkp_events) +{ + bool is_clkp = ev.has_category(CLKP_PERFETTO_CATEGORY); + SliceContext ctx; + ctx.is_clkp = is_clkp; + ctx.call_id = ev.get_uint("call_id", 0); + ctx.dispatch_id = ev.get_uint("dispatch_id", UINT64_MAX); + ctx.cmdbuf_id = ev.get_uint("cmdbuf_id", 0); + ctx.begin_timestamp = ev.timestamp; + ctx.name = ev.name; + if (is_clkp) { + ctx.begin_event_index = clkp_events.size(); + clkp_events.push_back(std::move(ev)); + } + stack.push_back(std::move(ctx)); +} + +void handle_slice_end( + clkp::ParsedEvent &&ev, std::vector &stack, std::vector &clkp_events) +{ + if (stack.empty()) { + if (ev.has_category(CLKP_PERFETTO_CATEGORY)) { + clkp_events.push_back(std::move(ev)); + } + return; + } + + SliceContext top = stack.back(); + stack.pop_back(); + if (!top.is_clkp) { + return; + } + + if (top.call_id != 0) { + add_uint_annotation(ev, "call_id", top.call_id); + } + if (top.dispatch_id != UINT64_MAX) { + add_uint_annotation(ev, "dispatch_id", top.dispatch_id); + } + ev.name = top.name; + clkp_events.push_back(std::move(ev)); +} + +void handle_instant_event( + clkp::ParsedEvent &&ev, const std::vector &stack, std::vector &clkp_events) +{ + if (!ev.has_category(CLKP_PERFETTO_CATEGORY)) { + return; + } + if (!ev.has_key("call_id")) { + for (auto rit = stack.rbegin(); rit != stack.rend(); ++rit) { + if (rit->is_clkp && rit->call_id != 0) { + add_uint_annotation(ev, "call_id", rit->call_id); + break; + } + } + } + clkp_events.push_back(std::move(ev)); +} + +void parse_trace_packets(const std::vector &trace_bytes, std::vector &clkp_events, + TrackDescriptorMap &track_descriptors) +{ + clkp::TraceIncrementState inc_state; + std::map> open_slices; + + perfetto::protos::pbzero::Trace_Decoder trace_dec(trace_bytes.data(), trace_bytes.size()); + size_t packet_idx = 0; + for (auto pkt_it = trace_dec.packet(); pkt_it; ++pkt_it, ++packet_idx) { + protozero::ConstBytes pkt_bytes = *pkt_it; + perfetto::protos::pbzero::TracePacket_Decoder pkt_dec(pkt_bytes.data, pkt_bytes.size); + + inc_state.update_from_packet(pkt_dec); + + if (pkt_dec.has_track_descriptor()) { + perfetto::protos::pbzero::TrackDescriptor_Decoder td(pkt_dec.track_descriptor()); + if (td.has_uuid()) { + track_descriptors[td.uuid()] = { pkt_bytes.data, pkt_bytes.size }; + } + } + + clkp::ParsedEvent ev; + if (!inc_state.decode_track_event(pkt_dec, packet_idx, ev)) { + continue; + } + + auto &stack = open_slices[{ ev.sequence_id, ev.track_uuid }]; + if (ev.event_type == 1) { + handle_slice_begin(std::move(ev), stack, clkp_events); + } else if (ev.event_type == 2) { + handle_slice_end(std::move(ev), stack, clkp_events); + } else if (ev.event_type == 3) { + handle_instant_event(std::move(ev), stack, clkp_events); + } + } +} + +bool is_dispatch_api(const std::string &name) +{ + return name == "clEnqueueNDRangeKernel" || name == "clCommandNDRangeKernelKHR" + || name == "clEnqueueCommandBufferKHR"; +} + +void record_dispatch_begin(const clkp::ParsedEvent &ev, std::map &dispatches, + std::map> &gpu_slice_starts) +{ + uint64_t did = ev.get_uint("dispatch_id", UINT64_MAX); + if (did == UINT64_MAX) { + return; + } + + if (!is_dispatch_api(ev.name)) { + gpu_slice_starts[{ ev.sequence_id, ev.track_uuid }] = { did, ev.timestamp }; + return; + } + + auto &d = dispatches[did]; + d.dispatch_id = did; + d.call_id = ev.get_uint("call_id", 0); + d.api_name = ev.name; + d.program = ev.get_string("program", ""); + d.kernel_name = ev.get_string("kernel_name", ""); + d.cmdbuf_id = ev.get_uint("cmdbuf_id", 0); + d.gws = ev.get_string("gws", ""); + if (d.gws.empty() && ev.has_key("gidX")) { + d.gws = std::to_string(ev.get_uint("gidX", 1)) + "," + std::to_string(ev.get_uint("gidY", 1)) + "," + + std::to_string(ev.get_uint("gidZ", 1)); + } + d.lws = ev.get_string("lws", ""); +} + +void record_dispatch_end(const clkp::ParsedEvent &ev, std::map &dispatches, + std::map> &gpu_slice_starts) +{ + auto it = gpu_slice_starts.find({ ev.sequence_id, ev.track_uuid }); + if (it == gpu_slice_starts.end()) { + return; + } + uint64_t did = it->second.first; + uint64_t start_ts = it->second.second; + if (ev.timestamp >= start_ts && dispatches.count(did)) { + dispatches[did].gpu_duration_ns = ev.timestamp - start_ts; + dispatches[did].has_gpu_duration = true; + } + gpu_slice_starts.erase(it); +} + +std::map build_dispatch_summaries(const std::vector &clkp_events) +{ + std::map dispatches; + std::map> gpu_slice_starts; + + for (const auto &ev : clkp_events) { + if (ev.event_type == 1) { + record_dispatch_begin(ev, dispatches, gpu_slice_starts); + } else if (ev.event_type == 2) { + record_dispatch_end(ev, dispatches, gpu_slice_starts); + } + } + return dispatches; +} + +void print_dispatch_table(const std::map &dispatches) +{ + std::cout << std::left << std::setw(12) << "DISPATCH_ID" << std::setw(10) << "CALL_ID" << std::setw(28) << "API" + << std::setw(12) << "PROGRAM" << std::setw(24) << "KERNEL/CMDBUF" << std::setw(16) << "GWS" + << std::setw(16) << "LWS" << "GPU_TIME_US\n"; + for (const auto &[did, d] : dispatches) { + std::string target = !d.kernel_name.empty() ? d.kernel_name : ("cmdbuf_" + std::to_string(d.cmdbuf_id)); + std::cout << std::left << std::setw(12) << d.dispatch_id << std::setw(10) << d.call_id << std::setw(28) + << d.api_name << std::setw(12) << d.program << std::setw(24) << target << std::setw(16) << d.gws + << std::setw(16) << (d.lws.empty() ? "NULL" : d.lws); + if (d.has_gpu_duration) { + std::cout << std::fixed << std::setprecision(2) << (static_cast(d.gpu_duration_ns) / 1000.0); + } else { + std::cout << "N/A"; + } + std::cout << "\n"; + } +} + +bool resolve_call_window(const ExtractorOptions &opts, const std::map &dispatches, + uint64_t &window_start_call, uint64_t &window_end_call) +{ + window_start_call = opts.has_start_call ? opts.start_call : 0; + window_end_call = opts.has_end_call ? opts.end_call : UINT64_MAX; + + if (!opts.has_start_dispatch && !opts.has_end_dispatch) { + return true; + } + + uint64_t min_call = UINT64_MAX; + uint64_t max_call = 0; + for (const auto &[did, d] : dispatches) { + if (did >= opts.start_dispatch && did <= opts.end_dispatch) { + min_call = std::min(min_call, d.call_id); + max_call = std::max(max_call, d.call_id); + } + } + if (min_call == UINT64_MAX) { + std::cerr << "Error: no dispatches found in requested dispatch range.\n"; + return false; + } + if (!opts.has_start_call) { + window_start_call = min_call; + } + if (!opts.has_end_call) { + window_end_call = max_call; + } + return true; +} + +struct DependencyClosure { + std::unordered_set contexts; + std::unordered_set queues; + std::unordered_set programs; + std::unordered_set kernels; + std::unordered_set mems; + std::unordered_set samplers; + std::unordered_set cmdbufs; + std::unordered_set kept_call_ids; + + static void insert_if_nonzero(std::unordered_set &set, uint64_t id) + { + if (id != 0) { + set.insert(id); + } + } + + static void insert_list_if_nonzero(std::unordered_set &set, const std::string &csv) + { + for (uint64_t id : clkp::string_to_uint64_list(csv)) { + insert_if_nonzero(set, id); + } + } + + void collect_refs_from_event(const clkp::ParsedEvent &ev) + { + insert_if_nonzero(contexts, ev.get_uint("context_id", 0)); + insert_if_nonzero(queues, ev.get_uint("queue_id", 0)); + insert_list_if_nonzero(queues, ev.get_string("queue_ids")); + + insert_if_nonzero(programs, ev.get_uint("program_id", 0)); + insert_list_if_nonzero(programs, ev.get_string("header_program_ids")); + insert_list_if_nonzero(programs, ev.get_string("input_program_ids")); + + insert_if_nonzero(kernels, ev.get_uint("kernel_id", 0)); + insert_if_nonzero(kernels, ev.get_uint("src_kernel_id", 0)); + + insert_if_nonzero(mems, ev.get_uint("mem_id", 0)); + insert_if_nonzero(mems, ev.get_uint("src_mem_id", 0)); + insert_if_nonzero(mems, ev.get_uint("dst_mem_id", 0)); + insert_if_nonzero(mems, ev.get_uint("parent_mem_id", 0)); + insert_list_if_nonzero(mems, ev.get_string("mem_ids")); + + insert_if_nonzero(samplers, ev.get_uint("sampler_id", 0)); + insert_if_nonzero(cmdbufs, ev.get_uint("cmdbuf_id", 0)); + } + + bool matches_needed_resource(const clkp::ParsedEvent &ev) const + { + if (ev.has_key("cmdbuf_id") && cmdbufs.count(ev.get_uint("cmdbuf_id")) > 0) { + return true; + } + if (ev.has_key("kernel_id") && kernels.count(ev.get_uint("kernel_id")) > 0) { + return true; + } + if (ev.has_key("program_id") && programs.count(ev.get_uint("program_id")) > 0) { + return true; + } + bool is_arg_set = (ev.name == "clSetKernelArg" || ev.name == "clUpdateMutableCommandsKHR-arg"); + if (!is_arg_set && ev.has_key("mem_id") && mems.count(ev.get_uint("mem_id")) > 0) { + return true; + } + if (!is_arg_set && ev.has_key("sampler_id") && samplers.count(ev.get_uint("sampler_id")) > 0) { + return true; + } + bool is_queue_setup = (ev.name == "clCreateCommandQueue" || ev.name == "clCreateCommandQueueWithProperties" + || ev.name.rfind("clkp-queue_", 0) == 0); + if (is_queue_setup && ev.has_key("queue_id") && queues.count(ev.get_uint("queue_id")) > 0) { + return true; + } + bool is_ctx_setup = (ev.name == "clCreateContext" || ev.name == "clCreateContextFromType"); + if (is_ctx_setup && ev.has_key("context_id") && contexts.count(ev.get_uint("context_id")) > 0) { + return true; + } + return false; + } +}; + +std::vector compute_retained_events(const std::vector &clkp_events, bool has_window, + uint64_t window_start_call, uint64_t window_end_call) +{ + std::vector keep_event(clkp_events.size(), !has_window); + if (!has_window) { + return keep_event; + } + + DependencyClosure closure; + for (size_t i = 0; i < clkp_events.size(); ++i) { + const auto &ev = clkp_events[i]; + uint64_t cid = ev.get_uint("call_id", 0); + if (cid != 0 && cid >= window_start_call && cid <= window_end_call) { + keep_event[i] = true; + closure.kept_call_ids.insert(cid); + closure.collect_refs_from_event(ev); + } + } + + bool changed = true; + while (changed) { + changed = false; + for (size_t i = 0; i < clkp_events.size(); ++i) { + if (keep_event[i]) { + continue; + } + const auto &ev = clkp_events[i]; + uint64_t cid = ev.get_uint("call_id", 0); + if (cid > window_end_call) { + continue; + } + bool keep_by_call = (cid != 0 && closure.kept_call_ids.count(cid) > 0); + bool keep_by_resource = is_creation_or_setup_event(ev.name) && closure.matches_needed_resource(ev); + if (keep_by_call || keep_by_resource) { + keep_event[i] = true; + if (cid != 0) { + closure.kept_call_ids.insert(cid); + } + closure.collect_refs_from_event(ev); + changed = true; + } + } + } + + return keep_event; +} + +void serialize_annotation(const clkp::ParsedAnnotation &ann, perfetto::protos::pbzero::DebugAnnotation *da) +{ + da->set_name(ann.name.c_str()); + switch (ann.type) { + case clkp::ParsedAnnotation::ValueType::kString: + da->set_string_value(ann.string_val.c_str()); + break; + case clkp::ParsedAnnotation::ValueType::kUint: + da->set_uint_value(ann.uint_val); + break; + case clkp::ParsedAnnotation::ValueType::kInt: + da->set_int_value(ann.int_val); + break; + case clkp::ParsedAnnotation::ValueType::kPointer: + da->set_pointer_value(ann.uint_val); + break; + case clkp::ParsedAnnotation::ValueType::kBool: + da->set_bool_value(ann.bool_val); + break; + case clkp::ParsedAnnotation::ValueType::kDouble: + da->set_double_value(ann.double_val); + break; + default: + break; + } +} + +void serialize_track_event(const clkp::ParsedEvent &ev, perfetto::protos::pbzero::TracePacket *pkt) +{ + pkt->set_timestamp(ev.timestamp); + pkt->set_trusted_packet_sequence_id(ev.sequence_id != 0 ? ev.sequence_id : 1); + + auto *te = pkt->set_track_event(); + te->set_type(static_cast(ev.event_type)); + if (ev.track_uuid != 0) { + te->set_track_uuid(ev.track_uuid); + } + if (ev.event_type == 2) { + return; + } + + if (ev.categories.empty()) { + te->add_categories(CLKP_PERFETTO_CATEGORY); + } else { + for (const auto &cat : ev.categories) { + te->add_categories(cat.c_str()); + } + } + if (!ev.name.empty()) { + te->set_name(ev.name.c_str()); + } + for (const auto &ann : ev.annotations) { + serialize_annotation(ann, te->add_debug_annotations()); + } +} + +bool write_filtered_trace(const std::string &output_path, const std::vector &clkp_events, + const std::vector &keep_event, const TrackDescriptorMap &track_descriptors) +{ + protozero::HeapBuffered out_trace; + std::unordered_set emitted_tracks; + + for (size_t i = 0; i < clkp_events.size(); ++i) { + if (!keep_event[i]) { + continue; + } + const auto &ev = clkp_events[i]; + if (ev.track_uuid != 0 && emitted_tracks.insert(ev.track_uuid).second) { + auto td_it = track_descriptors.find(ev.track_uuid); + if (td_it != track_descriptors.end()) { + auto *raw_pkt = out_trace->add_packet(); + raw_pkt->AppendRawProtoBytes(td_it->second.first, td_it->second.second); + } + } + serialize_track_event(ev, out_trace->add_packet()); + } + + std::vector serialized = out_trace.SerializeAsArray(); + std::ofstream out_file(output_path, std::ios::out | std::ios::binary | std::ios::trunc); + if (!out_file.is_open()) { + std::cerr << "Error: failed to open output file '" << output_path << "'\n"; + return false; + } + out_file.write(reinterpret_cast(serialized.data()), serialized.size()); + return true; +} + +} // namespace + +int main(int argc, char **argv) +{ + ExtractorOptions opts; + if (!parse_args(argc, argv, opts)) { + print_usage(argv[0]); + return 1; + } + + std::vector trace_bytes; + if (!clkp::read_file_bytes(opts.input_path, trace_bytes)) { + std::cerr << "Error: failed to read input file '" << opts.input_path << "'\n"; + return 1; + } + + std::vector clkp_events; + TrackDescriptorMap track_descriptors; + parse_trace_packets(trace_bytes, clkp_events, track_descriptors); + + auto dispatches = build_dispatch_summaries(clkp_events); + if (opts.list_dispatches) { + print_dispatch_table(dispatches); + if (opts.output_path.empty()) { + return 0; + } + } + + uint64_t window_start_call = 0; + uint64_t window_end_call = UINT64_MAX; + if (!resolve_call_window(opts, dispatches, window_start_call, window_end_call)) { + return 1; + } + + auto keep_event = compute_retained_events(clkp_events, opts.has_window(), window_start_call, window_end_call); + if (!write_filtered_trace(opts.output_path, clkp_events, keep_event, track_descriptors)) { + return 1; + } + + return 0; +} diff --git a/include/clkp_common.hpp b/include/clkp_common.hpp new file mode 100644 index 0000000..c8614bb --- /dev/null +++ b/include/clkp_common.hpp @@ -0,0 +1,457 @@ +// Copyright 2026 The OpenCL Kernel Profiler authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CLKP_COMMON_HPP +#define CLKP_COMMON_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __ANDROID__ +#include +#endif + +#ifdef CLKP_PERFETTO_AMALGAMATED +#include "perfetto.h" +#else +#include "perfetto/tracing.h" +#include "protos/perfetto/trace/interned_data/interned_data.pbzero.h" +#include "protos/perfetto/trace/profiling/profile_common.pbzero.h" +#include "protos/perfetto/trace/trace.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "protos/perfetto/trace/trace_packet_defaults.pbzero.h" +#include "protos/perfetto/trace/track_event/debug_annotation.pbzero.h" +#include "protos/perfetto/trace/track_event/track_descriptor.pbzero.h" +#include "protos/perfetto/trace/track_event/track_event.pbzero.h" +#endif + +#define CLKP_PERFETTO_CATEGORY "clkp" + +namespace clkp { + +// Chunk size (in bytes) for splitting large source strings or binary blobs across TrackEvents. +static constexpr size_t kMaxPayloadChunkBytes = 4096; + +inline std::string bytes_to_hex(const void *data, size_t len) +{ + if (data == nullptr || len == 0) { + return ""; + } + static const char kHexDigits[] = "0123456789abcdef"; + const uint8_t *bytes = static_cast(data); + std::string out; + out.resize(len * 2); + for (size_t i = 0; i < len; ++i) { + out[i * 2] = kHexDigits[(bytes[i] >> 4) & 0xF]; + out[i * 2 + 1] = kHexDigits[bytes[i] & 0xF]; + } + return out; +} + +inline int hex_nibble(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return 0; +} + +inline std::vector hex_to_bytes(const std::string &hex) +{ + std::vector out; + size_t count = hex.size() / 2; + out.resize(count); + for (size_t i = 0; i < count; ++i) { + int hi = hex_nibble(hex[i * 2]); + int lo = hex_nibble(hex[i * 2 + 1]); + out[i] = static_cast((hi << 4) | lo); + } + return out; +} + +inline std::string uint64_list_to_string(const std::vector &values) +{ + std::ostringstream oss; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + oss << ","; + } + oss << values[i]; + } + return oss.str(); +} + +inline std::vector string_to_uint64_list(const std::string &str) +{ + std::vector result; + if (str.empty()) { + return result; + } + size_t start = 0; + while (start < str.size()) { + size_t comma = str.find(',', start); + std::string token = (comma == std::string::npos) ? str.substr(start) : str.substr(start, comma - start); + if (!token.empty()) { + result.push_back(std::strtoull(token.c_str(), nullptr, 10)); + } + if (comma == std::string::npos) { + break; + } + start = comma + 1; + } + return result; +} + +inline std::string string_list_to_joined(const std::vector &values, char sep = ';') +{ + std::ostringstream oss; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + oss << sep; + } + oss << values[i]; + } + return oss.str(); +} + +inline std::vector joined_to_string_list(const std::string &str, char sep = ';') +{ + std::vector result; + if (str.empty()) { + return result; + } + size_t start = 0; + while (start <= str.size()) { + size_t pos = str.find(sep, start); + if (pos == std::string::npos) { + result.push_back(str.substr(start)); + break; + } + result.push_back(str.substr(start, pos - start)); + start = pos + 1; + } + return result; +} + +struct ParsedAnnotation { + enum class ValueType { + kNone, + kBool, + kUint, + kInt, + kDouble, + kPointer, + kString, + }; + + std::string name; + ValueType type = ValueType::kNone; + bool bool_val = false; + uint64_t uint_val = 0; + int64_t int_val = 0; + double double_val = 0.0; + std::string string_val; +}; + +struct ParsedEvent { + uint64_t timestamp = 0; + uint32_t sequence_id = 0; + uint64_t track_uuid = 0; + int32_t event_type = 0; // TrackEvent::Type (1=SLICE_BEGIN, 2=SLICE_END, 3=INSTANT) + std::string name; + std::vector categories; + std::vector annotations; + size_t packet_index = 0; + + bool has_category(const char *cat) const + { + for (const auto &c : categories) { + if (c == cat) { + return true; + } + } + return false; + } + + const ParsedAnnotation *find_annotation(const char *key) const + { + for (const auto &ann : annotations) { + if (ann.name == key) { + return &ann; + } + } + return nullptr; + } + + bool has_key(const char *key) const { return find_annotation(key) != nullptr; } + + uint64_t get_uint(const char *key, uint64_t default_val = 0) const + { + const auto *ann = find_annotation(key); + if (!ann) { + return default_val; + } + switch (ann->type) { + case ParsedAnnotation::ValueType::kUint: + case ParsedAnnotation::ValueType::kPointer: + return ann->uint_val; + case ParsedAnnotation::ValueType::kInt: + return static_cast(ann->int_val); + case ParsedAnnotation::ValueType::kBool: + return ann->bool_val ? 1 : 0; + case ParsedAnnotation::ValueType::kString: + return std::strtoull(ann->string_val.c_str(), nullptr, 0); + default: + return default_val; + } + } + + int64_t get_int(const char *key, int64_t default_val = 0) const + { + const auto *ann = find_annotation(key); + if (!ann) { + return default_val; + } + switch (ann->type) { + case ParsedAnnotation::ValueType::kInt: + return ann->int_val; + case ParsedAnnotation::ValueType::kUint: + case ParsedAnnotation::ValueType::kPointer: + return static_cast(ann->uint_val); + case ParsedAnnotation::ValueType::kBool: + return ann->bool_val ? 1 : 0; + case ParsedAnnotation::ValueType::kString: + return std::strtoll(ann->string_val.c_str(), nullptr, 0); + default: + return default_val; + } + } + + std::string get_string(const char *key, const std::string &default_val = "") const + { + const auto *ann = find_annotation(key); + if (!ann) { + return default_val; + } + if (ann->type == ParsedAnnotation::ValueType::kString) { + return ann->string_val; + } + if (ann->type == ParsedAnnotation::ValueType::kUint || ann->type == ParsedAnnotation::ValueType::kPointer) { + return std::to_string(ann->uint_val); + } + if (ann->type == ParsedAnnotation::ValueType::kInt) { + return std::to_string(ann->int_val); + } + return default_val; + } +}; + +struct SequenceInternTable { + std::unordered_map categories; + std::unordered_map event_names; + std::unordered_map annotation_names; + std::unordered_map annotation_strings; + uint64_t default_track_uuid = 0; + uint32_t default_clock_id = 0; + + void clear() + { + categories.clear(); + event_names.clear(); + annotation_names.clear(); + annotation_strings.clear(); + default_track_uuid = 0; + default_clock_id = 0; + } +}; + +class TraceIncrementState { +public: + SequenceInternTable &get_sequence(uint32_t seq_id) { return sequences_[seq_id]; } + + void update_from_packet(const perfetto::protos::pbzero::TracePacket_Decoder &pkt) + { + uint32_t seq_id = pkt.has_trusted_packet_sequence_id() ? pkt.trusted_packet_sequence_id() : 0; + auto &seq = sequences_[seq_id]; + + // SequenceFlags::SEQ_INCREMENTAL_STATE_CLEARED == 1 + if ((pkt.has_sequence_flags() && (pkt.sequence_flags() & 1u)) + || (pkt.has_incremental_state_cleared() && pkt.incremental_state_cleared())) { + seq.clear(); + } + + if (pkt.has_trace_packet_defaults()) { + perfetto::protos::pbzero::TracePacketDefaults_Decoder defs(pkt.trace_packet_defaults()); + if (defs.has_timestamp_clock_id()) { + seq.default_clock_id = defs.timestamp_clock_id(); + } + if (defs.has_track_event_defaults()) { + perfetto::protos::pbzero::TrackEventDefaults_Decoder ted(defs.track_event_defaults()); + if (ted.has_track_uuid()) { + seq.default_track_uuid = ted.track_uuid(); + } + } + } + + if (pkt.has_interned_data()) { + perfetto::protos::pbzero::InternedData_Decoder idata(pkt.interned_data()); + for (auto it = idata.event_categories(); it; ++it) { + perfetto::protos::pbzero::EventCategory_Decoder cat(*it); + if (cat.has_iid() && cat.has_name()) { + seq.categories[cat.iid()] = cat.name().ToStdString(); + } + } + for (auto it = idata.event_names(); it; ++it) { + perfetto::protos::pbzero::EventName_Decoder en(*it); + if (en.has_iid() && en.has_name()) { + seq.event_names[en.iid()] = en.name().ToStdString(); + } + } + for (auto it = idata.debug_annotation_names(); it; ++it) { + perfetto::protos::pbzero::DebugAnnotationName_Decoder an(*it); + if (an.has_iid() && an.has_name()) { + seq.annotation_names[an.iid()] = an.name().ToStdString(); + } + } + for (auto it = idata.debug_annotation_string_values(); it; ++it) { + perfetto::protos::pbzero::InternedString_Decoder is(*it); + if (is.has_iid() && is.has_str()) { + auto bytes = is.str(); + seq.annotation_strings[is.iid()] + = std::string(reinterpret_cast(bytes.data), bytes.size); + } + } + } + } + + bool decode_track_event( + const perfetto::protos::pbzero::TracePacket_Decoder &pkt, size_t packet_index, ParsedEvent &out_event) + { + if (!pkt.has_track_event()) { + return false; + } + uint32_t seq_id = pkt.has_trusted_packet_sequence_id() ? pkt.trusted_packet_sequence_id() : 0; + const auto &seq = sequences_[seq_id]; + + perfetto::protos::pbzero::TrackEvent_Decoder te(pkt.track_event()); + out_event = ParsedEvent(); + out_event.packet_index = packet_index; + out_event.sequence_id = seq_id; + out_event.timestamp = pkt.has_timestamp() ? pkt.timestamp() : 0; + out_event.event_type = te.has_type() ? te.type() : 0; + out_event.track_uuid = te.has_track_uuid() ? te.track_uuid() : seq.default_track_uuid; + + for (auto it = te.categories(); it; ++it) { + out_event.categories.push_back((*it).ToStdString()); + } + for (auto it = te.category_iids(); it; ++it) { + uint64_t iid = *it; + auto found = seq.categories.find(iid); + if (found != seq.categories.end()) { + out_event.categories.push_back(found->second); + } + } + + if (te.has_name()) { + out_event.name = te.name().ToStdString(); + } else if (te.has_name_iid()) { + auto found = seq.event_names.find(te.name_iid()); + if (found != seq.event_names.end()) { + out_event.name = found->second; + } + } + + for (auto it = te.debug_annotations(); it; ++it) { + perfetto::protos::pbzero::DebugAnnotation_Decoder da(*it); + ParsedAnnotation ann; + if (da.has_name()) { + ann.name = da.name().ToStdString(); + } else if (da.has_name_iid()) { + auto found = seq.annotation_names.find(da.name_iid()); + if (found != seq.annotation_names.end()) { + ann.name = found->second; + } + } + + if (da.has_string_value()) { + ann.type = ParsedAnnotation::ValueType::kString; + ann.string_val = da.string_value().ToStdString(); + } else if (da.has_string_value_iid()) { + ann.type = ParsedAnnotation::ValueType::kString; + auto found = seq.annotation_strings.find(da.string_value_iid()); + if (found != seq.annotation_strings.end()) { + ann.string_val = found->second; + } + } else if (da.has_uint_value()) { + ann.type = ParsedAnnotation::ValueType::kUint; + ann.uint_val = da.uint_value(); + } else if (da.has_int_value()) { + ann.type = ParsedAnnotation::ValueType::kInt; + ann.int_val = da.int_value(); + } else if (da.has_pointer_value()) { + ann.type = ParsedAnnotation::ValueType::kPointer; + ann.uint_val = da.pointer_value(); + } else if (da.has_bool_value()) { + ann.type = ParsedAnnotation::ValueType::kBool; + ann.bool_val = da.bool_value(); + } else if (da.has_double_value()) { + ann.type = ParsedAnnotation::ValueType::kDouble; + ann.double_val = da.double_value(); + } + out_event.annotations.push_back(std::move(ann)); + } + + return true; + } + +private: + std::unordered_map sequences_; +}; + +inline bool read_file_bytes(const std::string &path, std::vector &out) +{ + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + return false; + } + std::streamsize size = file.tellg(); + if (size < 0) { + return false; + } + file.seekg(0, std::ios::beg); + out.resize(static_cast(size)); + if (size > 0 && !file.read(reinterpret_cast(out.data()), size)) { + return false; + } + return true; +} + +} // namespace clkp + +#endif // CLKP_COMMON_HPP diff --git a/layer/layer.cpp b/layer/layer.cpp new file mode 100644 index 0000000..45b50b3 --- /dev/null +++ b/layer/layer.cpp @@ -0,0 +1,3249 @@ +// Copyright 2023 The OpenCL Kernel Profiler authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define CL_TARGET_OPENCL_VERSION 300 +#define CL_ENABLE_BETA_EXTENSIONS +#define CL_USE_DEPRECATED_OPENCL_1_0_APIS +#define CL_USE_DEPRECATED_OPENCL_1_1_APIS +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS +#define CL_USE_DEPRECATED_OPENCL_2_0_APIS + +#include "clkp_common.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef SPIRV_DISASSEMBLY +#include +#endif + +/*****************************************************************************/ +/* PERFETTO GLOBAL VARIABLES *************************************************/ +/*****************************************************************************/ + +PERFETTO_DEFINE_CATEGORIES(perfetto::Category(CLKP_PERFETTO_CATEGORY).SetDescription("OpenCL Kernel Profiler Events")); + +PERFETTO_TRACK_EVENT_STATIC_STORAGE(); + +#ifdef BACKEND_INPROCESS +static perfetto::TracingSession *gTracingSession; +#endif + +/*****************************************************************************/ +/* ICD DISPATCH GLOBAL VARIABLES *********************************************/ +/*****************************************************************************/ + +static struct _cl_icd_dispatch dispatch; +static const struct _cl_icd_dispatch *tdispatch; + +/*****************************************************************************/ +/* MACROS ********************************************************************/ +/*****************************************************************************/ + +#ifdef __ANDROID__ +#define PRINT(message, ...) \ + do { \ + __android_log_print( \ + android_LogPriority::ANDROID_LOG_ERROR, "CLKP", " %s: " message "\n", __func__, ##__VA_ARGS__); \ + } while (0) +#else +#define PRINT(message, ...) \ + do { \ + fprintf(stderr, "[CLKP] %s: " message "\n", __func__, ##__VA_ARGS__); \ + } while (0) +#endif +#define CHECK(test, statement, message, ...) \ + do { \ + if (!(test)) { \ + PRINT(message, ##__VA_ARGS__); \ + statement; \ + } \ + } while (0) +#define CHECK_CL(err, statement, message, ...) CHECK((err) == CL_SUCCESS, statement, message, ##__VA_ARGS__) +#define CHECK_ALLOC(ptr, statement) CHECK(ptr != nullptr, statement, "allocation failed") + +/*****************************************************************************/ +/* STATE TRACKING & SYNTHETIC IDS ********************************************/ +/*****************************************************************************/ + +static std::recursive_mutex g_lock; +static std::atomic g_next_call_id { 1 }; +static std::atomic g_next_dispatch_id { 0 }; + +static uint64_t next_call_id() { return g_next_call_id.fetch_add(1, std::memory_order_relaxed); } +static uint64_t next_dispatch_id() { return g_next_dispatch_id.fetch_add(1, std::memory_order_relaxed); } + +template class HandleIdTracker { +public: + uint64_t allocate_id() { return next_id_++; } + + void bind(HandleT handle, uint64_t id) + { + if (handle != nullptr && id != 0) { + map_[handle] = id; + } + } + + uint64_t get_or_create(HandleT handle) + { + if (handle == nullptr) { + return 0; + } + auto it = map_.find(handle); + if (it != map_.end()) { + return it->second; + } + uint64_t id = next_id_++; + map_[handle] = id; + return id; + } + + uint64_t create_fresh(HandleT handle) + { + if (handle == nullptr) { + return 0; + } + uint64_t id = next_id_++; + map_[handle] = id; + return id; + } + + uint64_t find(HandleT handle) const + { + if (handle == nullptr) { + return 0; + } + auto it = map_.find(handle); + return (it != map_.end()) ? it->second : 0; + } + + bool contains(HandleT handle) const + { + if (handle == nullptr) { + return false; + } + return map_.find(handle) != map_.end(); + } + +private: + std::unordered_map map_; + uint64_t next_id_ = 1; +}; + +static HandleIdTracker g_context_ids; +static HandleIdTracker g_queue_ids; +static HandleIdTracker g_program_ids; +static HandleIdTracker g_kernel_ids; +static HandleIdTracker g_mem_ids; +static HandleIdTracker g_sampler_ids; +static HandleIdTracker g_event_ids; +static HandleIdTracker g_map_ids; +static HandleIdTracker g_cmdbuf_ids; +static HandleIdTracker g_mutable_cmd_ids; + +static std::map program_to_string; +static std::map program_to_context_id; +static std::map kernel_to_kernel_name; +static std::map kernel_to_program; +static std::map> cmdbuf_to_queues; +static std::map cmdbuf_to_context_id; +static std::map queue_to_context_id; + +static std::string size_array_to_string(const size_t *arr, cl_uint count) +{ + if (arr == nullptr || count == 0) { + return ""; + } + std::vector vals(count); + for (cl_uint i = 0; i < count; ++i) { + vals[i] = static_cast(arr[i]); + } + return clkp::uint64_list_to_string(vals); +} + +static std::string event_list_to_string(cl_uint num_events, const cl_event *event_list) +{ + if (event_list == nullptr || num_events == 0) { + return ""; + } + std::vector ids; + ids.reserve(num_events); + for (cl_uint i = 0; i < num_events; ++i) { + ids.push_back(g_event_ids.get_or_create(event_list[i])); + } + return clkp::uint64_list_to_string(ids); +} + +static std::string sync_point_list_to_string(cl_uint num_sync_points, const cl_sync_point_khr *sync_points) +{ + if (sync_points == nullptr || num_sync_points == 0) { + return ""; + } + std::vector ids(num_sync_points); + for (cl_uint i = 0; i < num_sync_points; ++i) { + ids[i] = static_cast(sync_points[i]); + } + return clkp::uint64_list_to_string(ids); +} + +template static std::string properties_to_string(const PropT *props) +{ + if (props == nullptr) { + return ""; + } + std::vector vals; + for (size_t i = 0; props[i] != 0; i += 2) { + vals.push_back(static_cast(props[i])); + vals.push_back(static_cast(props[i + 1])); + } + return clkp::uint64_list_to_string(vals); +} + +/*****************************************************************************/ +/* CONTEXT LIFECYCLE *********************************************************/ +/*****************************************************************************/ + +static cl_context clkp_clCreateContext(const cl_context_properties *properties, cl_uint num_devices, + const cl_device_id *devices, + void(CL_CALLBACK *pfn_notify)(const char *errinfo, const void *private_info, size_t cb, void *user_data), + void *user_data, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.allocate_id(); + } + TRACE_EVENT( + CLKP_PERFETTO_CATEGORY, "clCreateContext", "call_id", cid, "context_id", ctx_id, "num_devices", num_devices); + cl_context ctx = tdispatch->clCreateContext(properties, num_devices, devices, pfn_notify, user_data, errcode_ret); + { + std::lock_guard lock(g_lock); + g_context_ids.bind(ctx, ctx_id); + } + return ctx; +} + +static cl_context clkp_clCreateContextFromType(const cl_context_properties *properties, cl_device_type device_type, + void(CL_CALLBACK *pfn_notify)(const char *errinfo, const void *private_info, size_t cb, void *user_data), + void *user_data, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateContextFromType", "call_id", cid, "context_id", ctx_id, "device_type", + static_cast(device_type)); + cl_context ctx = tdispatch->clCreateContextFromType(properties, device_type, pfn_notify, user_data, errcode_ret); + { + std::lock_guard lock(g_lock); + g_context_ids.bind(ctx, ctx_id); + } + return ctx; +} + +static cl_int clkp_clRetainContext(cl_context context) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainContext", "call_id", cid, "context_id", ctx_id); + return tdispatch->clRetainContext(context); +} + +static cl_int clkp_clReleaseContext(cl_context context) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.find(context); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseContext", "call_id", cid, "context_id", ctx_id); + return tdispatch->clReleaseContext(context); +} + +/*****************************************************************************/ +/* CREATE KERNEL & PROGRAM ***************************************************/ +/*****************************************************************************/ + +static void writeProgramOnDisk( + std::filesystem::path filename, cl_uint count, const char **data, const size_t *lengths, bool binary) +{ + auto dirname = filename.parent_path(); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "writeProgramOnDisk", "dir", + perfetto::DynamicString((const char *)dirname.u8string().c_str()), "program", + perfetto::DynamicString((const char *)filename.filename().u8string().c_str())); + CHECK(std::filesystem::exists(dirname), return; + , "'%s' does not exist, could not write program on disk", dirname.c_str()); + std::ofstream file(filename, binary ? std::ios::binary | std::ios::trunc : std::ios::trunc); + CHECK(file.is_open(), return;, "Could not open '%s'", filename.c_str()); + for (unsigned i = 0; i < count; i++) { + size_t length = (lengths == nullptr || lengths[i] == 0) ? strlen(data[i]) : lengths[i]; + file.write(data[i], length); + } +} + +#ifdef SPIRV_DISASSEMBLY +static bool tryDisassembleSpirv( + const void *il, size_t length, const std::string &program_name, std::string &disassembly) +{ + const uint32_t *spirv_data = (const uint32_t *)(il); + size_t spirv_words = length / sizeof(uint32_t); + + // Check for SPIR-V magic number + if (spirv_words == 0 || *spirv_data != 0x07230203) { + return false; + } + + spvtools::SpirvTools tools(SPV_ENV_OPENCL_2_2); + CHECK(tools.Disassemble(spirv_data, spirv_words, &disassembly), return false; + , "Failed to disassemble SPIR-V for program %s", program_name.c_str()); + return true; +} +#endif + +static std::string get_program_str(uint64_t *out_num = nullptr) +{ + static uint32_t program_number = 0; + uint32_t num = program_number++; + if (out_num) { + *out_num = num + 1; + } + return std::string("clkp_p") + std::to_string(num); +} + +static char *get_kernel_dir() { return getenv("CLKP_KERNEL_DIR"); } + +static bool is_program_binary_cache_disabled() +{ + const char *env = getenv("CLKP_DISABLE_PROGRAM_BINARY_CACHE"); + return env != nullptr && strcmp(env, "0") != 0 && strcmp(env, "") != 0; +} + +static cl_program clkp_clCreateProgramWithSource( + cl_context context, cl_uint count, const char **strings, const size_t *lengths, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t prog_id = 0; + std::string program_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + prog_id = g_program_ids.allocate_id(); + program_str = get_program_str(); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithSource", "call_id", cid, "context_id", ctx_id, "program_id", + prog_id, "program", perfetto::DynamicString(program_str), "count", count); + + if (auto dir = get_kernel_dir()) { + writeProgramOnDisk(std::filesystem::path(dir) / (program_str + ".cl"), count, strings, lengths, false); + } + + cl_program program = tdispatch->clCreateProgramWithSource(context, count, strings, lengths, errcode_ret); + { + std::lock_guard lock(g_lock); + g_program_ids.bind(program, prog_id); + program_to_string[program] = program_str; + program_to_context_id[program] = ctx_id; + } + + if (TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { + for (unsigned i = 0; i < count; i++) { + if (strings == nullptr || strings[i] == nullptr) { + continue; + } + size_t len = (lengths == nullptr || lengths[i] == 0) ? strlen(strings[i]) : lengths[i]; + std::string full_str(strings[i], len); + size_t chunk_size = clkp::kMaxPayloadChunkBytes; + size_t num_chunks = (full_str.size() + chunk_size - 1) / chunk_size; + if (num_chunks == 0) { + num_chunks = 1; + } + for (size_t c = 0; c < num_chunks; ++c) { + std::string chunk = full_str.substr(c * chunk_size, chunk_size); + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithSource-args", "call_id", cid, + "program_id", prog_id, "program", perfetto::DynamicString(program_str), "string_index", i, + "chunk_index", c, "chunk_count", num_chunks, "string", perfetto::DynamicString(chunk)); + } + } + } + return program; +} + +static cl_program clkp_clCreateProgramWithBinary(cl_context context, cl_uint num_devices, + const cl_device_id *device_list, const size_t *lengths, const unsigned char **binaries, cl_int *binary_status, + cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t prog_id = 0; + std::string program_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + prog_id = g_program_ids.allocate_id(); + program_str = get_program_str(); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithBinary", "call_id", cid, "context_id", ctx_id, "program_id", + prog_id, "program", perfetto::DynamicString(program_str), "num_devices", num_devices); + + if (auto dir = get_kernel_dir()) { + for (unsigned i = 0; i < num_devices; i++) { + writeProgramOnDisk(std::filesystem::path(dir) / (program_str + "-" + std::to_string(i) + ".bin"), 1, + (const char **)&binaries[i], &lengths[i], true); + } + } + cl_program program = tdispatch->clCreateProgramWithBinary( + context, num_devices, device_list, lengths, binaries, binary_status, errcode_ret); + { + std::lock_guard lock(g_lock); + g_program_ids.bind(program, prog_id); + program_to_string[program] = program_str; + program_to_context_id[program] = ctx_id; + } + + if (TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY) && binaries != nullptr && lengths != nullptr) { + for (unsigned i = 0; i < num_devices; ++i) { + if (binaries[i] == nullptr || lengths[i] == 0) { + continue; + } + std::string hex = clkp::bytes_to_hex(binaries[i], lengths[i]); + size_t chunk_size = clkp::kMaxPayloadChunkBytes; + size_t num_chunks = (hex.size() + chunk_size - 1) / chunk_size; + for (size_t c = 0; c < num_chunks; ++c) { + std::string chunk = hex.substr(c * chunk_size, chunk_size); + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithBinary-args", "call_id", cid, + "program_id", prog_id, "program", perfetto::DynamicString(program_str), "device_index", i, + "chunk_index", c, "chunk_count", num_chunks, "binary_hex", perfetto::DynamicString(chunk)); + } + } + } + return program; +} + +static cl_program create_program_with_il_common( + cl_context context, const void *il, size_t length, cl_int *errcode_ret, clCreateProgramWithILKHR_fn ext_fn) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t prog_id = 0; + std::string program_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + prog_id = g_program_ids.allocate_id(); + program_str = get_program_str(); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithIL", "call_id", cid, "context_id", ctx_id, "program_id", + prog_id, "program", perfetto::DynamicString(program_str.c_str()), "length", length); + + std::string disassembly; + bool is_spirv = false; +#ifdef SPIRV_DISASSEMBLY + is_spirv = tryDisassembleSpirv(il, length, program_str, disassembly); +#endif + + if (auto dir = get_kernel_dir()) { + std::filesystem::path filename = std::filesystem::path(dir) / program_str; + + // Write the raw IL binary + std::filesystem::path il_filename = filename.replace_extension(is_spirv ? "spv" : "il"); + const char *il_str = (const char *)il; + writeProgramOnDisk(il_filename, 1, &il_str, &length, true); + + if (is_spirv) { + // Write the disassembly + std::filesystem::path asm_filename = filename.replace_extension("spvasm"); + const char *disassembly_str = disassembly.c_str(); + size_t disassembly_length = disassembly.length(); + writeProgramOnDisk(asm_filename, 1, &disassembly_str, &disassembly_length, false); + } + } + + cl_program program = ext_fn ? ext_fn(context, il, length, errcode_ret) + : tdispatch->clCreateProgramWithIL(context, il, length, errcode_ret); + { + std::lock_guard lock(g_lock); + g_program_ids.bind(program, prog_id); + program_to_string[program] = program_str; + program_to_context_id[program] = ctx_id; + } + + if (TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { + std::string il_hex = clkp::bytes_to_hex(il, length); + size_t chunk_size = clkp::kMaxPayloadChunkBytes; + size_t num_chunks = (il_hex.size() + chunk_size - 1) / chunk_size; + if (num_chunks == 0) { + num_chunks = 1; + } + for (size_t c = 0; c < num_chunks; ++c) { + std::string chunk = il_hex.empty() ? "" : il_hex.substr(c * chunk_size, chunk_size); + if (c == 0 && is_spirv) { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithIL-args", "call_id", cid, "program_id", + prog_id, "program", perfetto::DynamicString(program_str), "disassembly", + perfetto::DynamicString(disassembly), "chunk_index", c, "chunk_count", num_chunks, "il_hex", + perfetto::DynamicString(chunk)); + } else { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithIL-args", "call_id", cid, "program_id", + prog_id, "program", perfetto::DynamicString(program_str), "chunk_index", c, "chunk_count", + num_chunks, "il_hex", perfetto::DynamicString(chunk)); + } + } + } + + return program; +} + +static cl_program clkp_clCreateProgramWithIL(cl_context context, const void *il, size_t length, cl_int *errcode_ret) +{ + return create_program_with_il_common(context, il, length, errcode_ret, nullptr); +} + +static cl_int clkp_clGetProgramInfo(cl_program program, cl_program_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) +{ + if (param_name == CL_PROGRAM_BINARY_SIZES && is_program_binary_cache_disabled()) { + cl_int err + = tdispatch->clGetProgramInfo(program, param_name, param_value_size, param_value, param_value_size_ret); + if (err == CL_SUCCESS && param_value != nullptr) { + memset(param_value, 0, param_value_size); + } + return err; + } + return tdispatch->clGetProgramInfo(program, param_name, param_value_size, param_value, param_value_size_ret); +} + +static cl_int clkp_clBuildProgram(cl_program program, cl_uint num_devices, const cl_device_id *device_list, + const char *options, void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), void *user_data) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + std::string prog_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.get_or_create(program); + prog_str = program_to_string.count(program) ? program_to_string[program] : "clkp_p?"; + } + std::string opts = options ? options : ""; + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clBuildProgram", "call_id", cid, "program_id", prog_id, "program", + perfetto::DynamicString(prog_str), "options", perfetto::DynamicString(opts)); + return tdispatch->clBuildProgram(program, num_devices, device_list, options, pfn_notify, user_data); +} + +static cl_int clkp_clCompileProgram(cl_program program, cl_uint num_devices, const cl_device_id *device_list, + const char *options, cl_uint num_input_headers, const cl_program *input_headers, const char **header_include_names, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), void *user_data) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + std::string prog_str; + std::vector header_ids; + std::vector header_names; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.get_or_create(program); + prog_str = program_to_string.count(program) ? program_to_string[program] : "clkp_p?"; + for (cl_uint i = 0; i < num_input_headers; ++i) { + if (input_headers) { + header_ids.push_back(g_program_ids.get_or_create(input_headers[i])); + } + if (header_include_names && header_include_names[i]) { + header_names.push_back(header_include_names[i]); + } + } + } + std::string opts = options ? options : ""; + std::string header_ids_str = clkp::uint64_list_to_string(header_ids); + std::string header_names_str = clkp::string_list_to_joined(header_names, ';'); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCompileProgram", "call_id", cid, "program_id", prog_id, "program", + perfetto::DynamicString(prog_str), "options", perfetto::DynamicString(opts), "num_input_headers", + num_input_headers, "header_program_ids", perfetto::DynamicString(header_ids_str), "header_include_names", + perfetto::DynamicString(header_names_str)); + return tdispatch->clCompileProgram(program, num_devices, device_list, options, num_input_headers, input_headers, + header_include_names, pfn_notify, user_data); +} + +static cl_program clkp_clLinkProgram(cl_context context, cl_uint num_devices, const cl_device_id *device_list, + const char *options, cl_uint num_input_programs, const cl_program *input_programs, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), void *user_data, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t prog_id = 0; + std::string program_str; + std::vector input_ids; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + prog_id = g_program_ids.allocate_id(); + program_str = get_program_str(); + for (cl_uint i = 0; i < num_input_programs; ++i) { + if (input_programs) { + input_ids.push_back(g_program_ids.get_or_create(input_programs[i])); + } + } + } + std::string opts = options ? options : ""; + std::string input_ids_str = clkp::uint64_list_to_string(input_ids); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clLinkProgram", "call_id", cid, "context_id", ctx_id, "program_id", prog_id, + "program", perfetto::DynamicString(program_str), "options", perfetto::DynamicString(opts), "num_input_programs", + num_input_programs, "input_program_ids", perfetto::DynamicString(input_ids_str)); + cl_program program = tdispatch->clLinkProgram(context, num_devices, device_list, options, num_input_programs, + input_programs, pfn_notify, user_data, errcode_ret); + { + std::lock_guard lock(g_lock); + g_program_ids.bind(program, prog_id); + program_to_string[program] = program_str; + program_to_context_id[program] = ctx_id; + } + return program; +} + +static cl_int clkp_clSetProgramSpecializationConstant( + cl_program program, cl_uint spec_id, size_t spec_size, const void *spec_value) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + std::string prog_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.get_or_create(program); + prog_str = program_to_string.count(program) ? program_to_string[program] : "clkp_p?"; + } + std::string val_hex = clkp::bytes_to_hex(spec_value, spec_size); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clSetProgramSpecializationConstant", "call_id", cid, "program_id", prog_id, + "program", perfetto::DynamicString(prog_str), "spec_id", spec_id, "spec_size", spec_size, "value_hex", + perfetto::DynamicString(val_hex)); + return tdispatch->clSetProgramSpecializationConstant(program, spec_id, spec_size, spec_value); +} + +static cl_int clkp_clRetainProgram(cl_program program) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.get_or_create(program); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainProgram", "call_id", cid, "program_id", prog_id); + return tdispatch->clRetainProgram(program); +} + +static cl_int clkp_clReleaseProgram(cl_program program) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.find(program); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseProgram", "call_id", cid, "program_id", prog_id); + return tdispatch->clReleaseProgram(program); +} + +static cl_kernel clkp_clCreateKernel(cl_program program, const char *kernel_name, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t kern_id = 0; + uint64_t prog_id = 0; + std::string prog_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + kern_id = g_kernel_ids.allocate_id(); + prog_id = g_program_ids.get_or_create(program); + prog_str = program_to_string.count(program) ? program_to_string[program] : "clkp_p?"; + } + std::string kname = kernel_name ? kernel_name : ""; + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateKernel", "call_id", cid, "program_id", prog_id, "program", + perfetto::DynamicString(prog_str), "kernel_id", kern_id, "kernel_name", perfetto::DynamicString(kname)); + cl_kernel kernel = tdispatch->clCreateKernel(program, kernel_name, errcode_ret); + { + std::lock_guard lock(g_lock); + g_kernel_ids.bind(kernel, kern_id); + if (kernel != nullptr) { + kernel_to_program[kernel] = program; + kernel_to_kernel_name[kernel] = kname; + } + } + return kernel; +} + +static cl_int clkp_clCreateKernelsInProgram( + cl_program program, cl_uint num_kernels, cl_kernel *kernels, cl_uint *num_kernels_ret) +{ + uint64_t cid = 0; + uint64_t prog_id = 0; + std::string prog_str; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + prog_id = g_program_ids.get_or_create(program); + prog_str = program_to_string.count(program) ? program_to_string[program] : "clkp_p?"; + } + cl_uint actual_kernels = 0; + cl_int err = tdispatch->clCreateKernelsInProgram(program, num_kernels, kernels, &actual_kernels); + if (num_kernels_ret != nullptr) { + *num_kernels_ret = actual_kernels; + } + cl_uint created_count = (err == CL_SUCCESS && kernels != nullptr) ? std::min(num_kernels, actual_kernels) : 0; + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateKernelsInProgram", "call_id", cid, "program_id", prog_id, "program", + perfetto::DynamicString(prog_str), "num_kernels_created", created_count); + + if (err == CL_SUCCESS && kernels != nullptr) { + for (cl_uint i = 0; i < created_count; ++i) { + cl_kernel k = kernels[i]; + char name_buf[256] = { 0 }; + size_t name_len = 0; + if (tdispatch->clGetKernelInfo) { + tdispatch->clGetKernelInfo(k, CL_KERNEL_FUNCTION_NAME, sizeof(name_buf) - 1, name_buf, &name_len); + } + std::string kname(name_buf); + uint64_t kern_id = 0; + { + std::lock_guard lock(g_lock); + kern_id = g_kernel_ids.create_fresh(k); + kernel_to_program[k] = program; + kernel_to_kernel_name[k] = kname; + } + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateKernelsInProgram-kernel", "call_id", cid, "program_id", + prog_id, "program", perfetto::DynamicString(prog_str), "kernel_id", kern_id, "kernel_name", + perfetto::DynamicString(kname)); + } + } + return err; +} + +static cl_kernel clkp_clCloneKernel(cl_kernel source_kernel, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t src_id = 0; + uint64_t dst_id = 0; + cl_program prog = nullptr; + uint64_t prog_id = 0; + std::string prog_str; + std::string kname; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + src_id = g_kernel_ids.get_or_create(source_kernel); + dst_id = g_kernel_ids.allocate_id(); + prog = kernel_to_program.count(source_kernel) ? kernel_to_program[source_kernel] : nullptr; + prog_id = g_program_ids.find(prog); + prog_str = program_to_string.count(prog) ? program_to_string[prog] : "clkp_p?"; + kname = kernel_to_kernel_name.count(source_kernel) ? kernel_to_kernel_name[source_kernel] : "?"; + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCloneKernel", "call_id", cid, "src_kernel_id", src_id, "kernel_id", dst_id, + "program_id", prog_id, "program", perfetto::DynamicString(prog_str), "kernel_name", + perfetto::DynamicString(kname)); + cl_kernel cloned = tdispatch->clCloneKernel(source_kernel, errcode_ret); + if (cloned != nullptr) { + std::lock_guard lock(g_lock); + g_kernel_ids.bind(cloned, dst_id); + kernel_to_program[cloned] = prog; + kernel_to_kernel_name[cloned] = kname; + } + return cloned; +} + +static void classify_kernel_arg(size_t arg_size, const void *arg_value, std::string &out_kind, uint64_t &out_mem_id, + uint64_t &out_sampler_id, std::string &out_val_hex) +{ + out_mem_id = 0; + out_sampler_id = 0; + out_val_hex.clear(); + + if (arg_value == nullptr) { + out_kind = "LOCAL"; + return; + } + if (arg_size == sizeof(cl_mem)) { + cl_mem candidate_mem = *reinterpret_cast(arg_value); + if (candidate_mem != nullptr && g_mem_ids.contains(candidate_mem)) { + out_kind = "MEM"; + out_mem_id = g_mem_ids.find(candidate_mem); + return; + } + } + if (arg_size == sizeof(cl_sampler)) { + cl_sampler candidate_sampler = *reinterpret_cast(arg_value); + if (candidate_sampler != nullptr && g_sampler_ids.contains(candidate_sampler)) { + out_kind = "SAMPLER"; + out_sampler_id = g_sampler_ids.find(candidate_sampler); + return; + } + } + out_kind = "VALUE"; + out_val_hex = clkp::bytes_to_hex(arg_value, arg_size); +} + +static cl_int clkp_clSetKernelArg(cl_kernel kernel, cl_uint arg_index, size_t arg_size, const void *arg_value) +{ + uint64_t cid = 0; + uint64_t kern_id = 0; + std::string arg_kind; + uint64_t mem_id = 0; + uint64_t sampler_id = 0; + std::string val_hex; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + kern_id = g_kernel_ids.get_or_create(kernel); + classify_kernel_arg(arg_size, arg_value, arg_kind, mem_id, sampler_id, val_hex); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clSetKernelArg", "call_id", cid, "kernel_id", kern_id, "arg_index", arg_index, + "arg_size", arg_size, "arg_kind", perfetto::DynamicString(arg_kind), "mem_id", mem_id, "sampler_id", sampler_id, + "value_hex", perfetto::DynamicString(val_hex)); + return tdispatch->clSetKernelArg(kernel, arg_index, arg_size, arg_value); +} + +static cl_int clkp_clRetainKernel(cl_kernel kernel) +{ + uint64_t cid = 0; + uint64_t kern_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + kern_id = g_kernel_ids.get_or_create(kernel); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainKernel", "call_id", cid, "kernel_id", kern_id); + return tdispatch->clRetainKernel(kernel); +} + +static cl_int clkp_clReleaseKernel(cl_kernel kernel) +{ + uint64_t cid = 0; + uint64_t kern_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + kern_id = g_kernel_ids.find(kernel); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseKernel", "call_id", cid, "kernel_id", kern_id); + return tdispatch->clReleaseKernel(kernel); +} + +/*****************************************************************************/ +/* MEMORY & SAMPLER LIFECYCLE ************************************************/ +/*****************************************************************************/ + +static cl_mem clkp_clCreateBuffer( + cl_context context, cl_mem_flags flags, size_t size, void *host_ptr, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateBuffer", "call_id", cid, "context_id", ctx_id, "mem_id", mem_id, + "flags", static_cast(flags), "size", size); + cl_mem mem = tdispatch->clCreateBuffer(context, flags, size, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(mem, mem_id); + } + return mem; +} + +static cl_mem clkp_clCreateBufferWithProperties(cl_context context, const cl_mem_properties *properties, + cl_mem_flags flags, size_t size, void *host_ptr, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + } + std::string props_str = properties_to_string(properties); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateBufferWithProperties", "call_id", cid, "context_id", ctx_id, "mem_id", + mem_id, "flags", static_cast(flags), "size", size, "properties", perfetto::DynamicString(props_str)); + cl_mem mem = tdispatch->clCreateBufferWithProperties(context, properties, flags, size, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(mem, mem_id); + } + return mem; +} + +static cl_mem clkp_clCreateSubBuffer(cl_mem buffer, cl_mem_flags flags, cl_buffer_create_type buffer_create_type, + const void *buffer_create_info, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t parent_id = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + parent_id = g_mem_ids.get_or_create(buffer); + mem_id = g_mem_ids.allocate_id(); + } + size_t origin = 0; + size_t size = 0; + if (buffer_create_type == CL_BUFFER_CREATE_TYPE_REGION && buffer_create_info != nullptr) { + const auto *reg = reinterpret_cast(buffer_create_info); + origin = reg->origin; + size = reg->size; + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateSubBuffer", "call_id", cid, "parent_mem_id", parent_id, "mem_id", + mem_id, "flags", static_cast(flags), "buffer_create_type", static_cast(buffer_create_type), + "origin", origin, "size", size); + cl_mem sub = tdispatch->clCreateSubBuffer(buffer, flags, buffer_create_type, buffer_create_info, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(sub, mem_id); + } + return sub; +} + +static cl_mem clkp_clCreateImage(cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + const cl_image_desc *image_desc, void *host_ptr, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + uint64_t parent_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + parent_id = (image_desc && image_desc->mem_object) ? g_mem_ids.get_or_create(image_desc->mem_object) : 0; + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateImage", "call_id", cid, "context_id", ctx_id, "mem_id", mem_id, + "flags", static_cast(flags), "channel_order", image_format ? image_format->image_channel_order : 0u, + "channel_data_type", image_format ? image_format->image_channel_data_type : 0u, "image_type", + image_desc ? image_desc->image_type : 0u, "width", image_desc ? image_desc->image_width : 0u, "height", + image_desc ? image_desc->image_height : 0u, "depth", image_desc ? image_desc->image_depth : 0u, "array_size", + image_desc ? image_desc->image_array_size : 0u, "row_pitch", image_desc ? image_desc->image_row_pitch : 0u, + "slice_pitch", image_desc ? image_desc->image_slice_pitch : 0u, "num_mip_levels", + image_desc ? image_desc->num_mip_levels : 0u, "num_samples", image_desc ? image_desc->num_samples : 0u, + "parent_mem_id", parent_id); + cl_mem img = tdispatch->clCreateImage(context, flags, image_format, image_desc, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(img, mem_id); + } + return img; +} + +static cl_mem clkp_clCreateImageWithProperties(cl_context context, const cl_mem_properties *properties, + cl_mem_flags flags, const cl_image_format *image_format, const cl_image_desc *image_desc, void *host_ptr, + cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + uint64_t parent_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + parent_id = (image_desc && image_desc->mem_object) ? g_mem_ids.get_or_create(image_desc->mem_object) : 0; + } + std::string props_str = properties_to_string(properties); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateImageWithProperties", "call_id", cid, "context_id", ctx_id, "mem_id", + mem_id, "flags", static_cast(flags), "channel_order", + image_format ? image_format->image_channel_order : 0u, "channel_data_type", + image_format ? image_format->image_channel_data_type : 0u, "image_type", + image_desc ? image_desc->image_type : 0u, "width", image_desc ? image_desc->image_width : 0u, "height", + image_desc ? image_desc->image_height : 0u, "depth", image_desc ? image_desc->image_depth : 0u, "array_size", + image_desc ? image_desc->image_array_size : 0u, "row_pitch", image_desc ? image_desc->image_row_pitch : 0u, + "slice_pitch", image_desc ? image_desc->image_slice_pitch : 0u, "num_mip_levels", + image_desc ? image_desc->num_mip_levels : 0u, "num_samples", image_desc ? image_desc->num_samples : 0u, + "parent_mem_id", parent_id, "properties", perfetto::DynamicString(props_str)); + cl_mem img = tdispatch->clCreateImageWithProperties( + context, properties, flags, image_format, image_desc, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(img, mem_id); + } + return img; +} + +static cl_mem clkp_clCreateImage2D(cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + size_t image_width, size_t image_height, size_t image_row_pitch, void *host_ptr, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateImage2D", "call_id", cid, "context_id", ctx_id, "mem_id", mem_id, + "flags", static_cast(flags), "channel_order", image_format ? image_format->image_channel_order : 0u, + "channel_data_type", image_format ? image_format->image_channel_data_type : 0u, "image_type", + static_cast(CL_MEM_OBJECT_IMAGE2D), "width", image_width, "height", image_height, "depth", 1u, + "row_pitch", image_row_pitch); + cl_mem img = tdispatch->clCreateImage2D( + context, flags, image_format, image_width, image_height, image_row_pitch, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(img, mem_id); + } + return img; +} + +static cl_mem clkp_clCreateImage3D(cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + size_t image_width, size_t image_height, size_t image_depth, size_t image_row_pitch, size_t image_slice_pitch, + void *host_ptr, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + mem_id = g_mem_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateImage3D", "call_id", cid, "context_id", ctx_id, "mem_id", mem_id, + "flags", static_cast(flags), "channel_order", image_format ? image_format->image_channel_order : 0u, + "channel_data_type", image_format ? image_format->image_channel_data_type : 0u, "image_type", + static_cast(CL_MEM_OBJECT_IMAGE3D), "width", image_width, "height", image_height, "depth", + image_depth, "row_pitch", image_row_pitch, "slice_pitch", image_slice_pitch); + cl_mem img = tdispatch->clCreateImage3D(context, flags, image_format, image_width, image_height, image_depth, + image_row_pitch, image_slice_pitch, host_ptr, errcode_ret); + { + std::lock_guard lock(g_lock); + g_mem_ids.bind(img, mem_id); + } + return img; +} + +static cl_int clkp_clRetainMemObject(cl_mem memobj) +{ + uint64_t cid = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + mem_id = g_mem_ids.get_or_create(memobj); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainMemObject", "call_id", cid, "mem_id", mem_id); + return tdispatch->clRetainMemObject(memobj); +} + +static cl_int clkp_clReleaseMemObject(cl_mem memobj) +{ + uint64_t cid = 0; + uint64_t mem_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + mem_id = g_mem_ids.find(memobj); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseMemObject", "call_id", cid, "mem_id", mem_id); + return tdispatch->clReleaseMemObject(memobj); +} + +static cl_sampler clkp_clCreateSampler(cl_context context, cl_bool normalized_coords, + cl_addressing_mode addressing_mode, cl_filter_mode filter_mode, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t samp_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + samp_id = g_sampler_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateSampler", "call_id", cid, "context_id", ctx_id, "sampler_id", samp_id, + "normalized_coords", normalized_coords, "addressing_mode", addressing_mode, "filter_mode", filter_mode); + cl_sampler samp = tdispatch->clCreateSampler(context, normalized_coords, addressing_mode, filter_mode, errcode_ret); + { + std::lock_guard lock(g_lock); + g_sampler_ids.bind(samp, samp_id); + } + return samp; +} + +static cl_sampler clkp_clCreateSamplerWithProperties( + cl_context context, const cl_sampler_properties *sampler_properties, cl_int *errcode_ret) +{ + uint64_t cid = 0; + uint64_t ctx_id = 0; + uint64_t samp_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + ctx_id = g_context_ids.get_or_create(context); + samp_id = g_sampler_ids.allocate_id(); + } + cl_bool normalized_coords = CL_TRUE; + cl_addressing_mode addressing_mode = CL_ADDRESS_CLAMP; + cl_filter_mode filter_mode = CL_FILTER_NEAREST; + if (sampler_properties != nullptr) { + for (size_t i = 0; sampler_properties[i] != 0; i += 2) { + if (sampler_properties[i] == CL_SAMPLER_NORMALIZED_COORDS) { + normalized_coords = static_cast(sampler_properties[i + 1]); + } else if (sampler_properties[i] == CL_SAMPLER_ADDRESSING_MODE) { + addressing_mode = static_cast(sampler_properties[i + 1]); + } else if (sampler_properties[i] == CL_SAMPLER_FILTER_MODE) { + filter_mode = static_cast(sampler_properties[i + 1]); + } + } + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateSamplerWithProperties", "call_id", cid, "context_id", ctx_id, + "sampler_id", samp_id, "normalized_coords", normalized_coords, "addressing_mode", addressing_mode, + "filter_mode", filter_mode); + cl_sampler samp = tdispatch->clCreateSamplerWithProperties(context, sampler_properties, errcode_ret); + { + std::lock_guard lock(g_lock); + g_sampler_ids.bind(samp, samp_id); + } + return samp; +} + +static cl_int clkp_clRetainSampler(cl_sampler sampler) +{ + uint64_t cid = 0; + uint64_t samp_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + samp_id = g_sampler_ids.get_or_create(sampler); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainSampler", "call_id", cid, "sampler_id", samp_id); + return tdispatch->clRetainSampler(sampler); +} + +static cl_int clkp_clReleaseSampler(cl_sampler sampler) +{ + uint64_t cid = 0; + uint64_t samp_id = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + samp_id = g_sampler_ids.find(sampler); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseSampler", "call_id", cid, "sampler_id", samp_id); + return tdispatch->clReleaseSampler(sampler); +} + +/*****************************************************************************/ +/* ENQUEUE NDRANGE KERNEL & PROFILING CALLBACKS ******************************/ +/*****************************************************************************/ + +struct ThreadInfo; + +struct callback_data { + cl_command_queue queue; + ThreadInfo *thread_info = nullptr; + cl_event event; + size_t gidX, gidY, gidZ; + int64_t time_offset; + std::string kernel_name; + std::string program_string; + uint64_t call_id = 0; + uint64_t dispatch_id = 0; + uint64_t queue_id = 0; + uint64_t kernel_id = 0; + uint64_t cmdbuf_id = 0; + bool is_cmdbuf = false; +}; + +struct ThreadInfo { + std::mutex lock; + std::condition_variable cv; + std::queue callbacks; + bool stop; + int pending_callbacks; +}; + +static std::map queue_to_thread_info; +static std::map queue_to_thread; +static std::map queue_to_time_offset; +static std::map queue_to_refcount; + +static void callback(cl_event event, cl_int event_command_exec_status, void *user_data) +{ + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clkp-callback"); + + struct callback_data *data = (struct callback_data *)user_data; + assert(data != nullptr); + assert(event_command_exec_status == CL_COMPLETE); + + ThreadInfo *thread_info = data->thread_info; + if (thread_info == nullptr) { + tdispatch->clReleaseEvent(data->event); + delete data; + return; + } + + { + std::lock_guard lock_ti(thread_info->lock); + thread_info->callbacks.push(data); + thread_info->cv.notify_all(); + } +} + +static callback_data *get_callback(ThreadInfo *thread_info) +{ + std::unique_lock lock(thread_info->lock); + while (thread_info->callbacks.empty()) { + if (thread_info->stop && thread_info->pending_callbacks == 0) { + return nullptr; + } + TRACE_EVENT_BEGIN(CLKP_PERFETTO_CATEGORY, "clkp_wait"); + thread_info->cv.wait(lock); + TRACE_EVENT_END(CLKP_PERFETTO_CATEGORY); + } + callback_data *data = thread_info->callbacks.front(); + thread_info->callbacks.pop(); + return data; +} + +static void trace_callback(callback_data *data) +{ + cl_command_queue queue = data->queue; + cl_event event = data->event; + size_t gidX = data->gidX, gidY = data->gidY, gidZ = data->gidZ; + int64_t time_offset = data->time_offset; + + cl_ulong start, end; + cl_int err; + err = tdispatch->clGetEventProfilingInfo(event, CL_PROFILING_COMMAND_START, sizeof(start), &start, nullptr); + CHECK_CL(err, tdispatch->clReleaseEvent(event); + return, "clGetEventProfilingInfo(CL_PROFILING_COMMAND_START) failed (%i)", err); + err = tdispatch->clGetEventProfilingInfo(event, CL_PROFILING_COMMAND_END, sizeof(end), &end, nullptr); + CHECK_CL(err, tdispatch->clReleaseEvent(event); + return, "clGetEventProfilingInfo(CL_PROFILING_COMMAND_END) failed (%i)", err); + if (end < start) { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, perfetto::StaticString("INVALID_TIMESTAMPS"), + perfetto::Track((uintptr_t)queue), "start", start, "end", end); + tdispatch->clReleaseEvent(event); + return; + } + + if (data->is_cmdbuf) { + std::string name = "clkp_cmdbuf_" + std::to_string(data->cmdbuf_id); + TRACE_EVENT_BEGIN(CLKP_PERFETTO_CATEGORY, perfetto::DynamicString(name), perfetto::Track((uintptr_t)queue), + (uint64_t)(start + time_offset), "cmdbuf_id", data->cmdbuf_id, "dispatch_id", data->dispatch_id, "call_id", + data->call_id, "queue_id", data->queue_id); + TRACE_EVENT_END(CLKP_PERFETTO_CATEGORY, perfetto::Track((uintptr_t)queue), (uint64_t)(end + time_offset)); + } else { + std::string name = data->program_string + "-" + data->kernel_name + "-" + std::to_string(gidX) + "." + + std::to_string(gidY) + "." + std::to_string(gidZ); + + TRACE_EVENT_BEGIN(CLKP_PERFETTO_CATEGORY, perfetto::DynamicString(name), perfetto::Track((uintptr_t)queue), + (uint64_t)(start + time_offset), "program", perfetto::DynamicString(data->program_string), "kernel_name", + perfetto::DynamicString(data->kernel_name), "gidX", gidX, "gidY", gidY, "gidZ", gidZ, "dispatch_id", + data->dispatch_id, "call_id", data->call_id, "queue_id", data->queue_id, "kernel_id", data->kernel_id); + TRACE_EVENT_END(CLKP_PERFETTO_CATEGORY, perfetto::Track((uintptr_t)queue), (uint64_t)(end + time_offset)); + } + + tdispatch->clReleaseEvent(event); +} + +static void queue_thread_function(ThreadInfo *thread_info) +{ +#ifdef __APPLE__ + pthread_setname_np("clkp"); +#else + pthread_setname_np(pthread_self(), "clkp"); +#endif + while (true) { + callback_data *data = get_callback(thread_info); + if (data == nullptr) { + return; + } + trace_callback(data); + { + std::lock_guard lock(thread_info->lock); + thread_info->pending_callbacks--; + thread_info->cv.notify_all(); + } + delete data; + } +} + +static cl_int attach_profiling_callback(cl_command_queue command_queue, cl_event *event_to_use, bool event_is_null, + cl_event local_event, callback_data *data) +{ + ThreadInfo *thread_info = nullptr; + { + std::lock_guard lock(g_lock); + auto it = queue_to_thread_info.find(command_queue); + if (it != queue_to_thread_info.end()) { + thread_info = it->second; + std::lock_guard lock_ti(thread_info->lock); + if (thread_info->stop) { + thread_info = nullptr; + } else { + thread_info->pending_callbacks++; + data->thread_info = thread_info; + } + } + } + + if (thread_info == nullptr) { + delete data; + if (event_is_null) { + tdispatch->clReleaseEvent(local_event); + } + return CL_SUCCESS; + } + + // Retain the event for the callback + tdispatch->clRetainEvent(*event_to_use); + + cl_int err_cb = tdispatch->clSetEventCallback(*event_to_use, CL_COMPLETE, callback, data); + if (err_cb != CL_SUCCESS) { + PRINT("clSetEventCallback failed (%i)", err_cb); + tdispatch->clReleaseEvent(*event_to_use); // Undo retain + delete data; + { + std::lock_guard lock_ti(thread_info->lock); + thread_info->pending_callbacks--; + thread_info->cv.notify_all(); + } + if (event_is_null) { + tdispatch->clReleaseEvent(local_event); + } + return err_cb; + } + + if (event_is_null) { + tdispatch->clReleaseEvent(local_event); + } + + return CL_SUCCESS; +} + +static cl_int clkp_clEnqueueNDRangeKernel(cl_command_queue command_queue, cl_kernel kernel, cl_uint work_dim, + const size_t *global_work_offset, const size_t *global_work_size, const size_t *local_work_size, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + size_t gidX = (work_dim > 0 && global_work_size) ? global_work_size[0] : 1; + size_t gidY = (work_dim > 1 && global_work_size) ? global_work_size[1] : 1; + size_t gidZ = (work_dim > 2 && global_work_size) ? global_work_size[2] : 1; + + std::string gwo_str = size_array_to_string(global_work_offset, work_dim); + std::string gws_str = size_array_to_string(global_work_size, work_dim); + std::string lws_str = size_array_to_string(local_work_size, work_dim); + + bool event_is_null = event == nullptr; + cl_event local_event = nullptr; + cl_event *event_to_use = event_is_null ? &local_event : event; + + uint64_t cid = 0; + uint64_t did = 0; + uint64_t q_id = 0; + uint64_t k_id = 0; + std::string kernel_name = "?"; + uint64_t prog_id = 0; + std::string program_string = "clkp_p?"; + std::string wait_str; + uint64_t ret_event_id = 0; + int64_t time_offset = 0; + { + std::lock_guard lock(g_lock); + cid = next_call_id(); + did = next_dispatch_id(); + q_id = g_queue_ids.get_or_create(command_queue); + k_id = g_kernel_ids.get_or_create(kernel); + if (kernel_to_kernel_name.count(kernel)) { + kernel_name = kernel_to_kernel_name[kernel]; + } + cl_program prog = kernel_to_program.count(kernel) ? kernel_to_program[kernel] : nullptr; + prog_id = g_program_ids.find(prog); + if (prog && program_to_string.count(prog)) { + program_string = program_to_string[prog]; + } + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + ret_event_id = !event_is_null ? g_event_ids.allocate_id() : 0; + if (queue_to_time_offset.count(command_queue)) { + time_offset = queue_to_time_offset[command_queue]; + } + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueNDRangeKernel", "call_id", cid, "dispatch_id", did, "queue_id", q_id, + "kernel_id", k_id, "program_id", prog_id, "program", perfetto::DynamicString(program_string), "kernel_name", + perfetto::DynamicString(kernel_name), "work_dim", work_dim, "gidX", gidX, "gidY", gidY, "gidZ", gidZ, "gwo", + perfetto::DynamicString(gwo_str), "gws", perfetto::DynamicString(gws_str), "lws", + perfetto::DynamicString(lws_str), "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", + ret_event_id); + + cl_int err = tdispatch->clEnqueueNDRangeKernel(command_queue, kernel, work_dim, global_work_offset, + global_work_size, local_work_size, num_events_in_wait_list, event_wait_list, event_to_use); + + if (err != CL_SUCCESS) { + return err; + } + if (!event_is_null && *event != nullptr) { + std::lock_guard lock(g_lock); + g_event_ids.bind(*event, ret_event_id); + } + + struct callback_data *data = new (std::nothrow) callback_data(); + if (data == nullptr) { + if (event_is_null) { + tdispatch->clReleaseEvent(local_event); + } + return CL_OUT_OF_HOST_MEMORY; + } + + data->queue = command_queue; + data->event = *event_to_use; + data->gidX = gidX; + data->gidY = gidY; + data->gidZ = gidZ; + data->time_offset = time_offset; + data->kernel_name = kernel_name; + data->program_string = program_string; + data->call_id = cid; + data->dispatch_id = did; + data->queue_id = q_id; + data->kernel_id = k_id; + data->is_cmdbuf = false; + + return attach_profiling_callback(command_queue, event_to_use, event_is_null, local_event, data); +} + +static cl_int clkp_clEnqueueTask(cl_command_queue command_queue, cl_kernel kernel, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + size_t gws[1] = { 1 }; + size_t lws[1] = { 1 }; + return clkp_clEnqueueNDRangeKernel( + command_queue, kernel, 1, nullptr, gws, lws, num_events_in_wait_list, event_wait_list, event); +} + +/*****************************************************************************/ +/* DATA TRANSFER, MAP/UNMAP & SYNCHRONIZATION ENQUEUES ***********************/ +/*****************************************************************************/ + +static cl_int clkp_clEnqueueReadBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, + size_t offset, size_t size, void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err = tdispatch->clEnqueueReadBuffer( + command_queue, buffer, blocking_read, offset, size, ptr, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueReadBuffer", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_read, "offset", offset, "size", size, "wait_events", perfetto::DynamicString(wait_str), + "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueWriteBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, + size_t offset, size_t size, const void *ptr, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err = tdispatch->clEnqueueWriteBuffer( + command_queue, buffer, blocking_write, offset, size, ptr, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueWriteBuffer", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_write, "offset", offset, "size", size, "wait_events", perfetto::DynamicString(wait_str), + "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueReadBufferRect(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, + const size_t *buffer_origin, const size_t *host_origin, const size_t *region, size_t buffer_row_pitch, + size_t buffer_slice_pitch, size_t host_row_pitch, size_t host_slice_pitch, void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string bo_str = size_array_to_string(buffer_origin, 3); + std::string ho_str = size_array_to_string(host_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueReadBufferRect(command_queue, buffer, blocking_read, buffer_origin, host_origin, + region, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch, ptr, num_events_in_wait_list, + event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueReadBufferRect", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_read, "buffer_origin", perfetto::DynamicString(bo_str), "host_origin", + perfetto::DynamicString(ho_str), "region", perfetto::DynamicString(reg_str), "buffer_row_pitch", + buffer_row_pitch, "buffer_slice_pitch", buffer_slice_pitch, "host_row_pitch", host_row_pitch, + "host_slice_pitch", host_slice_pitch, "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueWriteBufferRect(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, + const size_t *buffer_origin, const size_t *host_origin, const size_t *region, size_t buffer_row_pitch, + size_t buffer_slice_pitch, size_t host_row_pitch, size_t host_slice_pitch, const void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string bo_str = size_array_to_string(buffer_origin, 3); + std::string ho_str = size_array_to_string(host_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueWriteBufferRect(command_queue, buffer, blocking_write, buffer_origin, host_origin, + region, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch, ptr, num_events_in_wait_list, + event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueWriteBufferRect", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_write, "buffer_origin", perfetto::DynamicString(bo_str), "host_origin", + perfetto::DynamicString(ho_str), "region", perfetto::DynamicString(reg_str), "buffer_row_pitch", + buffer_row_pitch, "buffer_slice_pitch", buffer_slice_pitch, "host_row_pitch", host_row_pitch, + "host_slice_pitch", host_slice_pitch, "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueCopyBuffer(cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, + size_t src_offset, size_t dst_offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err = tdispatch->clEnqueueCopyBuffer(command_queue, src_buffer, dst_buffer, src_offset, dst_offset, size, + num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCopyBuffer", "call_id", cid, "queue_id", q_id, "src_mem_id", src_id, + "dst_mem_id", dst_id, "src_offset", src_offset, "dst_offset", dst_offset, "size", size, "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueCopyBufferRect(cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, size_t src_row_pitch, + size_t src_slice_pitch, size_t dst_row_pitch, size_t dst_slice_pitch, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string so_str = size_array_to_string(src_origin, 3); + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueCopyBufferRect(command_queue, src_buffer, dst_buffer, src_origin, dst_origin, + region, src_row_pitch, src_slice_pitch, dst_row_pitch, dst_slice_pitch, num_events_in_wait_list, + event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCopyBufferRect", "call_id", cid, "queue_id", q_id, "src_mem_id", + src_id, "dst_mem_id", dst_id, "src_origin", perfetto::DynamicString(so_str), "dst_origin", + perfetto::DynamicString(do_str), "region", perfetto::DynamicString(reg_str), "src_row_pitch", src_row_pitch, + "src_slice_pitch", src_slice_pitch, "dst_row_pitch", dst_row_pitch, "dst_slice_pitch", dst_slice_pitch, + "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueFillBuffer(cl_command_queue command_queue, cl_mem buffer, const void *pattern, + size_t pattern_size, size_t offset, size_t size, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string pat_hex = clkp::bytes_to_hex(pattern, pattern_size); + cl_int err = tdispatch->clEnqueueFillBuffer( + command_queue, buffer, pattern, pattern_size, offset, size, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueFillBuffer", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "pattern_hex", perfetto::DynamicString(pat_hex), "pattern_size", pattern_size, "offset", offset, "size", size, + "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueReadImage(cl_command_queue command_queue, cl_mem image, cl_bool blocking_read, + const size_t *origin, const size_t *region, size_t row_pitch, size_t slice_pitch, void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string orig_str = size_array_to_string(origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueReadImage(command_queue, image, blocking_read, origin, region, row_pitch, + slice_pitch, ptr, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueReadImage", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_read, "origin", perfetto::DynamicString(orig_str), "region", + perfetto::DynamicString(reg_str), "row_pitch", row_pitch, "slice_pitch", slice_pitch, "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueWriteImage(cl_command_queue command_queue, cl_mem image, cl_bool blocking_write, + const size_t *origin, const size_t *region, size_t input_row_pitch, size_t input_slice_pitch, const void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string orig_str = size_array_to_string(origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueWriteImage(command_queue, image, blocking_write, origin, region, input_row_pitch, + input_slice_pitch, ptr, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueWriteImage", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "blocking", blocking_write, "origin", perfetto::DynamicString(orig_str), "region", + perfetto::DynamicString(reg_str), "row_pitch", input_row_pitch, "slice_pitch", input_slice_pitch, "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueCopyImage(cl_command_queue command_queue, cl_mem src_image, cl_mem dst_image, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + src_id = g_mem_ids.get_or_create(src_image); + dst_id = g_mem_ids.get_or_create(dst_image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string so_str = size_array_to_string(src_origin, 3); + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueCopyImage(command_queue, src_image, dst_image, src_origin, dst_origin, region, + num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCopyImage", "call_id", cid, "queue_id", q_id, "src_mem_id", src_id, + "dst_mem_id", dst_id, "src_origin", perfetto::DynamicString(so_str), "dst_origin", + perfetto::DynamicString(do_str), "region", perfetto::DynamicString(reg_str), "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueFillImage(cl_command_queue command_queue, cl_mem image, const void *fill_color, + const size_t *origin, const size_t *region, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string color_hex = clkp::bytes_to_hex(fill_color, 16); + std::string orig_str = size_array_to_string(origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueFillImage( + command_queue, image, fill_color, origin, region, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueFillImage", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "fill_color_hex", perfetto::DynamicString(color_hex), "origin", perfetto::DynamicString(orig_str), "region", + perfetto::DynamicString(reg_str), "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueCopyImageToBuffer(cl_command_queue command_queue, cl_mem src_image, cl_mem dst_buffer, + const size_t *src_origin, const size_t *region, size_t dst_offset, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + src_id = g_mem_ids.get_or_create(src_image); + dst_id = g_mem_ids.get_or_create(dst_buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string so_str = size_array_to_string(src_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueCopyImageToBuffer(command_queue, src_image, dst_buffer, src_origin, region, + dst_offset, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCopyImageToBuffer", "call_id", cid, "queue_id", q_id, "src_mem_id", + src_id, "dst_mem_id", dst_id, "src_origin", perfetto::DynamicString(so_str), "region", + perfetto::DynamicString(reg_str), "dst_offset", dst_offset, "wait_events", perfetto::DynamicString(wait_str), + "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueCopyBufferToImage(cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_image, + size_t src_offset, const size_t *dst_origin, const size_t *region, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + cl_int err = tdispatch->clEnqueueCopyBufferToImage(command_queue, src_buffer, dst_image, src_offset, dst_origin, + region, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCopyBufferToImage", "call_id", cid, "queue_id", q_id, "src_mem_id", + src_id, "dst_mem_id", dst_id, "src_offset", src_offset, "dst_origin", perfetto::DynamicString(do_str), "region", + perfetto::DynamicString(reg_str), "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static void *clkp_clEnqueueMapBuffer(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_map, + cl_map_flags map_flags, size_t offset, size_t size, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event, cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + void *ptr = tdispatch->clEnqueueMapBuffer(command_queue, buffer, blocking_map, map_flags, offset, size, + num_events_in_wait_list, event_wait_list, event, errcode_ret); + uint64_t map_id; + uint64_t ret_ev = 0; + { + std::lock_guard lock(g_lock); + map_id = g_map_ids.get_or_create(ptr); + if (event && *event) { + ret_ev = g_event_ids.create_fresh(*event); + } + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueMapBuffer", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "map_id", map_id, "blocking", blocking_map, "map_flags", static_cast(map_flags), "offset", offset, + "size", size, "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return ptr; +} + +static void *clkp_clEnqueueMapImage(cl_command_queue command_queue, cl_mem image, cl_bool blocking_map, + cl_map_flags map_flags, const size_t *origin, const size_t *region, size_t *image_row_pitch, + size_t *image_slice_pitch, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event, + cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(image); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string orig_str = size_array_to_string(origin, 3); + std::string reg_str = size_array_to_string(region, 3); + void *ptr = tdispatch->clEnqueueMapImage(command_queue, image, blocking_map, map_flags, origin, region, + image_row_pitch, image_slice_pitch, num_events_in_wait_list, event_wait_list, event, errcode_ret); + uint64_t map_id; + uint64_t ret_ev = 0; + { + std::lock_guard lock(g_lock); + map_id = g_map_ids.get_or_create(ptr); + if (event && *event) { + ret_ev = g_event_ids.create_fresh(*event); + } + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueMapImage", "call_id", cid, "queue_id", q_id, "mem_id", m_id, "map_id", + map_id, "blocking", blocking_map, "map_flags", static_cast(map_flags), "origin", + perfetto::DynamicString(orig_str), "region", perfetto::DynamicString(reg_str), "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return ptr; +} + +static cl_int clkp_clEnqueueUnmapMemObject(cl_command_queue command_queue, cl_mem memobj, void *mapped_ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + uint64_t m_id; + uint64_t map_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + m_id = g_mem_ids.get_or_create(memobj); + map_id = g_map_ids.find(mapped_ptr); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err = tdispatch->clEnqueueUnmapMemObject( + command_queue, memobj, mapped_ptr, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueUnmapMemObject", "call_id", cid, "queue_id", q_id, "mem_id", m_id, + "map_id", map_id, "wait_events", perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueMigrateMemObjects(cl_command_queue command_queue, cl_uint num_mem_objects, + const cl_mem *mem_objects, cl_mem_migration_flags flags, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + std::vector m_ids; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + for (cl_uint i = 0; i < num_mem_objects && mem_objects != nullptr; ++i) { + m_ids.push_back(g_mem_ids.get_or_create(mem_objects[i])); + } + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + std::string mems_str = clkp::uint64_list_to_string(m_ids); + cl_int err = tdispatch->clEnqueueMigrateMemObjects( + command_queue, num_mem_objects, mem_objects, flags, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueMigrateMemObjects", "call_id", cid, "queue_id", q_id, "mem_ids", + perfetto::DynamicString(mems_str), "flags", static_cast(flags), "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueMarkerWithWaitList( + cl_command_queue command_queue, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err = tdispatch->clEnqueueMarkerWithWaitList(command_queue, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueMarkerWithWaitList", "call_id", cid, "queue_id", q_id, "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueBarrierWithWaitList( + cl_command_queue command_queue, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + } + cl_int err + = tdispatch->clEnqueueBarrierWithWaitList(command_queue, num_events_in_wait_list, event_wait_list, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueBarrierWithWaitList", "call_id", cid, "queue_id", q_id, "wait_events", + perfetto::DynamicString(wait_str), "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueMarker(cl_command_queue command_queue, cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + } + cl_int err = tdispatch->clEnqueueMarker(command_queue, event); + uint64_t ret_ev = 0; + if (event && err == CL_SUCCESS && *event) { + std::lock_guard lock(g_lock); + ret_ev = g_event_ids.create_fresh(*event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueMarker", "call_id", cid, "queue_id", q_id, "ret_event_id", ret_ev); + return err; +} + +static cl_int clkp_clEnqueueBarrier(cl_command_queue command_queue) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + } + cl_int err = tdispatch->clEnqueueBarrier(command_queue); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueBarrier", "call_id", cid, "queue_id", q_id); + return err; +} + +static cl_int clkp_clEnqueueWaitForEvents( + cl_command_queue command_queue, cl_uint num_events, const cl_event *event_list) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + std::string wait_str; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + wait_str = event_list_to_string(num_events, event_list); + } + cl_int err = tdispatch->clEnqueueWaitForEvents(command_queue, num_events, event_list); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueWaitForEvents", "call_id", cid, "queue_id", q_id, "wait_events", + perfetto::DynamicString(wait_str)); + return err; +} + +static cl_int clkp_clFlush(cl_command_queue command_queue) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clFlush", "call_id", cid, "queue_id", q_id); + return tdispatch->clFlush(command_queue); +} + +static cl_int clkp_clFinish(cl_command_queue command_queue) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clFinish", "call_id", cid, "queue_id", q_id); + return tdispatch->clFinish(command_queue); +} + +static cl_int clkp_clWaitForEvents(cl_uint num_events, const cl_event *event_list) +{ + uint64_t cid = next_call_id(); + std::string wait_str; + { + std::lock_guard lock(g_lock); + wait_str = event_list_to_string(num_events, event_list); + } + TRACE_EVENT( + CLKP_PERFETTO_CATEGORY, "clWaitForEvents", "call_id", cid, "wait_events", perfetto::DynamicString(wait_str)); + return tdispatch->clWaitForEvents(num_events, event_list); +} + +static cl_event clkp_clCreateUserEvent(cl_context context, cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t ctx_id; + { + std::lock_guard lock(g_lock); + ctx_id = g_context_ids.get_or_create(context); + } + cl_event ev = tdispatch->clCreateUserEvent(context, errcode_ret); + uint64_t ev_id; + { + std::lock_guard lock(g_lock); + ev_id = g_event_ids.create_fresh(ev); + } + TRACE_EVENT( + CLKP_PERFETTO_CATEGORY, "clCreateUserEvent", "call_id", cid, "context_id", ctx_id, "ret_event_id", ev_id); + return ev; +} + +static cl_int clkp_clSetUserEventStatus(cl_event event, cl_int execution_status) +{ + uint64_t cid = next_call_id(); + uint64_t ev_id; + { + std::lock_guard lock(g_lock); + ev_id = g_event_ids.get_or_create(event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clSetUserEventStatus", "call_id", cid, "event_id", ev_id, "execution_status", + execution_status); + return tdispatch->clSetUserEventStatus(event, execution_status); +} + +static cl_int clkp_clRetainEvent(cl_event event) +{ + uint64_t cid = next_call_id(); + uint64_t ev_id; + { + std::lock_guard lock(g_lock); + ev_id = g_event_ids.get_or_create(event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainEvent", "call_id", cid, "event_id", ev_id); + return tdispatch->clRetainEvent(event); +} + +static cl_int clkp_clReleaseEvent(cl_event event) +{ + uint64_t cid = next_call_id(); + uint64_t ev_id; + { + std::lock_guard lock(g_lock); + ev_id = g_event_ids.find(event); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseEvent", "call_id", cid, "event_id", ev_id); + return tdispatch->clReleaseEvent(event); +} + +/*****************************************************************************/ +/* CREATE COMMAND QUEUE ******************************************************/ +/*****************************************************************************/ + +static cl_int clkp_clRetainCommandQueue(cl_command_queue command_queue) +{ + uint64_t cid = next_call_id(); + uint64_t q_id; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.get_or_create(command_queue); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainCommandQueue", "call_id", cid, "queue_id", q_id); + cl_int ret = tdispatch->clRetainCommandQueue(command_queue); + if (ret == CL_SUCCESS && command_queue != nullptr) { + std::lock_guard lock(g_lock); + if (queue_to_refcount.count(command_queue)) { + queue_to_refcount[command_queue]++; + } + } + return ret; +} + +static cl_int clkp_clReleaseCommandQueue(cl_command_queue command_queue) +{ + uint64_t cid = next_call_id(); + uint64_t q_id = 0; + ThreadInfo *thread_info = nullptr; + std::thread thread_to_join; + { + std::lock_guard lock(g_lock); + q_id = g_queue_ids.find(command_queue); + if (command_queue != nullptr) { + bool destroy = true; + auto ref_it = queue_to_refcount.find(command_queue); + if (ref_it != queue_to_refcount.end()) { + if (ref_it->second > 1) { + ref_it->second--; + destroy = false; + } else { + queue_to_refcount.erase(ref_it); + } + } + if (destroy) { + if (queue_to_thread_info.count(command_queue)) { + thread_info = queue_to_thread_info[command_queue]; + queue_to_thread_info.erase(command_queue); + } + if (queue_to_thread.count(command_queue)) { + thread_to_join = std::move(queue_to_thread[command_queue]); + queue_to_thread.erase(command_queue); + } + queue_to_time_offset.erase(command_queue); + } + } + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseCommandQueue", "call_id", cid, "queue_id", q_id); + + auto ret = tdispatch->clReleaseCommandQueue(command_queue); + + if (thread_info) { + { + std::unique_lock lock_ti(thread_info->lock); + thread_info->stop = true; + thread_info->cv.notify_all(); + thread_info->cv.wait(lock_ti, [&] { return thread_info->pending_callbacks == 0; }); + } + if (thread_to_join.joinable()) { + thread_to_join.join(); + } + delete thread_info; + } + + return ret; +} + +static cl_command_queue create_command_queue( + cl_context context, cl_device_id device, const cl_queue_properties *properties, cl_int *errcode_ret, uint64_t q_id) +{ + std::vector properties_array; + bool cl_queue_properties_found = false; + if (properties) { + for (unsigned i = 0; properties[i] != 0; i += 2) { + cl_queue_properties key = properties[i]; + cl_queue_properties val = properties[i + 1]; + if (key == CL_QUEUE_PROPERTIES) { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue-properties", "properties", val); + + val |= CL_QUEUE_PROFILING_ENABLE; + cl_queue_properties_found = true; + } + properties_array.push_back(key); + properties_array.push_back(val); + } + } + if (!cl_queue_properties_found) { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue-properties-not-found"); + properties_array.push_back(CL_QUEUE_PROPERTIES); + properties_array.push_back(CL_QUEUE_PROFILING_ENABLE); + } + properties_array.push_back(0); + + auto command_queue + = tdispatch->clCreateCommandQueueWithProperties(context, device, properties_array.data(), errcode_ret); + + int64_t offset = 0; + if (command_queue != nullptr) { + if (tdispatch->clGetDeviceAndHostTimer) { + cl_ulong device_timestamp, host_timestamp; + cl_int timer_err = tdispatch->clGetDeviceAndHostTimer(device, &device_timestamp, &host_timestamp); + if (timer_err == CL_SUCCESS) { + uint64_t perfetto_timestamp = perfetto::TrackEvent::GetTraceTimeNs(); + offset = perfetto_timestamp - device_timestamp; + } else { + PRINT("clGetDeviceAndHostTimer failed (%i)", timer_err); + } + } else { + PRINT("clGetDeviceAndHostTimer is not available"); + } + } + + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + g_queue_ids.bind(command_queue, q_id); + ctx_id = g_context_ids.get_or_create(context); + if (command_queue != nullptr) { + queue_to_context_id[command_queue] = ctx_id; + ThreadInfo *thread_info = new ThreadInfo(); + thread_info->stop = false; + thread_info->pending_callbacks = 0; + queue_to_thread_info[command_queue] = thread_info; + queue_to_thread.emplace(command_queue, [thread_info] { queue_thread_function(thread_info); }); + queue_to_time_offset[command_queue] = offset; + queue_to_refcount[command_queue] = 1; + } + } + + if (command_queue != nullptr) { + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, + perfetto::DynamicString("clkp-queue_" + std::to_string((uintptr_t)command_queue)), + perfetto::Track((uintptr_t)command_queue), "queue_id", q_id, "context_id", ctx_id); + } + + return command_queue; +} + +static uint64_t extract_queue_properties_bitfield(const cl_queue_properties *properties) +{ + if (properties == nullptr) { + return 0; + } + for (unsigned i = 0; properties[i] != 0; i += 2) { + if (properties[i] == CL_QUEUE_PROPERTIES) { + return static_cast(properties[i + 1]); + } + } + return 0; +} + +static cl_command_queue clkp_clCreateCommandQueue( + cl_context context, cl_device_id device, cl_command_queue_properties properties, cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t ctx_id; + uint64_t q_id; + { + std::lock_guard lock(g_lock); + ctx_id = g_context_ids.get_or_create(context); + q_id = g_queue_ids.allocate_id(); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue", "call_id", cid, "context_id", ctx_id, "queue_id", q_id, + "properties", static_cast(properties)); + cl_queue_properties props[4] = { CL_QUEUE_PROPERTIES, properties, 0, 0 }; + return create_command_queue(context, device, props, errcode_ret, q_id); +} + +static cl_command_queue clkp_clCreateCommandQueueWithProperties( + cl_context context, cl_device_id device, const cl_queue_properties *properties, cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t ctx_id; + uint64_t q_id; + { + std::lock_guard lock(g_lock); + ctx_id = g_context_ids.get_or_create(context); + q_id = g_queue_ids.allocate_id(); + } + uint64_t orig_props = extract_queue_properties_bitfield(properties); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueueWithProperties", "call_id", cid, "context_id", ctx_id, + "queue_id", q_id, "properties", orig_props); + return create_command_queue(context, device, properties, errcode_ret, q_id); +} + +/*****************************************************************************/ +/* CL_KHR_COMMAND_BUFFER & EXTENSION FUNCTIONS *******************************/ +/*****************************************************************************/ + +struct ExtensionDispatch { + clCreateCommandBufferKHR_fn clCreateCommandBufferKHR = nullptr; + clFinalizeCommandBufferKHR_fn clFinalizeCommandBufferKHR = nullptr; + clRetainCommandBufferKHR_fn clRetainCommandBufferKHR = nullptr; + clReleaseCommandBufferKHR_fn clReleaseCommandBufferKHR = nullptr; + clEnqueueCommandBufferKHR_fn clEnqueueCommandBufferKHR = nullptr; + clCommandBarrierWithWaitListKHR_fn clCommandBarrierWithWaitListKHR = nullptr; + clCommandCopyBufferKHR_fn clCommandCopyBufferKHR = nullptr; + clCommandCopyBufferRectKHR_fn clCommandCopyBufferRectKHR = nullptr; + clCommandCopyBufferToImageKHR_fn clCommandCopyBufferToImageKHR = nullptr; + clCommandCopyImageKHR_fn clCommandCopyImageKHR = nullptr; + clCommandCopyImageToBufferKHR_fn clCommandCopyImageToBufferKHR = nullptr; + clCommandFillBufferKHR_fn clCommandFillBufferKHR = nullptr; + clCommandFillImageKHR_fn clCommandFillImageKHR = nullptr; + clCommandNDRangeKernelKHR_fn clCommandNDRangeKernelKHR = nullptr; + clGetCommandBufferInfoKHR_fn clGetCommandBufferInfoKHR = nullptr; + clUpdateMutableCommandsKHR_fn clUpdateMutableCommandsKHR = nullptr; + clGetMutableCommandInfoKHR_fn clGetMutableCommandInfoKHR = nullptr; + clCreateProgramWithILKHR_fn clCreateProgramWithILKHR = nullptr; + clCreateCommandQueueWithPropertiesKHR_fn clCreateCommandQueueWithPropertiesKHR = nullptr; +}; + +static ExtensionDispatch g_ext; + +static cl_command_buffer_khr CL_API_CALL clkp_clCreateCommandBufferKHR(cl_uint num_queues, + const cl_command_queue *queues, const cl_command_buffer_properties_khr *properties, cl_int *errcode_ret) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + std::vector q_ids; + std::vector q_vec; + uint64_t ctx_id = 0; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.allocate_id(); + for (cl_uint i = 0; i < num_queues && queues != nullptr; ++i) { + q_ids.push_back(g_queue_ids.get_or_create(queues[i])); + q_vec.push_back(queues[i]); + if (ctx_id == 0 && queue_to_context_id.count(queues[i])) { + ctx_id = queue_to_context_id[queues[i]]; + } + } + } + std::string q_ids_str = clkp::uint64_list_to_string(q_ids); + std::string props_str = properties_to_string(properties); + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateCommandBufferKHR", "call_id", cid, "cmdbuf_id", cb_id, "context_id", + ctx_id, "queue_ids", perfetto::DynamicString(q_ids_str), "properties", perfetto::DynamicString(props_str)); + cl_command_buffer_khr cmdbuf = g_ext.clCreateCommandBufferKHR(num_queues, queues, properties, errcode_ret); + { + std::lock_guard lock(g_lock); + g_cmdbuf_ids.bind(cmdbuf, cb_id); + if (cmdbuf != nullptr) { + cmdbuf_to_queues[cmdbuf] = std::move(q_vec); + cmdbuf_to_context_id[cmdbuf] = ctx_id; + } + } + return cmdbuf; +} + +static cl_int CL_API_CALL clkp_clFinalizeCommandBufferKHR(cl_command_buffer_khr command_buffer) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clFinalizeCommandBufferKHR", "call_id", cid, "cmdbuf_id", cb_id); + return g_ext.clFinalizeCommandBufferKHR(command_buffer); +} + +static cl_int CL_API_CALL clkp_clRetainCommandBufferKHR(cl_command_buffer_khr command_buffer) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clRetainCommandBufferKHR", "call_id", cid, "cmdbuf_id", cb_id); + return g_ext.clRetainCommandBufferKHR(command_buffer); +} + +static cl_int CL_API_CALL clkp_clReleaseCommandBufferKHR(cl_command_buffer_khr command_buffer) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.find(command_buffer); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseCommandBufferKHR", "call_id", cid, "cmdbuf_id", cb_id); + return g_ext.clReleaseCommandBufferKHR(command_buffer); +} + +static cl_int CL_API_CALL clkp_clEnqueueCommandBufferKHR(cl_uint num_queues, cl_command_queue *queues, + cl_command_buffer_khr command_buffer, cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) +{ + uint64_t cid = next_call_id(); + uint64_t did = next_dispatch_id(); + uint64_t cb_id; + std::vector q_ids; + cl_command_queue target_queue = nullptr; + std::string wait_str; + bool event_is_null = (event == nullptr); + cl_event local_event = nullptr; + cl_event *event_to_use = event_is_null ? &local_event : event; + uint64_t ret_ev = 0; + + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + if (num_queues > 0 && queues != nullptr) { + target_queue = queues[0]; + for (cl_uint i = 0; i < num_queues; ++i) { + q_ids.push_back(g_queue_ids.get_or_create(queues[i])); + } + } else if (cmdbuf_to_queues.count(command_buffer) && !cmdbuf_to_queues[command_buffer].empty()) { + target_queue = cmdbuf_to_queues[command_buffer][0]; + } + wait_str = event_list_to_string(num_events_in_wait_list, event_wait_list); + if (!event_is_null) { + ret_ev = g_event_ids.allocate_id(); + } + } + std::string q_ids_str = clkp::uint64_list_to_string(q_ids); + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueCommandBufferKHR", "call_id", cid, "dispatch_id", did, "cmdbuf_id", + cb_id, "queue_ids", perfetto::DynamicString(q_ids_str), "wait_events", perfetto::DynamicString(wait_str), + "ret_event_id", ret_ev); + + cl_int err = g_ext.clEnqueueCommandBufferKHR( + num_queues, queues, command_buffer, num_events_in_wait_list, event_wait_list, event_to_use); + + int64_t time_offset = 0; + uint64_t target_q_id = 0; + { + std::lock_guard lock(g_lock); + if (err == CL_SUCCESS && !event_is_null && *event != nullptr) { + g_event_ids.bind(*event, ret_ev); + } + if (target_queue != nullptr) { + time_offset = queue_to_time_offset.count(target_queue) ? queue_to_time_offset[target_queue] : 0; + target_q_id = g_queue_ids.find(target_queue); + } + } + + if (err != CL_SUCCESS || target_queue == nullptr || *event_to_use == nullptr) { + if (err == CL_SUCCESS && event_is_null && local_event != nullptr) { + tdispatch->clReleaseEvent(local_event); + } + return err; + } + + struct callback_data *data = new (std::nothrow) callback_data(); + if (data == nullptr) { + if (event_is_null) { + tdispatch->clReleaseEvent(local_event); + } + return CL_OUT_OF_HOST_MEMORY; + } + + data->queue = target_queue; + data->event = *event_to_use; + data->time_offset = time_offset; + data->call_id = cid; + data->dispatch_id = did; + data->queue_id = target_q_id; + data->cmdbuf_id = cb_id; + data->is_cmdbuf = true; + + return attach_profiling_callback(target_queue, event_to_use, event_is_null, local_event, data); +} + +static cl_int CL_API_CALL clkp_clCommandBarrierWithWaitListKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + } + std::string props_str = properties_to_string(properties); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandBarrierWithWaitListKHR(command_buffer, command_queue, properties, + num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandBarrierWithWaitListKHR", "call_id", cid, "cmdbuf_id", cb_id, + "queue_id", q_id, "properties", perfetto::DynamicString(props_str), "wait_sync_points", + perfetto::DynamicString(wait_sp_str), "ret_sync_point", ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandCopyBufferKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem src_buffer, cl_mem dst_buffer, + size_t src_offset, size_t dst_offset, size_t size, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_buffer); + } + std::string props_str = properties_to_string(properties); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandCopyBufferKHR(command_buffer, command_queue, properties, src_buffer, dst_buffer, + src_offset, dst_offset, size, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandCopyBufferKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", q_id, + "properties", perfetto::DynamicString(props_str), "src_mem_id", src_id, "dst_mem_id", dst_id, "src_offset", + src_offset, "dst_offset", dst_offset, "size", size, "wait_sync_points", perfetto::DynamicString(wait_sp_str), + "ret_sync_point", ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandCopyBufferRectKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem src_buffer, cl_mem dst_buffer, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, size_t src_row_pitch, + size_t src_slice_pitch, size_t dst_row_pitch, size_t dst_slice_pitch, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_buffer); + } + std::string props_str = properties_to_string(properties); + std::string so_str = size_array_to_string(src_origin, 3); + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandCopyBufferRectKHR(command_buffer, command_queue, properties, src_buffer, dst_buffer, + src_origin, dst_origin, region, src_row_pitch, src_slice_pitch, dst_row_pitch, dst_slice_pitch, + num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandCopyBufferRectKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", + q_id, "properties", perfetto::DynamicString(props_str), "src_mem_id", src_id, "dst_mem_id", dst_id, + "src_origin", perfetto::DynamicString(so_str), "dst_origin", perfetto::DynamicString(do_str), "region", + perfetto::DynamicString(reg_str), "src_row_pitch", src_row_pitch, "src_slice_pitch", src_slice_pitch, + "dst_row_pitch", dst_row_pitch, "dst_slice_pitch", dst_slice_pitch, "wait_sync_points", + perfetto::DynamicString(wait_sp_str), "ret_sync_point", ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandCopyBufferToImageKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem src_buffer, cl_mem dst_image, + size_t src_offset, const size_t *dst_origin, const size_t *region, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + src_id = g_mem_ids.get_or_create(src_buffer); + dst_id = g_mem_ids.get_or_create(dst_image); + } + std::string props_str = properties_to_string(properties); + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandCopyBufferToImageKHR(command_buffer, command_queue, properties, src_buffer, dst_image, + src_offset, dst_origin, region, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandCopyBufferToImageKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", + q_id, "properties", perfetto::DynamicString(props_str), "src_mem_id", src_id, "dst_mem_id", dst_id, + "src_offset", src_offset, "dst_origin", perfetto::DynamicString(do_str), "region", + perfetto::DynamicString(reg_str), "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", + ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandCopyImageKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem src_image, cl_mem dst_image, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + src_id = g_mem_ids.get_or_create(src_image); + dst_id = g_mem_ids.get_or_create(dst_image); + } + std::string props_str = properties_to_string(properties); + std::string so_str = size_array_to_string(src_origin, 3); + std::string do_str = size_array_to_string(dst_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandCopyImageKHR(command_buffer, command_queue, properties, src_image, dst_image, + src_origin, dst_origin, region, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandCopyImageKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", q_id, + "properties", perfetto::DynamicString(props_str), "src_mem_id", src_id, "dst_mem_id", dst_id, "src_origin", + perfetto::DynamicString(so_str), "dst_origin", perfetto::DynamicString(do_str), "region", + perfetto::DynamicString(reg_str), "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", + ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandCopyImageToBufferKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem src_image, cl_mem dst_buffer, + const size_t *src_origin, const size_t *region, size_t dst_offset, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t src_id; + uint64_t dst_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + src_id = g_mem_ids.get_or_create(src_image); + dst_id = g_mem_ids.get_or_create(dst_buffer); + } + std::string props_str = properties_to_string(properties); + std::string so_str = size_array_to_string(src_origin, 3); + std::string reg_str = size_array_to_string(region, 3); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandCopyImageToBufferKHR(command_buffer, command_queue, properties, src_image, dst_buffer, + src_origin, region, dst_offset, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandCopyImageToBufferKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", + q_id, "properties", perfetto::DynamicString(props_str), "src_mem_id", src_id, "dst_mem_id", dst_id, + "src_origin", perfetto::DynamicString(so_str), "region", perfetto::DynamicString(reg_str), "dst_offset", + dst_offset, "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", ret_sp, + "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandFillBufferKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem buffer, const void *pattern, + size_t pattern_size, size_t offset, size_t size, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t m_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + m_id = g_mem_ids.get_or_create(buffer); + } + std::string props_str = properties_to_string(properties); + std::string pat_hex = clkp::bytes_to_hex(pattern, pattern_size); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandFillBufferKHR(command_buffer, command_queue, properties, buffer, pattern, pattern_size, + offset, size, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandFillBufferKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", q_id, + "properties", perfetto::DynamicString(props_str), "mem_id", m_id, "pattern_hex", + perfetto::DynamicString(pat_hex), "pattern_size", pattern_size, "offset", offset, "size", size, + "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandFillImageKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_mem image, const void *fill_color, + const size_t *origin, const size_t *region, cl_uint num_sync_points_in_wait_list, + const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t m_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + m_id = g_mem_ids.get_or_create(image); + } + std::string props_str = properties_to_string(properties); + std::string color_hex = clkp::bytes_to_hex(fill_color, 16); + std::string orig_str = size_array_to_string(origin, 3); + std::string reg_str = size_array_to_string(region, 3); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandFillImageKHR(command_buffer, command_queue, properties, image, fill_color, origin, + region, num_sync_points_in_wait_list, sync_point_wait_list, sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandFillImageKHR", "call_id", cid, "cmdbuf_id", cb_id, "queue_id", q_id, + "properties", perfetto::DynamicString(props_str), "mem_id", m_id, "fill_color_hex", + perfetto::DynamicString(color_hex), "origin", perfetto::DynamicString(orig_str), "region", + perfetto::DynamicString(reg_str), "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", + ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clCommandNDRangeKernelKHR(cl_command_buffer_khr command_buffer, + cl_command_queue command_queue, const cl_command_properties_khr *properties, cl_kernel kernel, cl_uint work_dim, + const size_t *global_work_offset, const size_t *global_work_size, const size_t *local_work_size, + cl_uint num_sync_points_in_wait_list, const cl_sync_point_khr *sync_point_wait_list, cl_sync_point_khr *sync_point, + cl_mutable_command_khr *mutable_handle) +{ + uint64_t cid = next_call_id(); + uint64_t did = next_dispatch_id(); + uint64_t cb_id; + uint64_t q_id; + uint64_t k_id; + std::string kernel_name; + uint64_t prog_id; + std::string program_string; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + q_id = g_queue_ids.find(command_queue); + k_id = g_kernel_ids.get_or_create(kernel); + kernel_name = kernel_to_kernel_name.count(kernel) ? kernel_to_kernel_name[kernel] : "?"; + cl_program prog = kernel_to_program.count(kernel) ? kernel_to_program[kernel] : nullptr; + prog_id = g_program_ids.find(prog); + program_string = (prog && program_to_string.count(prog)) ? program_to_string[prog] : "clkp_p?"; + } + + size_t gidX = (work_dim > 0 && global_work_size) ? global_work_size[0] : 1; + size_t gidY = (work_dim > 1 && global_work_size) ? global_work_size[1] : 1; + size_t gidZ = (work_dim > 2 && global_work_size) ? global_work_size[2] : 1; + + std::string props_str = properties_to_string(properties); + std::string gwo_str = size_array_to_string(global_work_offset, work_dim); + std::string gws_str = size_array_to_string(global_work_size, work_dim); + std::string lws_str = size_array_to_string(local_work_size, work_dim); + std::string wait_sp_str = sync_point_list_to_string(num_sync_points_in_wait_list, sync_point_wait_list); + + cl_int err = g_ext.clCommandNDRangeKernelKHR(command_buffer, command_queue, properties, kernel, work_dim, + global_work_offset, global_work_size, local_work_size, num_sync_points_in_wait_list, sync_point_wait_list, + sync_point, mutable_handle); + + uint64_t ret_sp = (err == CL_SUCCESS && sync_point != nullptr) ? static_cast(*sync_point) : 0; + uint64_t mut_id = 0; + if (err == CL_SUCCESS && mutable_handle != nullptr) { + std::lock_guard lock(g_lock); + mut_id = g_mutable_cmd_ids.create_fresh(*mutable_handle); + } + + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCommandNDRangeKernelKHR", "call_id", cid, "dispatch_id", did, "cmdbuf_id", + cb_id, "queue_id", q_id, "properties", perfetto::DynamicString(props_str), "kernel_id", k_id, "program_id", + prog_id, "program", perfetto::DynamicString(program_string), "kernel_name", + perfetto::DynamicString(kernel_name), "work_dim", work_dim, "gidX", gidX, "gidY", gidY, "gidZ", gidZ, "gwo", + perfetto::DynamicString(gwo_str), "gws", perfetto::DynamicString(gws_str), "lws", + perfetto::DynamicString(lws_str), "wait_sync_points", perfetto::DynamicString(wait_sp_str), "ret_sync_point", + ret_sp, "mutable_cmd_id", mut_id); + return err; +} + +static cl_int CL_API_CALL clkp_clGetCommandBufferInfoKHR(cl_command_buffer_khr command_buffer, + cl_command_buffer_info_khr param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret) +{ + return g_ext.clGetCommandBufferInfoKHR( + command_buffer, param_name, param_value_size, param_value, param_value_size_ret); +} + +static cl_int CL_API_CALL clkp_clUpdateMutableCommandsKHR(cl_command_buffer_khr command_buffer, cl_uint num_configs, + const cl_command_buffer_update_type_khr *config_types, const void **configs) +{ + uint64_t cid = next_call_id(); + uint64_t cb_id; + { + std::lock_guard lock(g_lock); + cb_id = g_cmdbuf_ids.get_or_create(command_buffer); + } + TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clUpdateMutableCommandsKHR", "call_id", cid, "cmdbuf_id", cb_id, "num_configs", + num_configs); + + if (config_types != nullptr && configs != nullptr) { + std::lock_guard lock(g_lock); + for (cl_uint i = 0; i < num_configs; ++i) { + if (config_types[i] == CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR && configs[i] != nullptr) { + const auto *cfg = reinterpret_cast(configs[i]); + uint64_t mut_id = g_mutable_cmd_ids.get_or_create(cfg->command); + cl_uint dim = cfg->work_dim > 0 ? cfg->work_dim : 3; + std::string gwo_str = size_array_to_string(cfg->global_work_offset, dim); + std::string gws_str = size_array_to_string(cfg->global_work_size, dim); + std::string lws_str = size_array_to_string(cfg->local_work_size, dim); + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clUpdateMutableCommandsKHR-config", "call_id", cid, + "cmdbuf_id", cb_id, "mutable_cmd_id", mut_id, "work_dim", cfg->work_dim, "gwo", + perfetto::DynamicString(gwo_str), "gws", perfetto::DynamicString(gws_str), "lws", + perfetto::DynamicString(lws_str), "num_args", cfg->num_args); + + if (cfg->arg_list != nullptr) { + for (cl_uint j = 0; j < cfg->num_args; ++j) { + const auto &arg = cfg->arg_list[j]; + std::string arg_kind; + uint64_t mem_id = 0; + uint64_t sampler_id = 0; + std::string val_hex; + classify_kernel_arg(arg.arg_size, arg.arg_value, arg_kind, mem_id, sampler_id, val_hex); + TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clUpdateMutableCommandsKHR-arg", "call_id", cid, + "cmdbuf_id", cb_id, "mutable_cmd_id", mut_id, "arg_index", arg.arg_index, "arg_size", + arg.arg_size, "arg_kind", perfetto::DynamicString(arg_kind), "mem_id", mem_id, "sampler_id", + sampler_id, "value_hex", perfetto::DynamicString(val_hex)); + } + } + } + } + } + + return g_ext.clUpdateMutableCommandsKHR(command_buffer, num_configs, config_types, configs); +} + +static cl_int CL_API_CALL clkp_clGetMutableCommandInfoKHR(cl_mutable_command_khr command, + cl_mutable_command_info_khr param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret) +{ + return g_ext.clGetMutableCommandInfoKHR(command, param_name, param_value_size, param_value, param_value_size_ret); +} + +static cl_program CL_API_CALL clkp_clCreateProgramWithILKHR( + cl_context context, const void *il, size_t length, cl_int *errcode_ret) +{ + return create_program_with_il_common(context, il, length, errcode_ret, g_ext.clCreateProgramWithILKHR); +} + +static cl_command_queue CL_API_CALL clkp_clCreateCommandQueueWithPropertiesKHR( + cl_context context, cl_device_id device, const cl_queue_properties_khr *properties, cl_int *errcode_ret) +{ + return clkp_clCreateCommandQueueWithProperties(context, device, properties, errcode_ret); +} + +static void *wrap_extension_function(const char *func_name, void *real_fn) +{ + if (func_name == nullptr || real_fn == nullptr) { + return real_fn; + } + std::lock_guard lock(g_lock); +#define HANDLE_EXT(name) \ + if (strcmp(func_name, #name) == 0) { \ + g_ext.name = reinterpret_cast(real_fn); \ + return reinterpret_cast(&clkp_##name); \ + } + HANDLE_EXT(clCreateCommandBufferKHR) + HANDLE_EXT(clFinalizeCommandBufferKHR) + HANDLE_EXT(clRetainCommandBufferKHR) + HANDLE_EXT(clReleaseCommandBufferKHR) + HANDLE_EXT(clEnqueueCommandBufferKHR) + HANDLE_EXT(clCommandBarrierWithWaitListKHR) + HANDLE_EXT(clCommandCopyBufferKHR) + HANDLE_EXT(clCommandCopyBufferRectKHR) + HANDLE_EXT(clCommandCopyBufferToImageKHR) + HANDLE_EXT(clCommandCopyImageKHR) + HANDLE_EXT(clCommandCopyImageToBufferKHR) + HANDLE_EXT(clCommandFillBufferKHR) + HANDLE_EXT(clCommandFillImageKHR) + HANDLE_EXT(clCommandNDRangeKernelKHR) + HANDLE_EXT(clGetCommandBufferInfoKHR) + HANDLE_EXT(clUpdateMutableCommandsKHR) + HANDLE_EXT(clGetMutableCommandInfoKHR) + HANDLE_EXT(clCreateProgramWithILKHR) + HANDLE_EXT(clCreateCommandQueueWithPropertiesKHR) +#undef HANDLE_EXT + return real_fn; +} + +static void *clkp_clGetExtensionFunctionAddress(const char *func_name) +{ + void *real_fn + = tdispatch->clGetExtensionFunctionAddress ? tdispatch->clGetExtensionFunctionAddress(func_name) : nullptr; + return wrap_extension_function(func_name, real_fn); +} + +static void *clkp_clGetExtensionFunctionAddressForPlatform(cl_platform_id platform, const char *func_name) +{ + void *real_fn = nullptr; + if (tdispatch->clGetExtensionFunctionAddressForPlatform) { + real_fn = tdispatch->clGetExtensionFunctionAddressForPlatform(platform, func_name); + } + if (real_fn == nullptr && tdispatch->clGetExtensionFunctionAddress) { + real_fn = tdispatch->clGetExtensionFunctionAddress(func_name); + } + return wrap_extension_function(func_name, real_fn); +} + +/*****************************************************************************/ +/* PERFETTO TRACE PARAMETERS *************************************************/ +/*****************************************************************************/ + +#ifdef BACKEND_INPROCESS +static const char *get_trace_dest() +{ + if (auto trace_dest = getenv("CLKP_TRACE_DEST")) { + return trace_dest; + } + return TRACE_DEST; +} + +static const uint32_t get_trace_max_size() +{ + if (auto trace_max_size = getenv("CLKP_TRACE_MAX_SIZE")) { + return atoi(trace_max_size); + } + return TRACE_MAX_SIZE; +} +#endif + +/*****************************************************************************/ +/* LAYER FUNCTIONS ***********************************************************/ +/*****************************************************************************/ + +CL_API_ENTRY cl_int CL_API_CALL clGetLayerInfo( + cl_layer_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret) +{ + switch (param_name) { + case CL_LAYER_API_VERSION: + if (param_value) { + if (param_value_size < sizeof(cl_layer_api_version)) + return CL_INVALID_VALUE; + *((cl_layer_api_version *)param_value) = CL_LAYER_API_VERSION_100; + } + if (param_value_size_ret) + *param_value_size_ret = sizeof(cl_layer_api_version); + break; + default: + return CL_INVALID_VALUE; + } + return CL_SUCCESS; +} + +CL_API_ENTRY cl_int CL_API_CALL clDeinitLayer() +{ +#ifdef BACKEND_INPROCESS + gTracingSession->StopBlocking(); + std::vector trace_data(gTracingSession->ReadTraceBlocking()); + delete gTracingSession; + + std::ofstream output; + output.open(get_trace_dest(), std::ios::out | std::ios::binary); + output.write(&trace_data[0], trace_data.size()); + output.close(); +#else + perfetto::TrackEvent::Flush(); +#endif + return 0; +} + +CL_API_ENTRY cl_int CL_API_CALL clInitLayerWithProperties(cl_uint num_entries, const cl_icd_dispatch *target_dispatch, + cl_uint *num_entries_out, const cl_icd_dispatch **layer_dispatch_ret, const cl_layer_properties *properties) +{ + std::lock_guard lock(g_lock); + if (!target_dispatch || !num_entries_out || !layer_dispatch_ret) + return CL_INVALID_VALUE; + if (num_entries < sizeof(dispatch) / sizeof(dispatch.clGetPlatformIDs)) + return CL_INVALID_VALUE; + + perfetto::TracingInitArgs args; +#ifdef BACKEND_INPROCESS + args.backends |= perfetto::kInProcessBackend; +#else + args.backends |= perfetto::kSystemBackend; +#endif + perfetto::Tracing::Initialize(args); + perfetto::TrackEvent::Register(); + +#ifdef BACKEND_INPROCESS + perfetto::protos::gen::TrackEventConfig track_event_cfg; + perfetto::TraceConfig cfg; + cfg.add_buffers()->set_size_kb(get_trace_max_size()); + auto *ds_cfg = cfg.add_data_sources()->mutable_config(); + ds_cfg->set_name("track_event"); + ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString()); + + gTracingSession = perfetto::Tracing::NewTrace().release(); + gTracingSession->Setup(cfg); + gTracingSession->StartBlocking(); +#endif + + const uint32_t max_retry = 100; + uint32_t retry = 0; + while ((retry++ < max_retry) && !TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + if (!TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { + PRINT("perfetto category does not seem to be enabled"); + } + + memset(&dispatch, 0, sizeof(dispatch)); + dispatch.clCreateContext = clkp_clCreateContext; + dispatch.clCreateContextFromType = clkp_clCreateContextFromType; + dispatch.clRetainContext = clkp_clRetainContext; + dispatch.clReleaseContext = clkp_clReleaseContext; + + dispatch.clCreateCommandQueue = clkp_clCreateCommandQueue; + dispatch.clCreateCommandQueueWithProperties = clkp_clCreateCommandQueueWithProperties; + dispatch.clRetainCommandQueue = clkp_clRetainCommandQueue; + dispatch.clReleaseCommandQueue = clkp_clReleaseCommandQueue; + + dispatch.clCreateBuffer = clkp_clCreateBuffer; + dispatch.clCreateBufferWithProperties = clkp_clCreateBufferWithProperties; + dispatch.clCreateSubBuffer = clkp_clCreateSubBuffer; + dispatch.clCreateImage = clkp_clCreateImage; + dispatch.clCreateImageWithProperties = clkp_clCreateImageWithProperties; + dispatch.clCreateImage2D = clkp_clCreateImage2D; + dispatch.clCreateImage3D = clkp_clCreateImage3D; + dispatch.clRetainMemObject = clkp_clRetainMemObject; + dispatch.clReleaseMemObject = clkp_clReleaseMemObject; + + dispatch.clCreateSampler = clkp_clCreateSampler; + dispatch.clCreateSamplerWithProperties = clkp_clCreateSamplerWithProperties; + dispatch.clRetainSampler = clkp_clRetainSampler; + dispatch.clReleaseSampler = clkp_clReleaseSampler; + + dispatch.clCreateProgramWithSource = clkp_clCreateProgramWithSource; + dispatch.clCreateProgramWithBinary = clkp_clCreateProgramWithBinary; + dispatch.clCreateProgramWithIL = clkp_clCreateProgramWithIL; + dispatch.clGetProgramInfo = clkp_clGetProgramInfo; + dispatch.clBuildProgram = clkp_clBuildProgram; + dispatch.clCompileProgram = clkp_clCompileProgram; + dispatch.clLinkProgram = clkp_clLinkProgram; + dispatch.clSetProgramSpecializationConstant = clkp_clSetProgramSpecializationConstant; + dispatch.clRetainProgram = clkp_clRetainProgram; + dispatch.clReleaseProgram = clkp_clReleaseProgram; + + dispatch.clCreateKernel = clkp_clCreateKernel; + dispatch.clCreateKernelsInProgram = clkp_clCreateKernelsInProgram; + dispatch.clCloneKernel = clkp_clCloneKernel; + dispatch.clSetKernelArg = clkp_clSetKernelArg; + dispatch.clRetainKernel = clkp_clRetainKernel; + dispatch.clReleaseKernel = clkp_clReleaseKernel; + + dispatch.clEnqueueNDRangeKernel = clkp_clEnqueueNDRangeKernel; + dispatch.clEnqueueTask = clkp_clEnqueueTask; + dispatch.clEnqueueReadBuffer = clkp_clEnqueueReadBuffer; + dispatch.clEnqueueWriteBuffer = clkp_clEnqueueWriteBuffer; + dispatch.clEnqueueReadBufferRect = clkp_clEnqueueReadBufferRect; + dispatch.clEnqueueWriteBufferRect = clkp_clEnqueueWriteBufferRect; + dispatch.clEnqueueCopyBuffer = clkp_clEnqueueCopyBuffer; + dispatch.clEnqueueCopyBufferRect = clkp_clEnqueueCopyBufferRect; + dispatch.clEnqueueFillBuffer = clkp_clEnqueueFillBuffer; + dispatch.clEnqueueReadImage = clkp_clEnqueueReadImage; + dispatch.clEnqueueWriteImage = clkp_clEnqueueWriteImage; + dispatch.clEnqueueCopyImage = clkp_clEnqueueCopyImage; + dispatch.clEnqueueFillImage = clkp_clEnqueueFillImage; + dispatch.clEnqueueCopyImageToBuffer = clkp_clEnqueueCopyImageToBuffer; + dispatch.clEnqueueCopyBufferToImage = clkp_clEnqueueCopyBufferToImage; + dispatch.clEnqueueMapBuffer = clkp_clEnqueueMapBuffer; + dispatch.clEnqueueMapImage = clkp_clEnqueueMapImage; + dispatch.clEnqueueUnmapMemObject = clkp_clEnqueueUnmapMemObject; + dispatch.clEnqueueMigrateMemObjects = clkp_clEnqueueMigrateMemObjects; + dispatch.clEnqueueMarkerWithWaitList = clkp_clEnqueueMarkerWithWaitList; + dispatch.clEnqueueBarrierWithWaitList = clkp_clEnqueueBarrierWithWaitList; + dispatch.clEnqueueMarker = clkp_clEnqueueMarker; + dispatch.clEnqueueBarrier = clkp_clEnqueueBarrier; + dispatch.clEnqueueWaitForEvents = clkp_clEnqueueWaitForEvents; + + dispatch.clFlush = clkp_clFlush; + dispatch.clFinish = clkp_clFinish; + dispatch.clWaitForEvents = clkp_clWaitForEvents; + dispatch.clCreateUserEvent = clkp_clCreateUserEvent; + dispatch.clSetUserEventStatus = clkp_clSetUserEventStatus; + dispatch.clRetainEvent = clkp_clRetainEvent; + dispatch.clReleaseEvent = clkp_clReleaseEvent; + + dispatch.clGetExtensionFunctionAddress = clkp_clGetExtensionFunctionAddress; + dispatch.clGetExtensionFunctionAddressForPlatform = clkp_clGetExtensionFunctionAddressForPlatform; + + tdispatch = target_dispatch; + *layer_dispatch_ret = &dispatch; + *num_entries_out = sizeof(dispatch) / sizeof(dispatch.clGetPlatformIDs); + + return CL_SUCCESS; +} + +CL_API_ENTRY cl_int CL_API_CALL clInitLayer(cl_uint num_entries, const struct _cl_icd_dispatch *target_dispatch, + cl_uint *num_entries_out, const struct _cl_icd_dispatch **layer_dispatch_ret) +{ + return clInitLayerWithProperties(num_entries, target_dispatch, num_entries_out, layer_dispatch_ret, nullptr); +} diff --git a/runner/runner.cpp b/runner/runner.cpp new file mode 100644 index 0000000..09c718c --- /dev/null +++ b/runner/runner.cpp @@ -0,0 +1,1943 @@ +// Copyright 2026 The OpenCL Kernel Profiler authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define CL_TARGET_OPENCL_VERSION 300 +#define CL_ENABLE_BETA_EXTENSIONS +#define CL_USE_DEPRECATED_OPENCL_1_0_APIS +#define CL_USE_DEPRECATED_OPENCL_1_1_APIS +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS +#define CL_USE_DEPRECATED_OPENCL_2_0_APIS + +#include "clkp_common.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct ExtensionFunctions { + clCreateCommandBufferKHR_fn clCreateCommandBufferKHR = nullptr; + clFinalizeCommandBufferKHR_fn clFinalizeCommandBufferKHR = nullptr; + clRetainCommandBufferKHR_fn clRetainCommandBufferKHR = nullptr; + clReleaseCommandBufferKHR_fn clReleaseCommandBufferKHR = nullptr; + clEnqueueCommandBufferKHR_fn clEnqueueCommandBufferKHR = nullptr; + clCommandBarrierWithWaitListKHR_fn clCommandBarrierWithWaitListKHR = nullptr; + clCommandCopyBufferKHR_fn clCommandCopyBufferKHR = nullptr; + clCommandCopyBufferRectKHR_fn clCommandCopyBufferRectKHR = nullptr; + clCommandCopyBufferToImageKHR_fn clCommandCopyBufferToImageKHR = nullptr; + clCommandCopyImageKHR_fn clCommandCopyImageKHR = nullptr; + clCommandCopyImageToBufferKHR_fn clCommandCopyImageToBufferKHR = nullptr; + clCommandFillBufferKHR_fn clCommandFillBufferKHR = nullptr; + clCommandFillImageKHR_fn clCommandFillImageKHR = nullptr; + clCommandNDRangeKernelKHR_fn clCommandNDRangeKernelKHR = nullptr; + clUpdateMutableCommandsKHR_fn clUpdateMutableCommandsKHR = nullptr; + + void load(cl_platform_id platform) + { + auto get_ext = [&](const char *name) -> void * { + void *fn = clGetExtensionFunctionAddressForPlatform(platform, name); + if (!fn) { + fn = clGetExtensionFunctionAddress(name); + } + return fn; + }; + +#define LOAD_EXT(fn) this->fn = reinterpret_cast(get_ext(#fn)) + LOAD_EXT(clCreateCommandBufferKHR); + LOAD_EXT(clFinalizeCommandBufferKHR); + LOAD_EXT(clRetainCommandBufferKHR); + LOAD_EXT(clReleaseCommandBufferKHR); + LOAD_EXT(clEnqueueCommandBufferKHR); + LOAD_EXT(clCommandBarrierWithWaitListKHR); + LOAD_EXT(clCommandCopyBufferKHR); + LOAD_EXT(clCommandCopyBufferRectKHR); + LOAD_EXT(clCommandCopyBufferToImageKHR); + LOAD_EXT(clCommandCopyImageKHR); + LOAD_EXT(clCommandCopyImageToBufferKHR); + LOAD_EXT(clCommandFillBufferKHR); + LOAD_EXT(clCommandFillImageKHR); + LOAD_EXT(clCommandNDRangeKernelKHR); + LOAD_EXT(clUpdateMutableCommandsKHR); +#undef LOAD_EXT + } + + bool supports_command_buffer() const + { + return clCreateCommandBufferKHR != nullptr && clFinalizeCommandBufferKHR != nullptr + && clEnqueueCommandBufferKHR != nullptr && clCommandNDRangeKernelKHR != nullptr; + } +}; + +struct RunnerOptions { + std::string input_path; + std::string json_output_path; + uint32_t platform_index = 0; + uint32_t device_index = 0; + uint32_t warmup_iterations = 0; + uint32_t iterations = 1; + bool has_start_dispatch = false; + bool has_end_dispatch = false; + uint64_t start_dispatch = 0; + uint64_t end_dispatch = UINT64_MAX; + bool use_command_buffer = false; + bool emulate_command_buffer = false; + bool allow_lws_fallback = false; + bool verbose = false; +}; + +void print_usage(const char *prog) +{ + std::cout << "Usage: " << prog << " [options] -i \n" + << "\nOptions:\n" + << " -i, --input Input Perfetto trace file\n" + << " -p, --platform OpenCL platform index (default: 0)\n" + << " -d, --device OpenCL device index (default: 0)\n" + << " -m, --warmup Number of warmup iterations (default: 0)\n" + << " -n, --iterations Number of measured iterations (default: 1)\n" + << " --start-dispatch Start replay at dispatch_id (inclusive)\n" + << " --end-dispatch End replay at dispatch_id (inclusive)\n" + << " --use-command-buffer Promote queue dispatches into a cl_command_buffer_khr\n" + << " --emulate-command-buffer Emulate cl_khr_command_buffer via standard queue enqueues\n" + << " --allow-lws-fallback Fallback to NULL local_work_size if device rejects captured LWS\n" + << " --json Write profiling results to JSON file\n" + << " -v, --verbose Verbose logging of replayed OpenCL calls\n" + << " -h, --help Show this help message\n"; +} + +bool parse_args(int argc, char **argv, RunnerOptions &opts) +{ + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + print_usage(argv[0]); + std::exit(0); + } else if ((arg == "-i" || arg == "--input") && i + 1 < argc) { + opts.input_path = argv[++i]; + } else if ((arg == "-p" || arg == "--platform") && i + 1 < argc) { + opts.platform_index = static_cast(std::strtoul(argv[++i], nullptr, 10)); + } else if ((arg == "-d" || arg == "--device") && i + 1 < argc) { + opts.device_index = static_cast(std::strtoul(argv[++i], nullptr, 10)); + } else if ((arg == "-m" || arg == "--warmup") && i + 1 < argc) { + opts.warmup_iterations = static_cast(std::strtoul(argv[++i], nullptr, 10)); + } else if ((arg == "-n" || arg == "--iterations") && i + 1 < argc) { + opts.iterations = static_cast(std::strtoul(argv[++i], nullptr, 10)); + } else if (arg == "--start-dispatch" && i + 1 < argc) { + opts.has_start_dispatch = true; + opts.start_dispatch = std::strtoull(argv[++i], nullptr, 10); + } else if (arg == "--end-dispatch" && i + 1 < argc) { + opts.has_end_dispatch = true; + opts.end_dispatch = std::strtoull(argv[++i], nullptr, 10); + } else if (arg == "--use-command-buffer") { + opts.use_command_buffer = true; + } else if (arg == "--emulate-command-buffer") { + opts.emulate_command_buffer = true; + } else if (arg == "--allow-lws-fallback") { + opts.allow_lws_fallback = true; + } else if ((arg == "--json" || arg == "--json-output") && i + 1 < argc) { + opts.json_output_path = argv[++i]; + } else if (arg == "-v" || arg == "--verbose") { + opts.verbose = true; + } else if (arg[0] != '-' && opts.input_path.empty()) { + opts.input_path = arg; + } else { + std::cerr << "Unknown or incomplete option: " << arg << "\n"; + return false; + } + } + if (opts.input_path.empty()) { + std::cerr << "Error: missing input trace file (-i ).\n"; + return false; + } + return true; +} + +struct KernelArgSpec { + cl_uint arg_index = 0; + size_t arg_size = 0; + std::string arg_kind; + uint64_t mem_id = 0; + uint64_t sampler_id = 0; + std::vector value_bytes; +}; + +struct MutableConfigSpec { + uint64_t mutable_cmd_id = 0; + cl_uint work_dim = 0; + std::vector gwo; + std::vector gws; + std::vector lws; + std::vector args; +}; + +struct EmulatedCommand { + std::string api_name; + uint64_t queue_id = 0; + uint64_t kernel_id = 0; + uint64_t mutable_cmd_id = 0; + uint64_t src_mem_id = 0; + uint64_t dst_mem_id = 0; + uint64_t mem_id = 0; + size_t src_offset = 0; + size_t dst_offset = 0; + size_t offset = 0; + size_t size = 0; + std::vector src_origin; + std::vector dst_origin; + std::vector origin; + std::vector region; + size_t src_row_pitch = 0; + size_t src_slice_pitch = 0; + size_t dst_row_pitch = 0; + size_t dst_slice_pitch = 0; + std::vector pattern; + cl_uint work_dim = 1; + std::vector gwo; + std::vector gws; + std::vector lws; + std::map kernel_args_snapshot; +}; + +struct DispatchMetric { + uint64_t dispatch_id = 0; + uint64_t call_id = 0; + std::string label; + std::string gws; + std::string lws; + std::vector durations_ns; +}; + +struct TraceData { + std::vector primary_events; + std::map> aux_by_call_id; + std::map> source_chunks_by_prog; + std::map> il_chunks_by_prog; + std::map> bin_chunks_by_prog; +}; + +std::vector parse_size_list(const std::string &str) +{ + std::vector u64 = clkp::string_to_uint64_list(str); + std::vector out(u64.size()); + for (size_t i = 0; i < u64.size(); ++i) { + out[i] = static_cast(u64[i]); + } + return out; +} + +KernelArgSpec parse_kernel_arg_from_event(const clkp::ParsedEvent &ev) +{ + KernelArgSpec spec; + spec.arg_index = static_cast(ev.get_uint("arg_index", 0)); + spec.arg_size = static_cast(ev.get_uint("arg_size", 0)); + spec.arg_kind = ev.get_string("arg_kind", "VALUE"); + spec.mem_id = ev.get_uint("mem_id", 0); + spec.sampler_id = ev.get_uint("sampler_id", 0); + spec.value_bytes = clkp::hex_to_bytes(ev.get_string("value_hex", "")); + return spec; +} + +bool is_command_recording_call(const std::string &name) +{ + static const std::set kCmdRecordingNames = { + "clCommandBarrierWithWaitListKHR", + "clCommandCopyBufferKHR", + "clCommandCopyBufferRectKHR", + "clCommandCopyBufferToImageKHR", + "clCommandCopyImageKHR", + "clCommandCopyImageToBufferKHR", + "clCommandFillBufferKHR", + "clCommandFillImageKHR", + "clCommandNDRangeKernelKHR", + }; + return kCmdRecordingNames.count(name) > 0; +} + +bool is_setup_call(const std::string &name) +{ + static const std::set kSetupNames = { + "clCreateContext", + "clCreateContextFromType", + "clCreateCommandQueue", + "clCreateCommandQueueWithProperties", + "clCreateProgramWithSource", + "clCreateProgramWithIL", + "clCreateProgramWithBinary", + "clBuildProgram", + "clCompileProgram", + "clLinkProgram", + "clSetProgramSpecializationConstant", + "clCreateKernel", + "clCreateKernelsInProgram", + "clCloneKernel", + "clCreateBuffer", + "clCreateBufferWithProperties", + "clCreateSubBuffer", + "clCreateImage", + "clCreateImageWithProperties", + "clCreateImage2D", + "clCreateImage3D", + "clCreateSampler", + "clCreateSamplerWithProperties", + "clCreateCommandBufferKHR", + "clFinalizeCommandBufferKHR", + }; + return kSetupNames.count(name) > 0 || is_command_recording_call(name); +} + +bool is_execution_call(const std::string &name) +{ + static const std::set kExecNames = { + "clSetKernelArg", + "clEnqueueNDRangeKernel", + "clEnqueueTask", + "clEnqueueReadBuffer", + "clEnqueueWriteBuffer", + "clEnqueueReadBufferRect", + "clEnqueueWriteBufferRect", + "clEnqueueCopyBuffer", + "clEnqueueCopyBufferRect", + "clEnqueueFillBuffer", + "clEnqueueReadImage", + "clEnqueueWriteImage", + "clEnqueueCopyImage", + "clEnqueueFillImage", + "clEnqueueCopyImageToBuffer", + "clEnqueueCopyBufferToImage", + "clEnqueueMapBuffer", + "clEnqueueMapImage", + "clEnqueueUnmapMemObject", + "clEnqueueMigrateMemObjects", + "clEnqueueMarkerWithWaitList", + "clEnqueueBarrierWithWaitList", + "clEnqueueMarker", + "clEnqueueBarrier", + "clEnqueueWaitForEvents", + "clFlush", + "clFinish", + "clWaitForEvents", + "clCreateUserEvent", + "clSetUserEventStatus", + "clEnqueueCommandBufferKHR", + "clUpdateMutableCommandsKHR", + }; + return kExecNames.count(name) > 0; +} + +bool is_internal_or_gpu_slice(const std::string &name) +{ + return name == "writeProgramOnDisk" || name == "clkp-callback" || name == "clkp_wait" + || name.rfind("clkp_p", 0) == 0 || name.rfind("clkp_cmdbuf_", 0) == 0; +} + +void classify_instant_event(clkp::ParsedEvent &&ev, TraceData &td) +{ + if (ev.name == "clCreateProgramWithSource-args") { + td.source_chunks_by_prog[ev.get_string("program")].push_back(std::move(ev)); + } else if (ev.name == "clCreateProgramWithIL-args") { + td.il_chunks_by_prog[ev.get_string("program")].push_back(std::move(ev)); + } else if (ev.name == "clCreateProgramWithBinary-args") { + td.bin_chunks_by_prog[ev.get_string("program")].push_back(std::move(ev)); + } else if (ev.name == "clCreateKernelsInProgram-kernel" || ev.name == "clUpdateMutableCommandsKHR-config" + || ev.name == "clUpdateMutableCommandsKHR-arg") { + td.aux_by_call_id[ev.get_uint("call_id", 0)].push_back(std::move(ev)); + } +} + +void parse_trace_into_replay_stream(const std::vector &trace_bytes, TraceData &td) +{ + clkp::TraceIncrementState inc_state; + uint64_t synthetic_call_counter = 1; + perfetto::protos::pbzero::Trace_Decoder trace_dec(trace_bytes.data(), trace_bytes.size()); + size_t pkt_idx = 0; + + for (auto pkt_it = trace_dec.packet(); pkt_it; ++pkt_it, ++pkt_idx) { + protozero::ConstBytes pkt_bytes = *pkt_it; + perfetto::protos::pbzero::TracePacket_Decoder pkt_dec(pkt_bytes.data, pkt_bytes.size); + inc_state.update_from_packet(pkt_dec); + + clkp::ParsedEvent ev; + if (!inc_state.decode_track_event(pkt_dec, pkt_idx, ev) || !ev.has_category(CLKP_PERFETTO_CATEGORY)) { + continue; + } + + if (ev.event_type == 3) { + classify_instant_event(std::move(ev), td); + } else if (ev.event_type == 1 && !is_internal_or_gpu_slice(ev.name)) { + if (!ev.has_key("call_id")) { + clkp::ParsedAnnotation ann; + ann.name = "call_id"; + ann.type = clkp::ParsedAnnotation::ValueType::kUint; + ann.uint_val = synthetic_call_counter++; + ev.annotations.push_back(std::move(ann)); + } else { + synthetic_call_counter = std::max(synthetic_call_counter, ev.get_uint("call_id") + 1); + } + td.primary_events.push_back(std::move(ev)); + } + } + + std::sort( + td.primary_events.begin(), td.primary_events.end(), [](const clkp::ParsedEvent &a, const clkp::ParsedEvent &b) { + return a.get_uint("call_id") < b.get_uint("call_id"); + }); +} + +bool select_platform_and_device(const RunnerOptions &opts, cl_platform_id &platform, cl_device_id &device, + std::string &platform_name, std::string &device_name) +{ + cl_uint num_platforms = 0; + if (clGetPlatformIDs(0, nullptr, &num_platforms) != CL_SUCCESS || num_platforms == 0) { + std::cerr << "Error: no OpenCL platforms found.\n"; + return false; + } + std::vector platforms(num_platforms); + clGetPlatformIDs(num_platforms, platforms.data(), nullptr); + if (opts.platform_index >= num_platforms) { + std::cerr << "Error: platform index " << opts.platform_index << " out of range (found " << num_platforms + << ").\n"; + return false; + } + platform = platforms[opts.platform_index]; + + cl_uint num_devices = 0; + if (clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 0, nullptr, &num_devices) != CL_SUCCESS || num_devices == 0) { + std::cerr << "Error: no OpenCL devices found on platform " << opts.platform_index << ".\n"; + return false; + } + std::vector devices(num_devices); + clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, num_devices, devices.data(), nullptr); + if (opts.device_index >= num_devices) { + std::cerr << "Error: device index " << opts.device_index << " out of range (found " << num_devices << ").\n"; + return false; + } + device = devices[opts.device_index]; + + char pname[256] = { 0 }; + char dname[256] = { 0 }; + clGetPlatformInfo(platform, CL_PLATFORM_NAME, sizeof(pname) - 1, pname, nullptr); + clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(dname) - 1, dname, nullptr); + platform_name = pname; + device_name = dname; + + std::cout << "[clkp-runner] Platform: " << platform_name << "\n"; + std::cout << "[clkp-runner] Device: " << device_name << "\n"; + return true; +} + +bool resolve_dispatch_window(const RunnerOptions &opts, const std::vector &primary_events, + uint64_t &window_start_call, uint64_t &window_end_call) +{ + window_start_call = 0; + window_end_call = UINT64_MAX; + if (!opts.has_start_dispatch && !opts.has_end_dispatch) { + return true; + } + + uint64_t min_c = UINT64_MAX; + uint64_t max_c = 0; + for (const auto &ev : primary_events) { + if (ev.has_key("dispatch_id")) { + uint64_t did = ev.get_uint("dispatch_id"); + if (did >= opts.start_dispatch && did <= opts.end_dispatch) { + uint64_t cid = ev.get_uint("call_id"); + min_c = std::min(min_c, cid); + max_c = std::max(max_c, cid); + } + } + } + if (min_c == UINT64_MAX) { + std::cerr << "Error: no dispatches found in requested dispatch range.\n"; + return false; + } + window_start_call = min_c; + window_end_call = max_c; + return true; +} + +class ReplayEngine { +public: + ReplayEngine(const RunnerOptions &opts, cl_platform_id platform, cl_device_id device, TraceData &td) + : opts_(opts) + , platform_(platform) + , device_(device) + , td_(td) + , host_scratch_buffer_(64 * 1024, 0) + { + ext_.load(platform_); + } + + ~ReplayEngine() { release_resources(); } + + bool run_setup_phase( + uint64_t window_start_call, uint64_t window_end_call, std::vector &hot_loop_events) + { + for (const auto &ev : td_.primary_events) { + uint64_t cid = ev.get_uint("call_id", 0); + if (cid > window_end_call) { + continue; + } + if (opts_.verbose && is_setup_call(ev.name)) { + std::cout << "[Setup] call_id=" << cid << " " << ev.name << "\n"; + } + if (!dispatch_setup_event(ev, cid, window_start_call, window_end_call, hot_loop_events)) { + return false; + } + } + + for (auto &[qid, q] : queues_) { + clFinish(q); + } + + if (opts_.use_command_buffer && !build_promoted_command_buffer(hot_loop_events)) { + return false; + } + return true; + } + + bool run_iterations(const std::vector &hot_loop_events, + std::map &metrics, std::vector &iter_total_ns) + { + uint32_t total_iters = opts_.warmup_iterations + opts_.iterations; + iter_total_ns.assign(opts_.iterations, 0); + + for (uint32_t iter = 0; iter < total_iters; ++iter) { + bool is_measured = (iter >= opts_.warmup_iterations); + uint32_t measured_idx = is_measured ? (iter - opts_.warmup_iterations) : 0; + if (!run_single_iteration(hot_loop_events, is_measured, measured_idx, metrics, iter_total_ns)) { + return false; + } + } + return true; + } + +private: + const RunnerOptions &opts_; + cl_platform_id platform_; + cl_device_id device_; + ExtensionFunctions ext_; + TraceData &td_; + + std::map contexts_; + std::map queues_; + std::map programs_; + std::map programs_by_name_; + std::map kernels_; + std::map, cl_kernel> kernels_by_prog_and_name_; + std::map mems_; + std::map mem_sizes_; + std::map samplers_; + std::map cmdbufs_; + std::map cmdbuf_default_queue_; + std::map mutable_cmds_; + std::map> sync_points_; + std::map> current_kernel_args_; + std::map> emulated_cmdbufs_; + std::map active_maps_; + std::vector host_scratch_buffer_; + + cl_command_buffer_khr promoted_cmdbuf_ = nullptr; + cl_command_queue promoted_queue_ = nullptr; + + cl_context get_default_context() + { + if (!contexts_.empty()) { + return contexts_.begin()->second; + } + cl_int err = CL_SUCCESS; + cl_context ctx = clCreateContext(nullptr, 1, &device_, nullptr, nullptr, &err); + contexts_[1] = ctx; + return ctx; + } + + cl_context get_context_or_default(uint64_t ctx_id) + { + return contexts_.count(ctx_id) ? contexts_[ctx_id] : get_default_context(); + } + + cl_command_queue get_default_queue() + { + if (!queues_.empty()) { + return queues_.begin()->second; + } + cl_context ctx = get_default_context(); + cl_int err = CL_SUCCESS; + const cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_PROFILING_ENABLE, 0 }; + cl_command_queue q = clCreateCommandQueueWithProperties(ctx, device_, props, &err); + queues_[1] = q; + return q; + } + + cl_command_queue get_queue_or_default(uint64_t q_id) + { + return queues_.count(q_id) ? queues_[q_id] : get_default_queue(); + } + + cl_program resolve_program(uint64_t prog_id, const std::string &prog_str) + { + return programs_.count(prog_id) ? programs_[prog_id] : programs_by_name_[prog_str]; + } + + cl_kernel resolve_kernel(uint64_t kern_id, const std::string &prog_str, const std::string &kname) + { + return kernels_.count(kern_id) ? kernels_[kern_id] : kernels_by_prog_and_name_[{ prog_str, kname }]; + } + + cl_int apply_kernel_arg(cl_kernel k, const KernelArgSpec &spec) + { + if (!k) { + return CL_INVALID_KERNEL; + } + if (spec.arg_kind == "LOCAL") { + return clSetKernelArg(k, spec.arg_index, spec.arg_size, nullptr); + } + if (spec.arg_kind == "MEM") { + cl_mem m = mems_.count(spec.mem_id) ? mems_[spec.mem_id] : nullptr; + return clSetKernelArg(k, spec.arg_index, sizeof(cl_mem), &m); + } + if (spec.arg_kind == "SAMPLER") { + cl_sampler s = samplers_.count(spec.sampler_id) ? samplers_[spec.sampler_id] : nullptr; + return clSetKernelArg(k, spec.arg_index, sizeof(cl_sampler), &s); + } + const void *ptr = spec.value_bytes.empty() ? nullptr : spec.value_bytes.data(); + return clSetKernelArg(k, spec.arg_index, spec.arg_size, ptr); + } + + bool has_explicit_build_event(uint64_t prog_id, const std::string &prog_str) const + { + for (const auto &other : td_.primary_events) { + bool is_build = (other.name == "clBuildProgram" || other.name == "clCompileProgram"); + bool matches_prog = (other.get_uint("program_id") == prog_id || other.get_string("program") == prog_str); + if (is_build && matches_prog) { + return true; + } + } + return false; + } + + void register_program(uint64_t prog_id, const std::string &prog_str, cl_program prog, bool build_if_legacy) + { + if (prog_id != 0) { + programs_[prog_id] = prog; + } + if (!prog_str.empty()) { + programs_by_name_[prog_str] = prog; + } + if (build_if_legacy && !has_explicit_build_event(prog_id, prog_str)) { + clBuildProgram(prog, 1, &device_, nullptr, nullptr, nullptr); + } + } + + bool setup_create_context(const clkp::ParsedEvent &ev) + { + uint64_t ctx_id = ev.get_uint("context_id", 1); + cl_int err = CL_SUCCESS; + cl_context ctx = clCreateContext(nullptr, 1, &device_, nullptr, nullptr, &err); + if (err != CL_SUCCESS || !ctx) { + std::cerr << "Error: clCreateContext failed (" << err << ")\n"; + return false; + } + contexts_[ctx_id] = ctx; + return true; + } + + bool setup_create_command_queue(const clkp::ParsedEvent &ev) + { + uint64_t ctx_id = ev.get_uint("context_id", 1); + uint64_t q_id = ev.get_uint("queue_id", 1); + uint64_t props_bits = ev.get_uint("properties", 0) | CL_QUEUE_PROFILING_ENABLE; + cl_context ctx = get_context_or_default(ctx_id); + const cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, static_cast(props_bits), 0 }; + cl_int err = CL_SUCCESS; + cl_command_queue q = clCreateCommandQueueWithProperties(ctx, device_, props, &err); + if (err != CL_SUCCESS || !q) { + std::cerr << "Error: clCreateCommandQueue failed (" << err << ")\n"; + return false; + } + queues_[q_id] = q; + return true; + } + + bool setup_create_program_with_source(const clkp::ParsedEvent &ev) + { + uint64_t ctx_id = ev.get_uint("context_id", 1); + uint64_t prog_id = ev.get_uint("program_id", 0); + std::string prog_str = ev.get_string("program", ""); + cl_context ctx = get_context_or_default(ctx_id); + + std::map> strings_map; + auto it = td_.source_chunks_by_prog.find(prog_str); + if (it != td_.source_chunks_by_prog.end()) { + for (const auto &chunk_ev : it->second) { + strings_map[chunk_ev.get_uint("string_index", 0)][chunk_ev.get_uint("chunk_index", 0)] + = chunk_ev.get_string("string", ""); + } + } + + std::vector full_strings; + for (const auto &[s_idx, chunks] : strings_map) { + std::string combined; + for (const auto &[c_idx, part] : chunks) { + combined += part; + } + full_strings.push_back(std::move(combined)); + } + + std::vector c_strs; + std::vector lengths; + for (const auto &s : full_strings) { + c_strs.push_back(s.c_str()); + lengths.push_back(s.size()); + } + + cl_int err = CL_SUCCESS; + cl_program prog + = clCreateProgramWithSource(ctx, static_cast(c_strs.size()), c_strs.data(), lengths.data(), &err); + if (err != CL_SUCCESS || !prog) { + std::cerr << "Error: clCreateProgramWithSource failed (" << err << ") for " << prog_str << "\n"; + return false; + } + register_program(prog_id, prog_str, prog, true); + return true; + } + + bool setup_create_program_with_il(const clkp::ParsedEvent &ev) + { + uint64_t ctx_id = ev.get_uint("context_id", 1); + uint64_t prog_id = ev.get_uint("program_id", 0); + std::string prog_str = ev.get_string("program", ""); + cl_context ctx = get_context_or_default(ctx_id); + + std::map chunks; + auto it = td_.il_chunks_by_prog.find(prog_str); + if (it != td_.il_chunks_by_prog.end()) { + for (const auto &chunk_ev : it->second) { + if (chunk_ev.has_key("il_hex")) { + chunks[chunk_ev.get_uint("chunk_index", 0)] = chunk_ev.get_string("il_hex", ""); + } + } + } + std::string full_hex; + for (const auto &[c_idx, part] : chunks) { + full_hex += part; + } + std::vector il_bytes = clkp::hex_to_bytes(full_hex); + + cl_int err = CL_SUCCESS; + cl_program prog = clCreateProgramWithIL(ctx, il_bytes.data(), il_bytes.size(), &err); + if (err != CL_SUCCESS || !prog) { + std::cerr << "Error: clCreateProgramWithIL failed (" << err << ") for " << prog_str << "\n"; + return false; + } + register_program(prog_id, prog_str, prog, true); + return true; + } + + bool setup_create_program_with_binary(const clkp::ParsedEvent &ev) + { + uint64_t ctx_id = ev.get_uint("context_id", 1); + uint64_t prog_id = ev.get_uint("program_id", 0); + std::string prog_str = ev.get_string("program", ""); + cl_context ctx = get_context_or_default(ctx_id); + + std::map chunks; + auto it = td_.bin_chunks_by_prog.find(prog_str); + if (it != td_.bin_chunks_by_prog.end()) { + for (const auto &chunk_ev : it->second) { + if (chunk_ev.get_uint("device_index", 0) == 0) { + chunks[chunk_ev.get_uint("chunk_index", 0)] = chunk_ev.get_string("binary_hex", ""); + } + } + } + std::string full_hex; + for (const auto &[c_idx, part] : chunks) { + full_hex += part; + } + std::vector bin_bytes = clkp::hex_to_bytes(full_hex); + size_t len = bin_bytes.size(); + const unsigned char *bin_ptr = bin_bytes.data(); + cl_int bin_status = CL_SUCCESS; + cl_int err = CL_SUCCESS; + cl_program prog = clCreateProgramWithBinary(ctx, 1, &device_, &len, &bin_ptr, &bin_status, &err); + if (err != CL_SUCCESS || !prog) { + std::cerr << "Error: clCreateProgramWithBinary failed (" << err << ") for " << prog_str << "\n"; + return false; + } + register_program(prog_id, prog_str, prog, false); + return true; + } + + bool setup_specialization_constant(const clkp::ParsedEvent &ev) + { + cl_program prog = resolve_program(ev.get_uint("program_id", 0), ev.get_string("program")); + if (prog) { + cl_uint spec_id = static_cast(ev.get_uint("spec_id", 0)); + std::vector val = clkp::hex_to_bytes(ev.get_string("value_hex", "")); + clSetProgramSpecializationConstant(prog, spec_id, val.size(), val.data()); + } + return true; + } + + bool setup_build_program(const clkp::ParsedEvent &ev) + { + std::string prog_str = ev.get_string("program", ""); + cl_program prog = resolve_program(ev.get_uint("program_id", 0), prog_str); + std::string options = ev.get_string("options", ""); + cl_int err = clBuildProgram(prog, 1, &device_, options.empty() ? nullptr : options.c_str(), nullptr, nullptr); + if (err != CL_SUCCESS) { + size_t log_size = 0; + clGetProgramBuildInfo(prog, device_, CL_PROGRAM_BUILD_LOG, 0, nullptr, &log_size); + std::string build_log(log_size, '\0'); + if (log_size > 0) { + clGetProgramBuildInfo(prog, device_, CL_PROGRAM_BUILD_LOG, log_size, build_log.data(), nullptr); + } + std::cerr << "Error: clBuildProgram failed (" << err << ") for " << prog_str << ":\n" << build_log << "\n"; + return false; + } + return true; + } + + bool setup_compile_program(const clkp::ParsedEvent &ev) + { + cl_program prog = resolve_program(ev.get_uint("program_id", 0), ev.get_string("program")); + std::string options = ev.get_string("options", ""); + std::vector hdr_ids = clkp::string_to_uint64_list(ev.get_string("header_program_ids")); + std::vector hdr_names = clkp::joined_to_string_list(ev.get_string("header_include_names"), ';'); + size_t count = std::min(hdr_ids.size(), hdr_names.size()); + std::vector hdr_progs(count); + std::vector hdr_cstrs(count); + for (size_t i = 0; i < count; ++i) { + hdr_progs[i] = programs_[hdr_ids[i]]; + hdr_cstrs[i] = hdr_names[i].c_str(); + } + cl_int err = clCompileProgram(prog, 1, &device_, options.empty() ? nullptr : options.c_str(), + static_cast(count), count ? hdr_progs.data() : nullptr, count ? hdr_cstrs.data() : nullptr, + nullptr, nullptr); + if (err != CL_SUCCESS) { + std::cerr << "Error: clCompileProgram failed (" << err << ")\n"; + return false; + } + return true; + } + + bool setup_link_program(const clkp::ParsedEvent &ev) + { + cl_context ctx = get_context_or_default(ev.get_uint("context_id", 1)); + uint64_t prog_id = ev.get_uint("program_id", 0); + std::string prog_str = ev.get_string("program", ""); + std::string options = ev.get_string("options", ""); + std::vector in_ids = clkp::string_to_uint64_list(ev.get_string("input_program_ids")); + std::vector in_progs; + for (uint64_t id : in_ids) { + if (programs_.count(id)) { + in_progs.push_back(programs_[id]); + } + } + cl_int err = CL_SUCCESS; + cl_program linked = clLinkProgram(ctx, 1, &device_, options.empty() ? nullptr : options.c_str(), + static_cast(in_progs.size()), in_progs.data(), nullptr, nullptr, &err); + if (err != CL_SUCCESS || !linked) { + std::cerr << "Error: clLinkProgram failed (" << err << ")\n"; + return false; + } + register_program(prog_id, prog_str, linked, false); + return true; + } + + bool setup_create_kernel(const clkp::ParsedEvent &ev) + { + std::string prog_str = ev.get_string("program", ""); + uint64_t kern_id = ev.get_uint("kernel_id", 0); + std::string kname = ev.get_string("kernel_name", ""); + cl_program prog = resolve_program(ev.get_uint("program_id", 0), prog_str); + cl_int err = CL_SUCCESS; + cl_kernel k = clCreateKernel(prog, kname.c_str(), &err); + if (err != CL_SUCCESS || !k) { + std::cerr << "Error: clCreateKernel('" << kname << "') failed (" << err << ")\n"; + return false; + } + kernels_[kern_id] = k; + kernels_by_prog_and_name_[{ prog_str, kname }] = k; + return true; + } + + bool setup_create_kernels_in_program(const clkp::ParsedEvent &ev, uint64_t cid) + { + std::string prog_str = ev.get_string("program", ""); + cl_program prog = resolve_program(ev.get_uint("program_id", 0), prog_str); + for (const auto &kev : td_.aux_by_call_id[cid]) { + if (kev.name != "clCreateKernelsInProgram-kernel") { + continue; + } + uint64_t kern_id = kev.get_uint("kernel_id", 0); + std::string kname = kev.get_string("kernel_name", ""); + cl_int err = CL_SUCCESS; + cl_kernel k = clCreateKernel(prog, kname.c_str(), &err); + if (err == CL_SUCCESS && k) { + kernels_[kern_id] = k; + kernels_by_prog_and_name_[{ prog_str, kname }] = k; + } + } + return true; + } + + bool setup_clone_kernel(const clkp::ParsedEvent &ev) + { + uint64_t src_id = ev.get_uint("src_kernel_id", 0); + uint64_t dst_id = ev.get_uint("kernel_id", 0); + if (kernels_.count(src_id)) { + cl_int err = CL_SUCCESS; + cl_kernel cloned = clCloneKernel(kernels_[src_id], &err); + if (err == CL_SUCCESS && cloned) { + kernels_[dst_id] = cloned; + current_kernel_args_[dst_id] = current_kernel_args_[src_id]; + } + } + return true; + } + + bool setup_create_buffer(const clkp::ParsedEvent &ev) + { + cl_context ctx = get_context_or_default(ev.get_uint("context_id", 1)); + uint64_t mem_id = ev.get_uint("mem_id", 0); + cl_mem_flags flags = static_cast(ev.get_uint("flags", CL_MEM_READ_WRITE)); + flags &= ~(CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR); + if ((flags & (CL_MEM_READ_WRITE | CL_MEM_READ_ONLY | CL_MEM_WRITE_ONLY)) == 0) { + flags |= CL_MEM_READ_WRITE; + } + size_t size = std::max(static_cast(ev.get_uint("size", 4096)), 4); + cl_int err = CL_SUCCESS; + cl_mem buf = clCreateBuffer(ctx, flags, size, nullptr, &err); + if (err != CL_SUCCESS || !buf) { + std::cerr << "Error: clCreateBuffer(size=" << size << ") failed (" << err << ")\n"; + return false; + } + mems_[mem_id] = buf; + mem_sizes_[mem_id] = size; + + uint32_t zero_pattern = 0; + clEnqueueFillBuffer( + get_default_queue(), buf, &zero_pattern, sizeof(zero_pattern), 0, size, 0, nullptr, nullptr); + return true; + } + + bool setup_create_sub_buffer(const clkp::ParsedEvent &ev) + { + uint64_t parent_id = ev.get_uint("parent_mem_id", 0); + uint64_t mem_id = ev.get_uint("mem_id", 0); + if (!mems_.count(parent_id)) { + return true; + } + cl_mem_flags flags = static_cast(ev.get_uint("flags", CL_MEM_READ_WRITE)); + flags &= ~(CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR); + cl_buffer_region region = { + static_cast(ev.get_uint("origin", 0)), + static_cast(ev.get_uint("size", 0)), + }; + cl_int err = CL_SUCCESS; + cl_mem sub = clCreateSubBuffer(mems_[parent_id], flags, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + if (err == CL_SUCCESS && sub) { + mems_[mem_id] = sub; + mem_sizes_[mem_id] = region.size; + } + return true; + } + + bool setup_create_image(const clkp::ParsedEvent &ev) + { + cl_context ctx = get_context_or_default(ev.get_uint("context_id", 1)); + uint64_t mem_id = ev.get_uint("mem_id", 0); + cl_mem_flags flags = static_cast(ev.get_uint("flags", CL_MEM_READ_WRITE)); + flags &= ~(CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR); + cl_image_format fmt = { + static_cast(ev.get_uint("channel_order", CL_RGBA)), + static_cast(ev.get_uint("channel_data_type", CL_FLOAT)), + }; + cl_image_desc desc; + memset(&desc, 0, sizeof(desc)); + desc.image_type = static_cast(ev.get_uint("image_type", CL_MEM_OBJECT_IMAGE2D)); + desc.image_width = static_cast(ev.get_uint("width", 1)); + desc.image_height = static_cast(ev.get_uint("height", 1)); + desc.image_depth = static_cast(ev.get_uint("depth", 1)); + desc.image_array_size = static_cast(ev.get_uint("array_size", 0)); + desc.num_mip_levels = static_cast(ev.get_uint("num_mip_levels", 0)); + desc.num_samples = static_cast(ev.get_uint("num_samples", 0)); + uint64_t parent_id = ev.get_uint("parent_mem_id", 0); + if (parent_id != 0 && mems_.count(parent_id)) { + desc.mem_object = mems_[parent_id]; + desc.image_row_pitch = static_cast(ev.get_uint("row_pitch", 0)); + desc.image_slice_pitch = static_cast(ev.get_uint("slice_pitch", 0)); + } + + cl_int err = CL_SUCCESS; + cl_mem img = clCreateImage(ctx, flags, &fmt, &desc, nullptr, &err); + if (err != CL_SUCCESS || !img) { + std::cerr << "Error: clCreateImage failed (" << err << ")\n"; + return false; + } + mems_[mem_id] = img; + return true; + } + + bool setup_create_sampler(const clkp::ParsedEvent &ev) + { + cl_context ctx = get_context_or_default(ev.get_uint("context_id", 1)); + uint64_t samp_id = ev.get_uint("sampler_id", 0); + cl_bool norm = static_cast(ev.get_uint("normalized_coords", CL_TRUE)); + cl_addressing_mode addr = static_cast(ev.get_uint("addressing_mode", CL_ADDRESS_CLAMP)); + cl_filter_mode filt = static_cast(ev.get_uint("filter_mode", CL_FILTER_NEAREST)); + cl_int err = CL_SUCCESS; + cl_sampler s = clCreateSampler(ctx, norm, addr, filt, &err); + if (err == CL_SUCCESS && s) { + samplers_[samp_id] = s; + } + return true; + } + + bool setup_create_command_buffer(const clkp::ParsedEvent &ev) + { + uint64_t cb_id = ev.get_uint("cmdbuf_id", 0); + std::vector q_ids = clkp::string_to_uint64_list(ev.get_string("queue_ids")); + std::vector q_vec; + for (uint64_t qid : q_ids) { + if (queues_.count(qid)) { + q_vec.push_back(queues_[qid]); + } + } + if (q_vec.empty()) { + q_vec.push_back(get_default_queue()); + } + cmdbuf_default_queue_[cb_id] = q_ids.empty() ? 1 : q_ids[0]; + + if (opts_.emulate_command_buffer || !ext_.supports_command_buffer()) { + emulated_cmdbufs_[cb_id] = {}; + return true; + } + + std::vector raw_props = clkp::string_to_uint64_list(ev.get_string("properties")); + std::vector props(raw_props.begin(), raw_props.end()); + if (!props.empty()) { + props.push_back(0); + } + cl_int err = CL_SUCCESS; + cl_command_buffer_khr cb = ext_.clCreateCommandBufferKHR( + static_cast(q_vec.size()), q_vec.data(), props.empty() ? nullptr : props.data(), &err); + if (err != CL_SUCCESS || !cb) { + std::cerr << "Error: clCreateCommandBufferKHR failed (" << err << ")\n"; + return false; + } + cmdbufs_[cb_id] = cb; + return true; + } + + cl_int invoke_native_command_recording(const clkp::ParsedEvent &ev, cl_command_buffer_khr cb, + cl_command_queue explicit_q, const std::vector &wait_sps, cl_sync_point_khr *out_sp, + cl_mutable_command_khr *out_mut) + { + const std::string &name = ev.name; + cl_uint num_wait = static_cast(wait_sps.size()); + const cl_sync_point_khr *wait_ptr = wait_sps.empty() ? nullptr : wait_sps.data(); + + if (name == "clCommandNDRangeKernelKHR") { + cl_kernel k = kernels_[ev.get_uint("kernel_id", 0)]; + cl_uint work_dim = static_cast(ev.get_uint("work_dim", 1)); + auto gwo = parse_size_list(ev.get_string("gwo")); + auto gws = parse_size_list(ev.get_string("gws")); + auto lws = parse_size_list(ev.get_string("lws")); + auto raw_props = clkp::string_to_uint64_list(ev.get_string("properties")); + std::vector props(raw_props.begin(), raw_props.end()); + if (!props.empty()) { + props.push_back(0); + } + const cl_command_properties_khr *props_ptr = props.empty() ? nullptr : props.data(); + cl_int err = ext_.clCommandNDRangeKernelKHR(cb, explicit_q, props_ptr, k, work_dim, + gwo.empty() ? nullptr : gwo.data(), gws.data(), lws.empty() ? nullptr : lws.data(), num_wait, wait_ptr, + out_sp, out_mut); + if (err == CL_INVALID_WORK_GROUP_SIZE && opts_.allow_lws_fallback) { + err = ext_.clCommandNDRangeKernelKHR(cb, explicit_q, props_ptr, k, work_dim, + gwo.empty() ? nullptr : gwo.data(), gws.data(), nullptr, num_wait, wait_ptr, out_sp, out_mut); + } + return err; + } + if (name == "clCommandCopyBufferKHR" && ext_.clCommandCopyBufferKHR) { + return ext_.clCommandCopyBufferKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], static_cast(ev.get_uint("src_offset")), + static_cast(ev.get_uint("dst_offset")), static_cast(ev.get_uint("size")), num_wait, + wait_ptr, out_sp, out_mut); + } + if (name == "clCommandCopyBufferRectKHR" && ext_.clCommandCopyBufferRectKHR) { + auto so = parse_size_list(ev.get_string("src_origin")); + auto do_ = parse_size_list(ev.get_string("dst_origin")); + auto reg = parse_size_list(ev.get_string("region")); + return ext_.clCommandCopyBufferRectKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], so.data(), do_.data(), reg.data(), + static_cast(ev.get_uint("src_row_pitch")), static_cast(ev.get_uint("src_slice_pitch")), + static_cast(ev.get_uint("dst_row_pitch")), static_cast(ev.get_uint("dst_slice_pitch")), + num_wait, wait_ptr, out_sp, out_mut); + } + if (name == "clCommandCopyBufferToImageKHR" && ext_.clCommandCopyBufferToImageKHR) { + auto do_ = parse_size_list(ev.get_string("dst_origin")); + auto reg = parse_size_list(ev.get_string("region")); + return ext_.clCommandCopyBufferToImageKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], static_cast(ev.get_uint("src_offset")), do_.data(), + reg.data(), num_wait, wait_ptr, out_sp, out_mut); + } + if (name == "clCommandCopyImageKHR" && ext_.clCommandCopyImageKHR) { + auto so = parse_size_list(ev.get_string("src_origin")); + auto do_ = parse_size_list(ev.get_string("dst_origin")); + auto reg = parse_size_list(ev.get_string("region")); + return ext_.clCommandCopyImageKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], so.data(), do_.data(), reg.data(), num_wait, wait_ptr, out_sp, + out_mut); + } + if (name == "clCommandCopyImageToBufferKHR" && ext_.clCommandCopyImageToBufferKHR) { + auto so = parse_size_list(ev.get_string("src_origin")); + auto reg = parse_size_list(ev.get_string("region")); + return ext_.clCommandCopyImageToBufferKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], so.data(), reg.data(), static_cast(ev.get_uint("dst_offset")), + num_wait, wait_ptr, out_sp, out_mut); + } + if (name == "clCommandFillBufferKHR" && ext_.clCommandFillBufferKHR) { + auto pat = clkp::hex_to_bytes(ev.get_string("pattern_hex")); + return ext_.clCommandFillBufferKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("mem_id")], pat.data(), + pat.size(), static_cast(ev.get_uint("offset")), static_cast(ev.get_uint("size")), + num_wait, wait_ptr, out_sp, out_mut); + } + if (name == "clCommandFillImageKHR" && ext_.clCommandFillImageKHR) { + auto col = clkp::hex_to_bytes(ev.get_string("fill_color_hex")); + auto orig = parse_size_list(ev.get_string("origin")); + auto reg = parse_size_list(ev.get_string("region")); + return ext_.clCommandFillImageKHR(cb, explicit_q, nullptr, mems_[ev.get_uint("mem_id")], col.data(), + orig.data(), reg.data(), num_wait, wait_ptr, out_sp, out_mut); + } + if (name == "clCommandBarrierWithWaitListKHR" && ext_.clCommandBarrierWithWaitListKHR) { + return ext_.clCommandBarrierWithWaitListKHR(cb, explicit_q, nullptr, num_wait, wait_ptr, out_sp, out_mut); + } + return CL_SUCCESS; + } + + void record_emulated_command(const clkp::ParsedEvent &ev, uint64_t cb_id, uint64_t q_id, uint64_t mut_id) + { + EmulatedCommand ecmd; + ecmd.api_name = ev.name; + ecmd.queue_id = q_id != 0 ? q_id : cmdbuf_default_queue_[cb_id]; + ecmd.kernel_id = ev.get_uint("kernel_id", 0); + ecmd.mutable_cmd_id = mut_id; + ecmd.src_mem_id = ev.get_uint("src_mem_id", 0); + ecmd.dst_mem_id = ev.get_uint("dst_mem_id", 0); + ecmd.mem_id = ev.get_uint("mem_id", 0); + ecmd.src_offset = static_cast(ev.get_uint("src_offset", 0)); + ecmd.dst_offset = static_cast(ev.get_uint("dst_offset", 0)); + ecmd.offset = static_cast(ev.get_uint("offset", 0)); + ecmd.size = static_cast(ev.get_uint("size", 0)); + ecmd.src_origin = parse_size_list(ev.get_string("src_origin")); + ecmd.dst_origin = parse_size_list(ev.get_string("dst_origin")); + ecmd.origin = parse_size_list(ev.get_string("origin")); + ecmd.region = parse_size_list(ev.get_string("region")); + ecmd.src_row_pitch = static_cast(ev.get_uint("src_row_pitch", 0)); + ecmd.src_slice_pitch = static_cast(ev.get_uint("src_slice_pitch", 0)); + ecmd.dst_row_pitch = static_cast(ev.get_uint("dst_row_pitch", 0)); + ecmd.dst_slice_pitch = static_cast(ev.get_uint("dst_slice_pitch", 0)); + ecmd.pattern = clkp::hex_to_bytes( + ev.has_key("pattern_hex") ? ev.get_string("pattern_hex") : ev.get_string("fill_color_hex")); + ecmd.work_dim = static_cast(ev.get_uint("work_dim", 1)); + ecmd.gwo = parse_size_list(ev.get_string("gwo")); + ecmd.gws = parse_size_list(ev.get_string("gws")); + ecmd.lws = parse_size_list(ev.get_string("lws")); + ecmd.kernel_args_snapshot = current_kernel_args_[ecmd.kernel_id]; + emulated_cmdbufs_[cb_id].push_back(std::move(ecmd)); + } + + bool setup_command_recording(const clkp::ParsedEvent &ev) + { + uint64_t cb_id = ev.get_uint("cmdbuf_id", 0); + uint64_t q_id = ev.get_uint("queue_id", 0); + uint64_t traced_ret_sp = ev.get_uint("ret_sync_point", 0); + uint64_t mut_id = ev.get_uint("mutable_cmd_id", 0); + + if (opts_.emulate_command_buffer || !ext_.supports_command_buffer() || !cmdbufs_.count(cb_id)) { + record_emulated_command(ev, cb_id, q_id, mut_id); + return true; + } + + cl_command_queue explicit_q = (q_id != 0 && queues_.count(q_id)) ? queues_[q_id] : nullptr; + std::vector wait_sps; + for (uint64_t sp : clkp::string_to_uint64_list(ev.get_string("wait_sync_points"))) { + auto &sp_map = sync_points_[cb_id]; + if (sp_map.count(static_cast(sp))) { + wait_sps.push_back(sp_map[static_cast(sp)]); + } + } + + cl_sync_point_khr out_sp = 0; + cl_mutable_command_khr out_mut = nullptr; + cl_int err = invoke_native_command_recording( + ev, cmdbufs_[cb_id], explicit_q, wait_sps, traced_ret_sp ? &out_sp : nullptr, mut_id ? &out_mut : nullptr); + if (err != CL_SUCCESS) { + std::cerr << "Error: " << ev.name << " failed (" << err << ")\n"; + return false; + } + if (traced_ret_sp != 0) { + sync_points_[cb_id][static_cast(traced_ret_sp)] = out_sp; + } + if (mut_id != 0 && out_mut != nullptr) { + mutable_cmds_[mut_id] = out_mut; + } + return true; + } + + bool setup_finalize_command_buffer(const clkp::ParsedEvent &ev) + { + uint64_t cb_id = ev.get_uint("cmdbuf_id", 0); + if (!opts_.emulate_command_buffer && ext_.supports_command_buffer() && cmdbufs_.count(cb_id)) { + cl_int err = ext_.clFinalizeCommandBufferKHR(cmdbufs_[cb_id]); + if (err != CL_SUCCESS) { + std::cerr << "Error: clFinalizeCommandBufferKHR failed (" << err << ")\n"; + return false; + } + } + return true; + } + + bool dispatch_setup_event(const clkp::ParsedEvent &ev, uint64_t cid, uint64_t window_start_call, + uint64_t window_end_call, std::vector &hot_loop_events) + { + const std::string &name = ev.name; + if (name == "clCreateContext" || name == "clCreateContextFromType") { + return setup_create_context(ev); + } + if (name == "clCreateCommandQueue" || name == "clCreateCommandQueueWithProperties") { + return setup_create_command_queue(ev); + } + if (name == "clCreateProgramWithSource") { + return setup_create_program_with_source(ev); + } + if (name == "clCreateProgramWithIL") { + return setup_create_program_with_il(ev); + } + if (name == "clCreateProgramWithBinary") { + return setup_create_program_with_binary(ev); + } + if (name == "clSetProgramSpecializationConstant") { + return setup_specialization_constant(ev); + } + if (name == "clBuildProgram") { + return setup_build_program(ev); + } + if (name == "clCompileProgram") { + return setup_compile_program(ev); + } + if (name == "clLinkProgram") { + return setup_link_program(ev); + } + if (name == "clCreateKernel") { + return setup_create_kernel(ev); + } + if (name == "clCreateKernelsInProgram") { + return setup_create_kernels_in_program(ev, cid); + } + if (name == "clCloneKernel") { + return setup_clone_kernel(ev); + } + if (name == "clCreateBuffer" || name == "clCreateBufferWithProperties") { + return setup_create_buffer(ev); + } + if (name == "clCreateSubBuffer") { + return setup_create_sub_buffer(ev); + } + if (name == "clCreateImage" || name == "clCreateImageWithProperties" || name == "clCreateImage2D" + || name == "clCreateImage3D") { + return setup_create_image(ev); + } + if (name == "clCreateSampler" || name == "clCreateSamplerWithProperties") { + return setup_create_sampler(ev); + } + if (name == "clSetKernelArg") { + uint64_t kern_id = ev.get_uint("kernel_id", 0); + KernelArgSpec spec = parse_kernel_arg_from_event(ev); + current_kernel_args_[kern_id][spec.arg_index] = spec; + if (kernels_.count(kern_id)) { + apply_kernel_arg(kernels_[kern_id], spec); + } + if (cid >= window_start_call && cid <= window_end_call) { + hot_loop_events.push_back(&ev); + } + return true; + } + if (name == "clCreateCommandBufferKHR") { + return setup_create_command_buffer(ev); + } + if (is_command_recording_call(name)) { + return setup_command_recording(ev); + } + if (name == "clFinalizeCommandBufferKHR") { + return setup_finalize_command_buffer(ev); + } + if (cid >= window_start_call && cid <= window_end_call && is_execution_call(name)) { + hot_loop_events.push_back(&ev); + } + return true; + } + + bool promote_event_to_command_buffer(const clkp::ParsedEvent &ev) + { + if (ev.name == "clSetKernelArg") { + uint64_t kern_id = ev.get_uint("kernel_id", 0); + if (kernels_.count(kern_id)) { + apply_kernel_arg(kernels_[kern_id], parse_kernel_arg_from_event(ev)); + } + return true; + } + if (ev.name == "clEnqueueNDRangeKernel") { + cl_kernel k + = resolve_kernel(ev.get_uint("kernel_id", 0), ev.get_string("program"), ev.get_string("kernel_name")); + cl_uint work_dim = static_cast(ev.get_uint("work_dim", 1)); + auto gwo = parse_size_list(ev.get_string("gwo")); + auto gws = parse_size_list(ev.get_string("gws")); + if (gws.empty()) { + gws = { static_cast(ev.get_uint("gidX", 1)), static_cast(ev.get_uint("gidY", 1)), + static_cast(ev.get_uint("gidZ", 1)) }; + work_dim = 3; + } + auto lws = parse_size_list(ev.get_string("lws")); + cl_int err = ext_.clCommandNDRangeKernelKHR(promoted_cmdbuf_, nullptr, nullptr, k, work_dim, + gwo.empty() ? nullptr : gwo.data(), gws.data(), lws.empty() ? nullptr : lws.data(), 0, nullptr, nullptr, + nullptr); + if (err == CL_INVALID_WORK_GROUP_SIZE && opts_.allow_lws_fallback) { + err = ext_.clCommandNDRangeKernelKHR(promoted_cmdbuf_, nullptr, nullptr, k, work_dim, + gwo.empty() ? nullptr : gwo.data(), gws.data(), nullptr, 0, nullptr, nullptr, nullptr); + } + if (err != CL_SUCCESS) { + std::cerr << "Error: clCommandNDRangeKernelKHR failed (" << err << ")\n"; + return false; + } + return true; + } + if (ev.name == "clEnqueueCopyBuffer" && ext_.clCommandCopyBufferKHR) { + ext_.clCommandCopyBufferKHR(promoted_cmdbuf_, nullptr, nullptr, mems_[ev.get_uint("src_mem_id")], + mems_[ev.get_uint("dst_mem_id")], static_cast(ev.get_uint("src_offset")), + static_cast(ev.get_uint("dst_offset")), static_cast(ev.get_uint("size")), 0, nullptr, + nullptr, nullptr); + } else if (ev.name == "clEnqueueFillBuffer" && ext_.clCommandFillBufferKHR) { + auto pat = clkp::hex_to_bytes(ev.get_string("pattern_hex")); + ext_.clCommandFillBufferKHR(promoted_cmdbuf_, nullptr, nullptr, mems_[ev.get_uint("mem_id")], pat.data(), + pat.size(), static_cast(ev.get_uint("offset")), static_cast(ev.get_uint("size")), 0, + nullptr, nullptr, nullptr); + } + return true; + } + + bool build_promoted_command_buffer(const std::vector &hot_loop_events) + { + if (!ext_.supports_command_buffer()) { + std::cerr << "Error: --use-command-buffer requested, but device does not support cl_khr_command_buffer.\n"; + return false; + } + promoted_queue_ = get_default_queue(); + cl_int err = CL_SUCCESS; + promoted_cmdbuf_ = ext_.clCreateCommandBufferKHR(1, &promoted_queue_, nullptr, &err); + if (err != CL_SUCCESS || !promoted_cmdbuf_) { + std::cerr << "Error: failed to create command buffer for --use-command-buffer (" << err << ").\n"; + return false; + } + for (const auto *ev_ptr : hot_loop_events) { + if (!promote_event_to_command_buffer(*ev_ptr)) { + return false; + } + } + err = ext_.clFinalizeCommandBufferKHR(promoted_cmdbuf_); + if (err != CL_SUCCESS) { + std::cerr << "Error: clFinalizeCommandBufferKHR failed (" << err << ")\n"; + return false; + } + return true; + } + + bool exec_ndrange_kernel(const clkp::ParsedEvent &ev, cl_command_queue q, const std::vector &wait_list, + uint64_t ret_ev_id, std::map &live_events, + std::vector> &iter_prof_events, std::map &metrics) + { + uint64_t cid = ev.get_uint("call_id", 0); + uint64_t did = ev.get_uint("dispatch_id", cid); + std::string prog_str = ev.get_string("program", ""); + std::string kname = ev.get_string("kernel_name", ""); + cl_kernel k = resolve_kernel(ev.get_uint("kernel_id", 0), prog_str, kname); + cl_uint work_dim = static_cast(ev.get_uint("work_dim", 0)); + auto gwo = parse_size_list(ev.get_string("gwo")); + auto gws = parse_size_list(ev.get_string("gws")); + auto lws = parse_size_list(ev.get_string("lws")); + if (gws.empty()) { + gws = { static_cast(ev.get_uint("gidX", 1)), static_cast(ev.get_uint("gidY", 1)), + static_cast(ev.get_uint("gidZ", 1)) }; + if (work_dim == 0) { + work_dim = (gws[2] > 1) ? 3 : ((gws[1] > 1) ? 2 : 1); + } + } + + cl_event prof_ev = nullptr; + cl_int err = clEnqueueNDRangeKernel(q, k, work_dim, gwo.empty() ? nullptr : gwo.data(), gws.data(), + lws.empty() ? nullptr : lws.data(), static_cast(wait_list.size()), + wait_list.empty() ? nullptr : wait_list.data(), &prof_ev); + if (err == CL_INVALID_WORK_GROUP_SIZE && opts_.allow_lws_fallback) { + err = clEnqueueNDRangeKernel(q, k, work_dim, gwo.empty() ? nullptr : gwo.data(), gws.data(), nullptr, + static_cast(wait_list.size()), wait_list.empty() ? nullptr : wait_list.data(), &prof_ev); + } + if (err != CL_SUCCESS) { + std::cerr << "Error: clEnqueueNDRangeKernel('" << kname << "') failed (" << err << ")\n"; + return false; + } + if (ret_ev_id != 0) { + live_events[ret_ev_id] = prof_ev; + } + auto &m = metrics[did]; + m.dispatch_id = did; + m.call_id = cid; + m.label = prog_str + ":" + kname; + m.gws = ev.get_string("gws"); + m.lws = ev.get_string("lws", "NULL"); + iter_prof_events.push_back({ did, prof_ev }); + return true; + } + + std::vector parse_mutable_configs(uint64_t cid) + { + std::vector cfg_specs; + for (const auto &aev : td_.aux_by_call_id[cid]) { + if (aev.name == "clUpdateMutableCommandsKHR-config") { + MutableConfigSpec cs; + cs.mutable_cmd_id = aev.get_uint("mutable_cmd_id", 0); + cs.work_dim = static_cast(aev.get_uint("work_dim", 0)); + cs.gwo = parse_size_list(aev.get_string("gwo")); + cs.gws = parse_size_list(aev.get_string("gws")); + cs.lws = parse_size_list(aev.get_string("lws")); + cfg_specs.push_back(std::move(cs)); + } else if (aev.name == "clUpdateMutableCommandsKHR-arg" && !cfg_specs.empty()) { + cfg_specs.back().args.push_back(parse_kernel_arg_from_event(aev)); + } + } + return cfg_specs; + } + + void apply_native_mutable_update(uint64_t cb_id, const MutableConfigSpec &cs) + { + if (!mutable_cmds_.count(cs.mutable_cmd_id)) { + return; + } + std::vector cl_args; + std::vector mem_holders(cs.args.size(), nullptr); + std::vector samp_holders(cs.args.size(), nullptr); + for (size_t i = 0; i < cs.args.size(); ++i) { + const auto &aspec = cs.args[i]; + cl_mutable_dispatch_arg_khr a = { aspec.arg_index, aspec.arg_size, nullptr }; + if (aspec.arg_kind == "MEM") { + mem_holders[i] = mems_[aspec.mem_id]; + a.arg_size = sizeof(cl_mem); + a.arg_value = &mem_holders[i]; + } else if (aspec.arg_kind == "SAMPLER") { + samp_holders[i] = samplers_[aspec.sampler_id]; + a.arg_size = sizeof(cl_sampler); + a.arg_value = &samp_holders[i]; + } else if (aspec.arg_kind != "LOCAL") { + a.arg_value = aspec.value_bytes.data(); + } + cl_args.push_back(a); + } + cl_mutable_dispatch_config_khr dcfg; + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.command = mutable_cmds_[cs.mutable_cmd_id]; + dcfg.num_args = static_cast(cl_args.size()); + dcfg.arg_list = cl_args.empty() ? nullptr : cl_args.data(); + dcfg.work_dim = cs.work_dim; + dcfg.global_work_offset = cs.gwo.empty() ? nullptr : cs.gwo.data(); + dcfg.global_work_size = cs.gws.empty() ? nullptr : cs.gws.data(); + dcfg.local_work_size = cs.lws.empty() ? nullptr : cs.lws.data(); + cl_command_buffer_update_type_khr utype = CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR; + const void *configs_arr[1] = { &dcfg }; + ext_.clUpdateMutableCommandsKHR(cmdbufs_[cb_id], 1, &utype, configs_arr); + } + + void apply_emulated_mutable_update(uint64_t cb_id, const MutableConfigSpec &cs) + { + for (auto &ecmd : emulated_cmdbufs_[cb_id]) { + if (ecmd.mutable_cmd_id != cs.mutable_cmd_id || cs.mutable_cmd_id == 0) { + continue; + } + if (!cs.gwo.empty()) { + ecmd.gwo = cs.gwo; + } + if (!cs.gws.empty()) { + ecmd.gws = cs.gws; + } + if (!cs.lws.empty()) { + ecmd.lws = cs.lws; + } + for (const auto &aspec : cs.args) { + ecmd.kernel_args_snapshot[aspec.arg_index] = aspec; + } + } + } + + void exec_update_mutable_commands(const clkp::ParsedEvent &ev) + { + uint64_t cid = ev.get_uint("call_id", 0); + uint64_t cb_id = ev.get_uint("cmdbuf_id", 0); + auto cfg_specs = parse_mutable_configs(cid); + + if (!opts_.emulate_command_buffer && ext_.supports_command_buffer() && cmdbufs_.count(cb_id) + && ext_.clUpdateMutableCommandsKHR) { + for (const auto &cs : cfg_specs) { + apply_native_mutable_update(cb_id, cs); + } + } else if (emulated_cmdbufs_.count(cb_id)) { + for (const auto &cs : cfg_specs) { + apply_emulated_mutable_update(cb_id, cs); + } + } + } + + bool exec_emulated_command_buffer(uint64_t cb_id, uint64_t did, uint64_t cid, cl_command_queue target_q, + std::vector> &iter_prof_events, std::map &metrics) + { + for (const auto &ecmd : emulated_cmdbufs_[cb_id]) { + if (ecmd.api_name == "clCommandNDRangeKernelKHR") { + cl_kernel k = kernels_[ecmd.kernel_id]; + for (const auto &[aidx, aspec] : ecmd.kernel_args_snapshot) { + apply_kernel_arg(k, aspec); + } + cl_event prof_ev = nullptr; + cl_int err + = clEnqueueNDRangeKernel(target_q, k, ecmd.work_dim, ecmd.gwo.empty() ? nullptr : ecmd.gwo.data(), + ecmd.gws.data(), ecmd.lws.empty() ? nullptr : ecmd.lws.data(), 0, nullptr, &prof_ev); + if (err == CL_INVALID_WORK_GROUP_SIZE && opts_.allow_lws_fallback) { + err = clEnqueueNDRangeKernel(target_q, k, ecmd.work_dim, + ecmd.gwo.empty() ? nullptr : ecmd.gwo.data(), ecmd.gws.data(), nullptr, 0, nullptr, &prof_ev); + } + if (err != CL_SUCCESS) { + std::cerr << "Error: emulated clCommandNDRangeKernelKHR failed (" << err << ")\n"; + return false; + } + auto &m = metrics[did]; + m.dispatch_id = did; + m.call_id = cid; + m.label = "emulated_cmdbuf_" + std::to_string(cb_id); + iter_prof_events.push_back({ did, prof_ev }); + } else if (ecmd.api_name == "clCommandCopyBufferKHR") { + clEnqueueCopyBuffer(target_q, mems_[ecmd.src_mem_id], mems_[ecmd.dst_mem_id], ecmd.src_offset, + ecmd.dst_offset, ecmd.size, 0, nullptr, nullptr); + } else if (ecmd.api_name == "clCommandFillBufferKHR") { + clEnqueueFillBuffer(target_q, mems_[ecmd.mem_id], ecmd.pattern.data(), ecmd.pattern.size(), ecmd.offset, + ecmd.size, 0, nullptr, nullptr); + } else if (ecmd.api_name == "clCommandBarrierWithWaitListKHR") { + clEnqueueBarrierWithWaitList(target_q, 0, nullptr, nullptr); + } + } + return true; + } + + bool exec_command_buffer(const clkp::ParsedEvent &ev, const std::vector &wait_list, uint64_t ret_ev_id, + std::map &live_events, std::vector> &iter_prof_events, + std::map &metrics) + { + uint64_t cid = ev.get_uint("call_id", 0); + uint64_t did = ev.get_uint("dispatch_id", cid); + uint64_t cb_id = ev.get_uint("cmdbuf_id", 0); + std::vector q_vec; + for (uint64_t qid : clkp::string_to_uint64_list(ev.get_string("queue_ids"))) { + if (queues_.count(qid)) { + q_vec.push_back(queues_[qid]); + } + } + + if (!opts_.emulate_command_buffer && ext_.supports_command_buffer() && cmdbufs_.count(cb_id)) { + cl_event prof_ev = nullptr; + cl_int err = ext_.clEnqueueCommandBufferKHR(static_cast(q_vec.size()), + q_vec.empty() ? nullptr : q_vec.data(), cmdbufs_[cb_id], static_cast(wait_list.size()), + wait_list.empty() ? nullptr : wait_list.data(), &prof_ev); + if (err != CL_SUCCESS) { + std::cerr << "Error: clEnqueueCommandBufferKHR failed (" << err << ")\n"; + return false; + } + if (ret_ev_id != 0) { + live_events[ret_ev_id] = prof_ev; + } + auto &m = metrics[did]; + m.dispatch_id = did; + m.call_id = cid; + m.label = "cmdbuf_" + std::to_string(cb_id); + iter_prof_events.push_back({ did, prof_ev }); + return true; + } + + if (emulated_cmdbufs_.count(cb_id)) { + cl_command_queue target_q = !q_vec.empty() ? q_vec[0] : get_queue_or_default(cmdbuf_default_queue_[cb_id]); + return exec_emulated_command_buffer(cb_id, did, cid, target_q, iter_prof_events, metrics); + } + return true; + } + + void exec_auxiliary_queue_command(const clkp::ParsedEvent &ev, cl_command_queue q, + const std::vector &wait_list, uint64_t ret_ev_id, std::map &live_events) + { + const std::string &name = ev.name; + cl_uint num_wait = static_cast(wait_list.size()); + const cl_event *wait_ptr = wait_list.empty() ? nullptr : wait_list.data(); + cl_event out_ev = nullptr; + cl_event *out_ptr = ret_ev_id ? &out_ev : nullptr; + + if (name == "clEnqueueCopyBuffer") { + clEnqueueCopyBuffer(q, mems_[ev.get_uint("src_mem_id")], mems_[ev.get_uint("dst_mem_id")], + static_cast(ev.get_uint("src_offset")), static_cast(ev.get_uint("dst_offset")), + static_cast(ev.get_uint("size")), num_wait, wait_ptr, out_ptr); + } else if (name == "clEnqueueFillBuffer") { + auto pat = clkp::hex_to_bytes(ev.get_string("pattern_hex")); + clEnqueueFillBuffer(q, mems_[ev.get_uint("mem_id")], pat.data(), pat.size(), + static_cast(ev.get_uint("offset")), static_cast(ev.get_uint("size")), num_wait, + wait_ptr, out_ptr); + } else if (name == "clEnqueueWriteBuffer" || name == "clEnqueueReadBuffer") { + size_t sz = static_cast(ev.get_uint("size", 0)); + size_t off = static_cast(ev.get_uint("offset", 0)); + uint64_t mid = ev.get_uint("mem_id", 0); + if (mems_.count(mid) && sz > 0) { + if (host_scratch_buffer_.size() < sz) { + host_scratch_buffer_.resize(sz, 0); + } + if (name == "clEnqueueWriteBuffer") { + clEnqueueWriteBuffer( + q, mems_[mid], CL_FALSE, off, sz, host_scratch_buffer_.data(), num_wait, wait_ptr, out_ptr); + } else { + clEnqueueReadBuffer( + q, mems_[mid], CL_FALSE, off, sz, host_scratch_buffer_.data(), num_wait, wait_ptr, out_ptr); + } + } + } else if (name == "clEnqueueMapBuffer") { + uint64_t mid = ev.get_uint("mem_id", 0); + uint64_t map_id = ev.get_uint("map_id", 0); + if (mems_.count(mid)) { + cl_int err = CL_SUCCESS; + void *ptr = clEnqueueMapBuffer(q, mems_[mid], CL_TRUE, + static_cast(ev.get_uint("map_flags", CL_MAP_READ)), + static_cast(ev.get_uint("offset", 0)), static_cast(ev.get_uint("size", 0)), + num_wait, wait_ptr, out_ptr, &err); + if (ptr && map_id != 0) { + active_maps_[map_id] = ptr; + } + } + } else if (name == "clEnqueueUnmapMemObject") { + uint64_t mid = ev.get_uint("mem_id", 0); + uint64_t map_id = ev.get_uint("map_id", 0); + if (mems_.count(mid) && active_maps_.count(map_id)) { + clEnqueueUnmapMemObject(q, mems_[mid], active_maps_[map_id], num_wait, wait_ptr, out_ptr); + active_maps_.erase(map_id); + } + } else if (name == "clEnqueueBarrierWithWaitList") { + clEnqueueBarrierWithWaitList(q, num_wait, wait_ptr, out_ptr); + } else if (name == "clEnqueueMarkerWithWaitList") { + clEnqueueMarkerWithWaitList(q, num_wait, wait_ptr, out_ptr); + } else if (name == "clFlush") { + clFlush(q); + } else if (name == "clFinish") { + clFinish(q); + } else if (name == "clWaitForEvents" && !wait_list.empty()) { + clWaitForEvents(num_wait, wait_ptr); + } + + if (ret_ev_id != 0 && out_ev != nullptr) { + live_events[ret_ev_id] = out_ev; + } + } + + bool execute_hot_loop_event(const clkp::ParsedEvent &ev, std::map &live_events, + std::vector> &iter_prof_events, std::map &metrics) + { + const std::string &name = ev.name; + cl_command_queue q = get_queue_or_default(ev.get_uint("queue_id", 1)); + std::vector wait_list; + for (uint64_t wid : clkp::string_to_uint64_list(ev.get_string("wait_events"))) { + if (live_events.count(wid) && live_events[wid] != nullptr) { + wait_list.push_back(live_events[wid]); + } + } + uint64_t ret_ev_id = ev.get_uint("ret_event_id", 0); + + if (name == "clSetKernelArg") { + uint64_t kern_id = ev.get_uint("kernel_id", 0); + KernelArgSpec spec = parse_kernel_arg_from_event(ev); + current_kernel_args_[kern_id][spec.arg_index] = spec; + if (kernels_.count(kern_id)) { + apply_kernel_arg(kernels_[kern_id], spec); + } + return true; + } + if (name == "clEnqueueNDRangeKernel" || name == "clEnqueueTask") { + return exec_ndrange_kernel(ev, q, wait_list, ret_ev_id, live_events, iter_prof_events, metrics); + } + if (name == "clUpdateMutableCommandsKHR") { + exec_update_mutable_commands(ev); + return true; + } + if (name == "clEnqueueCommandBufferKHR") { + return exec_command_buffer(ev, wait_list, ret_ev_id, live_events, iter_prof_events, metrics); + } + exec_auxiliary_queue_command(ev, q, wait_list, ret_ev_id, live_events); + return true; + } + + void collect_iteration_profiling(bool is_measured, uint32_t measured_idx, + const std::vector> &iter_prof_events, + const std::map &live_events, std::map &metrics, + std::vector &iter_total_ns) + { + std::set released_events; + for (const auto &[did, ev_handle] : iter_prof_events) { + if (!ev_handle) { + continue; + } + if (is_measured) { + cl_ulong start_ns = 0; + cl_ulong end_ns = 0; + if (clGetEventProfilingInfo(ev_handle, CL_PROFILING_COMMAND_START, sizeof(start_ns), &start_ns, nullptr) + == CL_SUCCESS + && clGetEventProfilingInfo(ev_handle, CL_PROFILING_COMMAND_END, sizeof(end_ns), &end_ns, nullptr) + == CL_SUCCESS + && end_ns >= start_ns) { + uint64_t dur = static_cast(end_ns - start_ns); + metrics[did].durations_ns.push_back(dur); + iter_total_ns[measured_idx] += dur; + } + } + if (released_events.insert(ev_handle).second) { + clReleaseEvent(ev_handle); + } + } + for (const auto &[eid, ev_handle] : live_events) { + if (ev_handle && released_events.insert(ev_handle).second) { + clReleaseEvent(ev_handle); + } + } + } + + bool run_single_iteration(const std::vector &hot_loop_events, bool is_measured, + uint32_t measured_idx, std::map &metrics, std::vector &iter_total_ns) + { + std::vector> iter_prof_events; + std::map live_events; + + if (promoted_cmdbuf_ != nullptr) { + cl_event ev_out = nullptr; + cl_int err = ext_.clEnqueueCommandBufferKHR(0, nullptr, promoted_cmdbuf_, 0, nullptr, &ev_out); + if (err != CL_SUCCESS) { + std::cerr << "Error: clEnqueueCommandBufferKHR failed (" << err << ")\n"; + return false; + } + auto &m = metrics[0]; + m.dispatch_id = 0; + m.call_id = 0; + m.label = "promoted_cmdbuf"; + iter_prof_events.push_back({ 0, ev_out }); + clFinish(promoted_queue_); + } else { + for (const auto *ev_ptr : hot_loop_events) { + if (!execute_hot_loop_event(*ev_ptr, live_events, iter_prof_events, metrics)) { + return false; + } + } + for (auto &[qid, q] : queues_) { + clFinish(q); + } + } + + collect_iteration_profiling(is_measured, measured_idx, iter_prof_events, live_events, metrics, iter_total_ns); + return true; + } + + void release_resources() + { + if (promoted_cmdbuf_ && ext_.clReleaseCommandBufferKHR) { + ext_.clReleaseCommandBufferKHR(promoted_cmdbuf_); + } + for (auto &[id, cb] : cmdbufs_) { + if (cb && ext_.clReleaseCommandBufferKHR) { + ext_.clReleaseCommandBufferKHR(cb); + } + } + for (auto &[id, s] : samplers_) { + if (s) { + clReleaseSampler(s); + } + } + for (auto &[id, m] : mems_) { + if (m) { + clReleaseMemObject(m); + } + } + for (auto &[id, k] : kernels_) { + if (k) { + clReleaseKernel(k); + } + } + for (auto &[id, p] : programs_) { + if (p) { + clReleaseProgram(p); + } + } + for (auto &[id, q] : queues_) { + if (q) { + clReleaseCommandQueue(q); + } + } + for (auto &[id, c] : contexts_) { + if (c) { + clReleaseContext(c); + } + } + } +}; + +std::tuple compute_stats_us(std::vector vals) +{ + if (vals.empty()) { + return { 0.0, 0.0, 0.0, 0.0 }; + } + std::sort(vals.begin(), vals.end()); + double min_us = static_cast(vals.front()) / 1000.0; + double max_us = static_cast(vals.back()) / 1000.0; + double sum_ns = 0.0; + for (uint64_t v : vals) { + sum_ns += static_cast(v); + } + double avg_us = (sum_ns / static_cast(vals.size())) / 1000.0; + double med_us = (vals.size() % 2 == 1) + ? (static_cast(vals[vals.size() / 2]) / 1000.0) + : (static_cast(vals[vals.size() / 2 - 1] + vals[vals.size() / 2]) / 2000.0); + return { min_us, avg_us, med_us, max_us }; +} + +void print_profiling_report(const RunnerOptions &opts, const std::map &metrics, + const std::vector &iter_total_ns) +{ + std::cout << "\n=== Replay Profiling Results (warmup=" << opts.warmup_iterations + << ", iterations=" << opts.iterations << ") ===\n"; + std::cout << std::left << std::setw(12) << "DISPATCH" << std::setw(28) << "TARGET" << std::setw(14) << "MIN (us)" + << std::setw(14) << "AVG (us)" << std::setw(14) << "MEDIAN (us)" << "MAX (us)\n"; + + for (const auto &[did, m] : metrics) { + auto [min_us, avg_us, med_us, max_us] = compute_stats_us(m.durations_ns); + std::cout << std::left << std::setw(12) << m.dispatch_id << std::setw(28) << m.label << std::fixed + << std::setprecision(2) << std::setw(14) << min_us << std::setw(14) << avg_us << std::setw(14) + << med_us << std::setw(14) << max_us << "\n"; + } + + auto [tot_min, tot_avg, tot_med, tot_max] = compute_stats_us(iter_total_ns); + std::cout << "--------------------------------------------------------------------------------------------\n"; + std::cout << std::left << std::setw(12) << "TOTAL/ITER" << std::setw(28) + << ("(" + std::to_string(metrics.size()) + " dispatches)") << std::fixed << std::setprecision(2) + << std::setw(14) << tot_min << std::setw(14) << tot_avg << std::setw(14) << tot_med << std::setw(14) + << tot_max << "\n"; +} + +void write_json_report(const RunnerOptions &opts, const std::string &platform_name, const std::string &device_name, + const std::map &metrics, const std::vector &iter_total_ns) +{ + if (opts.json_output_path.empty()) { + return; + } + std::ofstream jf(opts.json_output_path, std::ios::out | std::ios::trunc); + if (!jf.is_open()) { + return; + } + + auto [tot_min, tot_avg, tot_med, tot_max] = compute_stats_us(iter_total_ns); + jf << "{\n" + << " \"platform\": \"" << platform_name << "\",\n" + << " \"device\": \"" << device_name << "\",\n" + << " \"warmup_iterations\": " << opts.warmup_iterations << ",\n" + << " \"iterations\": " << opts.iterations << ",\n" + << " \"total_iteration_us\": { \"min\": " << tot_min << ", \"avg\": " << tot_avg << ", \"median\": " << tot_med + << ", \"max\": " << tot_max << " },\n" + << " \"dispatches\": [\n"; + size_t idx = 0; + for (const auto &[did, m] : metrics) { + auto [min_us, avg_us, med_us, max_us] = compute_stats_us(m.durations_ns); + jf << " { \"dispatch_id\": " << m.dispatch_id << ", \"call_id\": " << m.call_id << ", \"label\": \"" + << m.label << "\", \"min_us\": " << min_us << ", \"avg_us\": " << avg_us << ", \"median_us\": " << med_us + << ", \"max_us\": " << max_us << " }" << (++idx < metrics.size() ? "," : "") << "\n"; + } + jf << " ]\n}\n"; +} + +} // namespace + +int main(int argc, char **argv) +{ + RunnerOptions opts; + if (!parse_args(argc, argv, opts)) { + print_usage(argv[0]); + return 1; + } + + std::vector trace_bytes; + if (!clkp::read_file_bytes(opts.input_path, trace_bytes)) { + std::cerr << "Error: failed to read trace file '" << opts.input_path << "'\n"; + return 1; + } + + TraceData trace_data; + parse_trace_into_replay_stream(trace_bytes, trace_data); + + cl_platform_id platform = nullptr; + cl_device_id device = nullptr; + std::string platform_name; + std::string device_name; + if (!select_platform_and_device(opts, platform, device, platform_name, device_name)) { + return 1; + } + + uint64_t window_start_call = 0; + uint64_t window_end_call = UINT64_MAX; + if (!resolve_dispatch_window(opts, trace_data.primary_events, window_start_call, window_end_call)) { + return 1; + } + + ReplayEngine engine(opts, platform, device, trace_data); + std::vector hot_loop_events; + if (!engine.run_setup_phase(window_start_call, window_end_call, hot_loop_events)) { + return 1; + } + + std::map metrics; + std::vector iter_total_ns; + if (!engine.run_iterations(hot_loop_events, metrics, iter_total_ns)) { + return 1; + } + + print_profiling_report(opts, metrics, iter_total_ns); + write_json_report(opts, platform_name, device_name, metrics, iter_total_ns); + return 0; +} diff --git a/src/opencl-kernel-profiler.cpp b/src/opencl-kernel-profiler.cpp deleted file mode 100644 index 7cbdc38..0000000 --- a/src/opencl-kernel-profiler.cpp +++ /dev/null @@ -1,676 +0,0 @@ -// Copyright 2023 The OpenCL Kernel Profiler authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#define CL_TARGET_OPENCL_VERSION 300 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef SPIRV_DISASSEMBLY -#include -#endif - -#ifdef CLKP_PERFETTO_AMALGAMATED -#include "perfetto.h" -#else -#include "perfetto/tracing.h" -#endif - -#ifdef __ANDROID__ -#include -#endif - -/*****************************************************************************/ -/* PERFETTO GLOBAL VARIABLES *************************************************/ -/*****************************************************************************/ - -#define CLKP_PERFETTO_CATEGORY "clkp" - -PERFETTO_DEFINE_CATEGORIES(perfetto::Category(CLKP_PERFETTO_CATEGORY).SetDescription("OpenCL Kernel Profiler Events")); - -PERFETTO_TRACK_EVENT_STATIC_STORAGE(); - -#ifdef BACKEND_INPROCESS -static perfetto::TracingSession *gTracingSession; -#endif - -/*****************************************************************************/ -/* ICD DISPATCH GLOBAL VARIABLES *********************************************/ -/*****************************************************************************/ - -static struct _cl_icd_dispatch dispatch; -static const struct _cl_icd_dispatch *tdispatch; - -/*****************************************************************************/ -/* MACROS ********************************************************************/ -/*****************************************************************************/ - -#ifdef __ANDROID__ -#define PRINT(message, ...) \ - do { \ - __android_log_print( \ - android_LogPriority::ANDROID_LOG_ERROR, "CLKP", " %s: " message "\n", __func__, ##__VA_ARGS__); \ - } while (0) -#else -#define PRINT(message, ...) \ - do { \ - fprintf(stderr, "[CLKP] %s: " message "\n", __func__, ##__VA_ARGS__); \ - } while (0) -#endif -#define CHECK(test, statement, message, ...) \ - do { \ - if (!(test)) { \ - PRINT(message, ##__VA_ARGS__); \ - statement; \ - } \ - } while (0) -#define CHECK_CL(err, statement, message, ...) CHECK((err) == CL_SUCCESS, statement, message, ##__VA_ARGS__) -#define CHECK_ALLOC(ptr, statement) CHECK(ptr != nullptr, statement, "allocation failed") - -/*****************************************************************************/ -/* CREATE KERNEL & PROGRAM ***************************************************/ -/*****************************************************************************/ - -static std::mutex g_lock; - -static void writeProgramOnDisk( - std::filesystem::path filename, cl_uint count, const char **data, const size_t *lengths, bool binary) -{ - auto dirname = filename.parent_path(); - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "writeProgramOnDisk", "dir", - perfetto::DynamicString((const char *)dirname.u8string().c_str()), "program", - perfetto::DynamicString((const char *)filename.filename().u8string().c_str())); - CHECK(std::filesystem::exists(dirname), return; - , "'%s' does not exist, could not write program on disk", dirname.c_str()); - std::ofstream file(filename, binary ? std::ios::binary | std::ios::trunc : std::ios::trunc); - CHECK(file.is_open(), return;, "Could not open '%s'", filename.c_str()); - for (unsigned i = 0; i < count; i++) { - size_t length = lengths == nullptr ? strlen(data[i]) : lengths[i]; - file.write(data[i], length); - } -} - -#ifdef SPIRV_DISASSEMBLY -static bool tryDisassembleSpirv( - const void *il, size_t length, const std::string &program_name, std::string &disassembly) -{ - const uint32_t *spirv_data = (const uint32_t *)(il); - size_t spirv_words = length / sizeof(uint32_t); - - // Check for SPIR-V magic number - if (spirv_words == 0 || *spirv_data != 0x07230203) { - return false; - } - - spvtools::SpirvTools tools(SPV_ENV_OPENCL_2_2); - CHECK(tools.Disassemble(spirv_data, spirv_words, &disassembly), return false; - , "Failed to disassemble SPIR-V for program %s", program_name.c_str()); - return true; -} -#endif - -static std::string get_program_str() -{ - static uint32_t program_number = 0; - return std::string("clkp_p") + std::to_string(program_number++); -} - -static char *get_kernel_dir() { return getenv("CLKP_KERNEL_DIR"); } - -static std::map program_to_string; -static cl_program clkp_clCreateProgramWithSource( - cl_context context, cl_uint count, const char **strings, const size_t *lengths, cl_int *errcode_ret) -{ - std::lock_guard lock(g_lock); - std::string program_str = get_program_str(); - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithSource", "program", perfetto::DynamicString(program_str), - "count", count); - - if (auto dir = get_kernel_dir()) { - writeProgramOnDisk(std::filesystem::path(dir) / (program_str + ".cl"), count, strings, lengths, false); - } - - cl_program program = tdispatch->clCreateProgramWithSource(context, count, strings, lengths, errcode_ret); - program_to_string[program] = program_str; - for (unsigned i = 0; i < count; i++) { - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithSource-args", "program", - perfetto::DynamicString(program_str), "string", perfetto::DynamicString(strings[i])); - } - return program; -} - -static cl_program clkp_clCreateProgramWithBinary(cl_context context, cl_uint num_devices, - const cl_device_id *device_list, const size_t *lengths, const unsigned char **binaries, cl_int *binary_status, - cl_int *errcode_ret) -{ - std::lock_guard lock(g_lock); - std::string program_str = get_program_str(); - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithBinary", "program", perfetto::DynamicString(program_str), - "num_devices", num_devices); - if (auto dir = get_kernel_dir()) { - for (unsigned i = 0; i < num_devices; i++) { - writeProgramOnDisk(std::filesystem::path(dir) / (program_str + "-" + std::to_string(i) + ".bin"), 1, - (const char **)&binaries[i], &lengths[i], true); - } - } - cl_program program = tdispatch->clCreateProgramWithBinary( - context, num_devices, device_list, lengths, binaries, binary_status, errcode_ret); - program_to_string[program] = program_str; - return program; -} - -static cl_program clkp_clCreateProgramWithIL(cl_context context, const void *il, size_t length, cl_int *errcode_ret) -{ - std::lock_guard lock(g_lock); - std::string program_str = get_program_str(); - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithIL", "program", - perfetto::DynamicString(program_str.c_str()), "length", length); - - std::string disassembly; - bool is_spirv = false; -#ifdef SPIRV_DISASSEMBLY - is_spirv = tryDisassembleSpirv(il, length, program_str, disassembly); - if (is_spirv) { - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateProgramWithIL-args", "program", - perfetto::DynamicString(program_str), "disassembly", perfetto::DynamicString(disassembly)); - } -#endif - - if (auto dir = get_kernel_dir()) { - std::filesystem::path filename = std::filesystem::path(dir) / program_str; - - // Write the raw IL binary - std::filesystem::path il_filename = filename.replace_extension(is_spirv ? "spv" : "il"); - const char *il_str = (const char *)il; - writeProgramOnDisk(il_filename, 1, &il_str, &length, true); - - if (is_spirv) { - // Write the disassembly - std::filesystem::path asm_filename = filename.replace_extension("spvasm"); - const char *disassembly_str = disassembly.c_str(); - size_t disassembly_length = disassembly.length(); - writeProgramOnDisk(asm_filename, 1, &disassembly_str, &disassembly_length, false); - } - } - - cl_program program = tdispatch->clCreateProgramWithIL(context, il, length, errcode_ret); - program_to_string[program] = program_str; - - return program; -} - -static std::map kernel_to_kernel_name; -static std::map kernel_to_program; -static cl_kernel clkp_clCreateKernel(cl_program program, const char *kernel_name, cl_int *errcode_ret) -{ - std::lock_guard lock(g_lock); - cl_kernel kernel = tdispatch->clCreateKernel(program, kernel_name, errcode_ret); - kernel_to_program[kernel] = program; - kernel_to_kernel_name[kernel] = std::string(kernel_name); - return kernel; -} - -/*****************************************************************************/ -/* ENQUEUE NDRANGE KERNEL ****************************************************/ -/*****************************************************************************/ - -struct callback_data { - cl_command_queue queue; - cl_event event; - size_t gidX, gidY, gidZ; - int64_t time_offset; - std::string kernel_name; - std::string program_string; -}; - -struct ThreadInfo { - std::mutex lock; - std::condition_variable cv; - std::queue callbacks; - bool stop; - int pending_callbacks; -}; - -static std::map queue_to_thread_info; -static std::map queue_to_thread; -static std::map queue_to_time_offset; - -static void callback(cl_event event, cl_int event_command_exec_status, void *user_data) -{ - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clkp-callback"); - - struct callback_data *data = (struct callback_data *)user_data; - assert(data != nullptr); - assert(event_command_exec_status == CL_COMPLETE); - cl_command_queue queue = data->queue; - - std::lock_guard lock(g_lock); - if (queue_to_thread_info.count(queue) == 0) { - // Queue was released before this callback fired. - // Clean up resources. - tdispatch->clReleaseEvent(data->event); - delete data; - return; - } - - ThreadInfo *thread_info = queue_to_thread_info[queue]; - { - std::lock_guard lock_ti(thread_info->lock); - thread_info->callbacks.push(data); - thread_info->cv.notify_all(); - } -} - -static callback_data *get_callback(ThreadInfo *thread_info) -{ - std::unique_lock lock(thread_info->lock); - while (thread_info->callbacks.empty()) { - if (thread_info->stop && thread_info->pending_callbacks == 0) { - return nullptr; - } - TRACE_EVENT_BEGIN(CLKP_PERFETTO_CATEGORY, "clkp_wait"); - thread_info->cv.wait(lock); - TRACE_EVENT_END(CLKP_PERFETTO_CATEGORY); - } - callback_data *data = thread_info->callbacks.front(); - thread_info->callbacks.pop(); - return data; -} - -static void trace_callback(callback_data *data) -{ - cl_command_queue queue = data->queue; - cl_event event = data->event; - size_t gidX = data->gidX, gidY = data->gidY, gidZ = data->gidZ; - int64_t time_offset = data->time_offset; - - cl_ulong start, end; - cl_int err; - err = tdispatch->clGetEventProfilingInfo(event, CL_PROFILING_COMMAND_START, sizeof(start), &start, nullptr); - CHECK_CL(err, tdispatch->clReleaseEvent(event); - return, "clGetEventProfilingInfo(CL_PROFILING_COMMAND_START) failed (%i)", err); - err = tdispatch->clGetEventProfilingInfo(event, CL_PROFILING_COMMAND_END, sizeof(end), &end, nullptr); - CHECK_CL(err, tdispatch->clReleaseEvent(event); - return, "clGetEventProfilingInfo(CL_PROFILING_COMMAND_END) failed (%i)", err); - if (end < start) { - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, perfetto::StaticString("INVALID_TIMESTAMPS"), - perfetto::Track((uintptr_t)queue), "start", start, "end", end); - tdispatch->clReleaseEvent(event); - return; - } - - std::string name = data->program_string + "-" + data->kernel_name + "-" + std::to_string(gidX) + "." - + std::to_string(gidY) + "." + std::to_string(gidZ); - - TRACE_EVENT_BEGIN(CLKP_PERFETTO_CATEGORY, perfetto::DynamicString(name), perfetto::Track((uintptr_t)queue), - (uint64_t)(start + time_offset), "program", perfetto::DynamicString(data->program_string), "kernel_name", - perfetto::DynamicString(data->kernel_name), "gidX", gidX, "gidY", gidY, "gidZ", gidZ); - TRACE_EVENT_END(CLKP_PERFETTO_CATEGORY, perfetto::Track((uintptr_t)queue), (uint64_t)(end + time_offset)); - - tdispatch->clReleaseEvent(event); -} - -static void queue_thread_function(ThreadInfo *thread_info) -{ -#ifdef __APPLE__ - pthread_setname_np("clkp"); -#else - pthread_setname_np(pthread_self(), "clkp"); -#endif - while (true) { - callback_data *data = get_callback(thread_info); - if (data == nullptr) { - return; - } - trace_callback(data); - { - std::lock_guard lock(thread_info->lock); - thread_info->pending_callbacks--; - thread_info->cv.notify_all(); - } - delete data; - } -} - -static cl_int clkp_clEnqueueNDRangeKernel(cl_command_queue command_queue, cl_kernel kernel, cl_uint work_dim, - const size_t *global_work_offset, const size_t *global_work_size, const size_t *local_work_size, - cl_uint num_events_in_wait_list, const cl_event *event_wait_list, cl_event *event) -{ - std::lock_guard lock(g_lock); - size_t gidX = work_dim > 0 ? global_work_size[0] : 1; - size_t gidY = work_dim > 1 ? global_work_size[1] : 1; - size_t gidZ = work_dim > 2 ? global_work_size[2] : 1; - - std::string kernel_name = "?"; - if (kernel_to_kernel_name.count(kernel)) { - kernel_name = kernel_to_kernel_name[kernel]; - } - - std::string program_string = "clkp_p?"; - if (kernel_to_program.count(kernel) && program_to_string.count(kernel_to_program[kernel])) { - program_string = program_to_string[kernel_to_program[kernel]]; - } - - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clEnqueueNDRangeKernel", "program", perfetto::DynamicString(program_string), - "kernel_name", perfetto::DynamicString(kernel_name), "gidX", gidX, "gidY", gidY, "gidZ", gidZ); - - bool event_is_null = event == nullptr; - cl_event local_event; - cl_event *event_to_use = event; - if (event_is_null) { - event_to_use = &local_event; - } - - cl_int err = tdispatch->clEnqueueNDRangeKernel(command_queue, kernel, work_dim, global_work_offset, - global_work_size, local_work_size, num_events_in_wait_list, event_wait_list, event_to_use); - - if (err != CL_SUCCESS) { - return err; - } - - ThreadInfo *thread_info = queue_to_thread_info[command_queue]; - { - std::lock_guard lock_ti(thread_info->lock); - thread_info->pending_callbacks++; - } - - struct callback_data *data = new (std::nothrow) callback_data(); - if (data == nullptr) { - { - std::lock_guard lock_ti(thread_info->lock); - thread_info->pending_callbacks--; - } - if (event_is_null) { - tdispatch->clReleaseEvent(local_event); - } - return CL_OUT_OF_HOST_MEMORY; - } - - data->queue = command_queue; - data->event = *event_to_use; - data->gidX = gidX; - data->gidY = gidY; - data->gidZ = gidZ; - data->time_offset = queue_to_time_offset[command_queue]; - data->kernel_name = kernel_name; - data->program_string = program_string; - - // Retain the event for the callback - tdispatch->clRetainEvent(*event_to_use); - - cl_int err_cb = tdispatch->clSetEventCallback(*event_to_use, CL_COMPLETE, callback, data); - if (err_cb != CL_SUCCESS) { - PRINT("clSetEventCallback failed (%i)", err_cb); - tdispatch->clReleaseEvent(*event_to_use); // Undo retain - delete data; - { - std::lock_guard lock_ti(thread_info->lock); - thread_info->pending_callbacks--; - } - if (event_is_null) { - tdispatch->clReleaseEvent(local_event); // Release the one we created - } - return err_cb; - } - - return CL_SUCCESS; -} - -/*****************************************************************************/ -/* CREATE COMMAND QUEUE ******************************************************/ -/*****************************************************************************/ - -static cl_int clkp_clReleaseCommandQueue(cl_command_queue command_queue) -{ - ThreadInfo *thread_info = nullptr; - std::thread thread_to_join; - { - std::lock_guard lock(g_lock); - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clReleaseCommandQueue"); - thread_info = queue_to_thread_info[command_queue]; - thread_to_join = std::move(queue_to_thread[command_queue]); - } - - auto ret = tdispatch->clReleaseCommandQueue(command_queue); - - if (thread_info) { - { - std::unique_lock lock_ti(thread_info->lock); - thread_info->stop = true; - thread_info->cv.notify_all(); - thread_info->cv.wait(lock_ti, [&] { return thread_info->pending_callbacks == 0; }); - } - if (thread_to_join.joinable()) { - thread_to_join.join(); - } - } - - { - std::lock_guard lock(g_lock); - queue_to_thread.erase(command_queue); - queue_to_thread_info.erase(command_queue); - queue_to_time_offset.erase(command_queue); - } - - delete thread_info; - - return ret; -} - -static cl_command_queue create_command_queue( - cl_context context, cl_device_id device, const cl_queue_properties *properties, cl_int *errcode_ret) -{ - std::lock_guard lock(g_lock); - std::vector properties_array; - bool cl_queue_properties_found = false; - if (properties) { - for (unsigned i = 0; properties[i] != 0; i += 2) { - cl_queue_properties key = properties[i]; - cl_queue_properties val = properties[i + 1]; - if (key == CL_QUEUE_PROPERTIES) { - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue-properties", "properties", val); - - val |= CL_QUEUE_PROFILING_ENABLE; - cl_queue_properties_found = true; - } - properties_array.push_back(key); - properties_array.push_back(val); - } - } - if (!cl_queue_properties_found) { - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue-properties-not-found"); - properties_array.push_back(CL_QUEUE_PROPERTIES); - properties_array.push_back(CL_QUEUE_PROFILING_ENABLE); - } - properties_array.push_back(0); - - auto command_queue - = tdispatch->clCreateCommandQueueWithProperties(context, device, properties_array.data(), errcode_ret); - - TRACE_EVENT_INSTANT(CLKP_PERFETTO_CATEGORY, - perfetto::DynamicString("clkp-queue_" + std::to_string((uintptr_t)command_queue)), - perfetto::Track((uintptr_t)command_queue)); - ThreadInfo *thread_info = new ThreadInfo(); - thread_info->stop = false; - thread_info->pending_callbacks = 0; - queue_to_thread_info[command_queue] = thread_info; - queue_to_thread.emplace(command_queue, [thread_info] { queue_thread_function(thread_info); }); - - int64_t offset = 0; - if (tdispatch->clGetDeviceAndHostTimer) { - cl_ulong device_timestamp, host_timestamp; - cl_int timer_err = tdispatch->clGetDeviceAndHostTimer(device, &device_timestamp, &host_timestamp); - if (timer_err == CL_SUCCESS) { - uint64_t perfetto_timestamp = perfetto::TrackEvent::GetTraceTimeNs(); - offset = perfetto_timestamp - device_timestamp; - } else { - PRINT("clGetDeviceAndHostTimer failed (%i)", timer_err); - } - } else { - PRINT("clGetDeviceAndHostTimer is not available"); - } - queue_to_time_offset[command_queue] = offset; - - return command_queue; -} - -static cl_command_queue clkp_clCreateCommandQueue( - cl_context context, cl_device_id device, cl_command_queue_properties properties, cl_int *errcode_ret) -{ - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueue"); - cl_queue_properties props[4] = { CL_QUEUE_PROPERTIES, properties, 0, 0 }; - return create_command_queue(context, device, props, errcode_ret); -} - -static cl_command_queue clkp_clCreateCommandQueueWithProperties( - cl_context context, cl_device_id device, const cl_queue_properties *properties, cl_int *errcode_ret) -{ - TRACE_EVENT(CLKP_PERFETTO_CATEGORY, "clCreateCommandQueueWithProperties"); - return create_command_queue(context, device, properties, errcode_ret); -} - -/*****************************************************************************/ -/* PERFETTO TRACE PARAMETERS *************************************************/ -/*****************************************************************************/ - -#ifdef BACKEND_INPROCESS -static const char *get_trace_dest() -{ - if (auto trace_dest = getenv("CLKP_TRACE_DEST")) { - return trace_dest; - } - return TRACE_DEST; -} - -static const uint32_t get_trace_max_size() -{ - if (auto trace_max_size = getenv("CLKP_TRACE_MAX_SIZE")) { - return atoi(trace_max_size); - } - return TRACE_MAX_SIZE; -} -#endif - -/*****************************************************************************/ -/* LAYER FUNCTIONS ***********************************************************/ -/*****************************************************************************/ - -CL_API_ENTRY cl_int CL_API_CALL clGetLayerInfo( - cl_layer_info param_name, size_t param_value_size, void *param_value, size_t *param_value_size_ret) -{ - switch (param_name) { - case CL_LAYER_API_VERSION: - if (param_value) { - if (param_value_size < sizeof(cl_layer_api_version)) - return CL_INVALID_VALUE; - *((cl_layer_api_version *)param_value) = CL_LAYER_API_VERSION_100; - } - if (param_value_size_ret) - *param_value_size_ret = sizeof(cl_layer_api_version); - break; - default: - return CL_INVALID_VALUE; - } - return CL_SUCCESS; -} - -CL_API_ENTRY cl_int CL_API_CALL clDeinitLayer() -{ -#ifdef BACKEND_INPROCESS - gTracingSession->StopBlocking(); - std::vector trace_data(gTracingSession->ReadTraceBlocking()); - delete gTracingSession; - - std::ofstream output; - output.open(get_trace_dest(), std::ios::out | std::ios::binary); - output.write(&trace_data[0], trace_data.size()); - output.close(); -#else - perfetto::TrackEvent::Flush(); -#endif - return 0; -} - -CL_API_ENTRY cl_int CL_API_CALL clInitLayerWithProperties(cl_uint num_entries, const cl_icd_dispatch *target_dispatch, - cl_uint *num_entries_out, const cl_icd_dispatch **layer_dispatch_ret, const cl_layer_properties *properties) -{ - std::lock_guard lock(g_lock); - if (!target_dispatch || !num_entries_out || !layer_dispatch_ret) - return CL_INVALID_VALUE; - if (num_entries < sizeof(dispatch) / sizeof(dispatch.clGetPlatformIDs)) - return CL_INVALID_VALUE; - - perfetto::TracingInitArgs args; -#ifdef BACKEND_INPROCESS - args.backends |= perfetto::kInProcessBackend; -#else - args.backends |= perfetto::kSystemBackend; -#endif - perfetto::Tracing::Initialize(args); - perfetto::TrackEvent::Register(); - -#ifdef BACKEND_INPROCESS - perfetto::protos::gen::TrackEventConfig track_event_cfg; - perfetto::TraceConfig cfg; - cfg.add_buffers()->set_size_kb(get_trace_max_size()); - auto *ds_cfg = cfg.add_data_sources()->mutable_config(); - ds_cfg->set_name("track_event"); - ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString()); - - gTracingSession = perfetto::Tracing::NewTrace().release(); - gTracingSession->Setup(cfg); - gTracingSession->StartBlocking(); -#endif - - const uint32_t max_retry = 100; - uint32_t retry = 0; - while ((retry++ < max_retry) && !TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - if (!TRACE_EVENT_CATEGORY_ENABLED(CLKP_PERFETTO_CATEGORY)) { - PRINT("perfetto category does not seem to be enabled"); - } - - memset(&dispatch, 0, sizeof(dispatch)); - dispatch.clCreateProgramWithSource = clkp_clCreateProgramWithSource; - dispatch.clCreateProgramWithBinary = clkp_clCreateProgramWithBinary; - dispatch.clCreateProgramWithIL = clkp_clCreateProgramWithIL; - dispatch.clCreateKernel = clkp_clCreateKernel; - dispatch.clEnqueueNDRangeKernel = clkp_clEnqueueNDRangeKernel; - dispatch.clCreateCommandQueue = clkp_clCreateCommandQueue; - dispatch.clCreateCommandQueueWithProperties = clkp_clCreateCommandQueueWithProperties; - dispatch.clReleaseCommandQueue = clkp_clReleaseCommandQueue; - - tdispatch = target_dispatch; - *layer_dispatch_ret = &dispatch; - *num_entries_out = sizeof(dispatch) / sizeof(dispatch.clGetPlatformIDs); - - return CL_SUCCESS; -} - -CL_API_ENTRY cl_int CL_API_CALL clInitLayer(cl_uint num_entries, const struct _cl_icd_dispatch *target_dispatch, - cl_uint *num_entries_out, const struct _cl_icd_dispatch **layer_dispatch_ret) -{ - return clInitLayerWithProperties(num_entries, target_dispatch, num_entries_out, layer_dispatch_ret, nullptr); -} diff --git a/tests/host.c b/tests/host.c index 398c479..a3eea65 100644 --- a/tests/host.c +++ b/tests/host.c @@ -12,15 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#define CL_TARGET_OPENCL_VERSION 220 +#define CL_TARGET_OPENCL_VERSION 300 +#define CL_ENABLE_BETA_EXTENSIONS #include +#include #include #include #include #include -void vector_inc(cl_device_id device, cl_context context, cl_command_queue command_queue, size_t buffer_size, - void *buffer, const char **source, const size_t *source_length, const size_t *global_work_size) +#include + +void vector_inc(cl_platform_id platform, cl_device_id device, cl_context context, cl_command_queue command_queue, + size_t buffer_size, void *buffer, const char **source, const size_t *source_length, const size_t *global_work_size, + bool use_cmdbuf) { // Check if source is SPIR-V binary (starts with SPIR-V magic number 0x07230203) bool is_spirv = (*source_length >= 4) && (((const uint32_t *)*source)[0] == 0x07230203); @@ -39,10 +44,37 @@ void vector_inc(cl_device_id device, cl_context context, cl_command_queue comman cl_mem cl_buffer = clCreateBuffer(context, CL_MEM_READ_WRITE, buffer_size, NULL, NULL); clSetKernelArg(kernel, 0, sizeof(cl_mem), &cl_buffer); - // Write buffer to device, execute kernel and read buffer from device - clEnqueueWriteBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); - clEnqueueNDRangeKernel(command_queue, kernel, 1, NULL, global_work_size, NULL, 0, NULL, NULL); - clEnqueueReadBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); + if (use_cmdbuf) { + clCreateCommandBufferKHR_fn pfn_clCreateCommandBufferKHR + = (clCreateCommandBufferKHR_fn)clGetExtensionFunctionAddressForPlatform( + platform, "clCreateCommandBufferKHR"); + clCommandNDRangeKernelKHR_fn pfn_clCommandNDRangeKernelKHR + = (clCommandNDRangeKernelKHR_fn)clGetExtensionFunctionAddressForPlatform( + platform, "clCommandNDRangeKernelKHR"); + clFinalizeCommandBufferKHR_fn pfn_clFinalizeCommandBufferKHR + = (clFinalizeCommandBufferKHR_fn)clGetExtensionFunctionAddressForPlatform( + platform, "clFinalizeCommandBufferKHR"); + clEnqueueCommandBufferKHR_fn pfn_clEnqueueCommandBufferKHR + = (clEnqueueCommandBufferKHR_fn)clGetExtensionFunctionAddressForPlatform( + platform, "clEnqueueCommandBufferKHR"); + clReleaseCommandBufferKHR_fn pfn_clReleaseCommandBufferKHR + = (clReleaseCommandBufferKHR_fn)clGetExtensionFunctionAddressForPlatform( + platform, "clReleaseCommandBufferKHR"); + + clEnqueueWriteBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); + cl_command_buffer_khr cmdbuf = pfn_clCreateCommandBufferKHR(1, &command_queue, NULL, NULL); + pfn_clCommandNDRangeKernelKHR(cmdbuf, NULL, NULL, kernel, 1, NULL, global_work_size, NULL, 0, NULL, NULL, NULL); + pfn_clFinalizeCommandBufferKHR(cmdbuf); + pfn_clEnqueueCommandBufferKHR(0, NULL, cmdbuf, 0, NULL, NULL); + clFinish(command_queue); + pfn_clReleaseCommandBufferKHR(cmdbuf); + clEnqueueReadBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); + } else { + // Write buffer to device, execute kernel and read buffer from device + clEnqueueWriteBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); + clEnqueueNDRangeKernel(command_queue, kernel, 1, NULL, global_work_size, NULL, 0, NULL, NULL); + clEnqueueReadBuffer(command_queue, cl_buffer, CL_BLOCKING, 0, buffer_size, buffer, 0, NULL, NULL); + } clReleaseMemObject(cl_buffer); clReleaseKernel(kernel); @@ -55,10 +87,16 @@ int main(int argc, char **argv) { printf("Starting OpenCL application\n"); - if (argc < 2) { - fprintf(stderr, - "At least 1 argument is expected. It should be the path(s) to the kernel source code or SPIR-V " - "binary\n"); + bool use_cmdbuf = false; + int first_file_idx = 1; + if (argc >= 2 && strcmp(argv[1], "--cmdbuf") == 0) { + use_cmdbuf = true; + first_file_idx = 2; + } + + if (argc <= first_file_idx) { + fprintf(stderr, "At least 1 file argument is expected. Usage: %s [--cmdbuf] ...\n", + argv[0]); return -2; } @@ -77,7 +115,7 @@ int main(int argc, char **argv) printf("CL_PLATFORM_NAME: %s\n", platform_name); // Process each file argument - for (int file_idx = 1; file_idx < argc; file_idx++) { + for (int file_idx = first_file_idx; file_idx < argc; file_idx++) { printf("Processing file: %s\n", argv[file_idx]); FILE *f_source = fopen(argv[file_idx], "rb"); // Open in binary mode to support both text and binary @@ -102,8 +140,8 @@ int main(int argc, char **argv) } size_t global_work_size = NB_ELEM; - vector_inc(device, context, command_queue, sizeof(buffer), buffer, (const char **)&source, &source_length, - &global_work_size); + vector_inc(platform, device, context, command_queue, sizeof(buffer), buffer, (const char **)&source, + &source_length, &global_work_size, use_cmdbuf); for (unsigned i = 0; i < NB_ELEM; i++) { if (buffer[i] != i + 43) { diff --git a/tests/test.sh b/tests/test.sh index 91fde8f..53775f9 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -20,6 +20,9 @@ set -xe SCRIPT_DIR="$(dirname $(realpath "${BASH_SOURCE[0]}"))" CLKP_HOST_TEST="$1" +BUILD_DIR="$(dirname $(dirname $(realpath "${CLKP_HOST_TEST}")))" +CLKP_EXTRACTOR="${BUILD_DIR}/clkp-extractor" +CLKP_RUNNER="${BUILD_DIR}/clkp-runner" EXPECTATION_FILE="${SCRIPT_DIR}/trace-expectation.txt" GPU_SRC_FILE="${SCRIPT_DIR}/gpu.cl" GPU_SRC_SPVASM_FILE="${SCRIPT_DIR}/gpu.spvasm" @@ -30,6 +33,10 @@ TRACE_PROCESSOR_SHELL=${TRACE_PROCESSOR_SHELL:-"trace_processor_shell"} TMP_DIR="$(mktemp -d)" KERNELS_DIR="${TMP_DIR}/kernels" TRACE_FILE="${TMP_DIR}/trace" +EXTRACTED_TRACE="${TMP_DIR}/extracted.trace" +WINDOWED_TRACE="${TMP_DIR}/windowed.trace" +CMDBUF_TRACE="${TMP_DIR}/cmdbuf.trace" +CMDBUF_EXTRACTED_TRACE="${TMP_DIR}/cmdbuf_extracted.trace" OUTPUT_FILE="${TMP_DIR}/output.txt" EXPECTATION_SORTED_FILE="${TMP_DIR}/trace-expectation.sorted" GPU_SPV_FILE="${TMP_DIR}/gpu.spv" @@ -75,3 +82,25 @@ cat "${OUTPUT_FILE}" grep -F "OpName %inc \"inc\"" ${OUTPUT_FILE} diff "${GPU_SPVASM_FILE}" "${KERNELS_DIR}/clkp_p1.spvasm" + +# Test clkp-extractor --list-dispatches +"${CLKP_EXTRACTOR}" --list-dispatches "${TRACE_FILE}" | tee "${OUTPUT_FILE}" +grep -F "clkp_p0" "${OUTPUT_FILE}" +grep -F "clkp_p1" "${OUTPUT_FILE}" + +# Test full trace extraction and multi-dispatch replay via clkp-runner +"${CLKP_EXTRACTOR}" "${TRACE_FILE}" -o "${EXTRACTED_TRACE}" +CLKP_TRACE_DEST="${TMP_DIR}/runner_full.trace" "${CLKP_RUNNER}" "${EXTRACTED_TRACE}" -m 1 -n 3 --json "${TMP_DIR}/replay.json" +grep -F "\"iterations\": 3" "${TMP_DIR}/replay.json" + +# Test windowed extraction (single dispatch) and replay +"${CLKP_EXTRACTOR}" "${TRACE_FILE}" --start-dispatch 0 --end-dispatch 0 -o "${WINDOWED_TRACE}" +CLKP_TRACE_DEST="${TMP_DIR}/runner_windowed.trace" "${CLKP_RUNNER}" "${WINDOWED_TRACE}" -m 1 -n 2 + +# Test cl_khr_command_buffer capture, extraction, and replay +CLKP_TRACE_DEST="${CMDBUF_TRACE}" "${CLKP_HOST_TEST}" --cmdbuf "${GPU_SRC_FILE}" +"${CLKP_EXTRACTOR}" --list-dispatches "${CMDBUF_TRACE}" | tee "${OUTPUT_FILE}" +grep -F "clEnqueueCommandBufferKHR" "${OUTPUT_FILE}" +"${CLKP_EXTRACTOR}" "${CMDBUF_TRACE}" -o "${CMDBUF_EXTRACTED_TRACE}" +CLKP_TRACE_DEST="${TMP_DIR}/runner_cmdbuf.trace" "${CLKP_RUNNER}" "${CMDBUF_EXTRACTED_TRACE}" -m 1 -n 2 +CLKP_TRACE_DEST="${TMP_DIR}/runner_cmdbuf_emu.trace" "${CLKP_RUNNER}" "${CMDBUF_EXTRACTED_TRACE}" -m 1 -n 2 --emulate-command-buffer diff --git a/tests/trace-expectation.txt b/tests/trace-expectation.txt index 9e7998c..adfc147 100644 --- a/tests/trace-expectation.txt +++ b/tests/trace-expectation.txt @@ -1,15 +1,26 @@ +"clBuildProgram" +"clCreateBuffer" +"clCreateCommandQueue-properties-not-found" "clCreateCommandQueueWithProperties" +"clCreateContext" +"clCreateKernel" "clCreateProgramWithIL" "clCreateProgramWithIL-args" "clCreateProgramWithSource" "clCreateProgramWithSource-args" "clEnqueueNDRangeKernel" +"clEnqueueReadBuffer" +"clEnqueueWriteBuffer" "clkp-callback" -"clkp_queue" "clkp_p0-inc-1024.1.1" "clkp_p1-inc-1024.1.1" -"name" -"clReleaseCommandQueue" +"clkp_queue" "clkp_wait" -"clCreateCommandQueue-properties-not-found" +"clReleaseCommandQueue" +"clReleaseContext" +"clReleaseKernel" +"clReleaseMemObject" +"clReleaseProgram" +"clSetKernelArg" +"name" "writeProgramOnDisk"