From 89a391152ed7474d41f4f521d7889ba22501ebf0 Mon Sep 17 00:00:00 2001 From: kimbochen <40498137+kimbochen@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:22:27 +0000 Subject: [PATCH] feat(experimental): package Mooncake H100 transfer traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Archive recorder and Slurm patches, compact results, bilingual documentation, and offline trace-analysis tools with reproducible validation evidence. 中文：打包 Mooncake H100 传输跟踪实验，包含记录器与 Slurm 补丁、精简结果、 中英文说明和离线分析工具，并保留可复现的验证证据。 --- docs/index.md | 1 + docs/index_zh.md | 1 + experimental/README.md | 2 + .../mooncake-transfer-traces/LICENSE-Mooncake | 202 +++ .../mooncake-transfer-traces/README.md | 190 +++ .../mooncake-transfer-traces/README_zh.md | 177 +++ .../mooncake-transfer-traces/SHA256SUMS | 58 + .../mooncake-transfer-traces/manifest.json | 129 ++ .../0001-tent-transfer-event-recorder.patch | 1283 +++++++++++++++++ .../patches/0002-tebench-slurm.patch | 506 +++++++ .../patches/0003-store-benchmark-slurm.patch | 440 ++++++ .../mooncake-transfer-traces/patches/series | 3 + .../results/tebench-1612.XD95we/commit.txt | 1 + .../results/tebench-1612.XD95we/read-1.jsonl | 15 + .../results/tebench-1612.XD95we/read-2.jsonl | 15 + .../results/tebench-1612.XD95we/read-3.jsonl | 15 + .../results/tebench-1612.XD95we/summary.csv | 31 + .../results/tebench-1612.XD95we/topology.txt | 4 + .../results/tebench-1612.XD95we/write-1.jsonl | 15 + .../results/tebench-1612.XD95we/write-2.jsonl | 15 + .../results/tebench-1612.XD95we/write-3.jsonl | 15 + .../tebench-1695.6UKR6R/analysis-job.out | 6 + .../bandwidth-comparison.png | Bin 0 -> 167024 bytes .../tebench-1695.6UKR6R/bandwidth-summary.csv | 31 + .../bandwidth-summary.json | 512 +++++++ .../results/tebench-1695.6UKR6R/commit.txt | 1 + .../results/tebench-1695.6UKR6R/endpoint.txt | 1 + .../host-slurm-h100-206-025.txt | 71 + .../host-slurm-h100-206-035.txt | 71 + .../results/tebench-1695.6UKR6R/job.log | 10 + .../tebench-1695.6UKR6R/read-1.command.txt | 1 + .../results/tebench-1695.6UKR6R/read-1.jsonl | 15 + .../results/tebench-1695.6UKR6R/read-1.log | 59 + .../tebench-1695.6UKR6R/read-2.command.txt | 1 + .../results/tebench-1695.6UKR6R/read-2.jsonl | 15 + .../results/tebench-1695.6UKR6R/read-2.log | 59 + .../tebench-1695.6UKR6R/read-3.command.txt | 1 + .../results/tebench-1695.6UKR6R/read-3.jsonl | 15 + .../results/tebench-1695.6UKR6R/read-3.log | 59 + .../results/tebench-1695.6UKR6R/slurm-job.txt | 30 + .../tebench-1695.6UKR6R/target-command.txt | 1 + .../results/tebench-1695.6UKR6R/target.log | 41 + .../results/tebench-1695.6UKR6R/topology.txt | 4 + .../tebench-1695.6UKR6R/trace-analysis.json | 998 +++++++++++++ .../tebench-1695.6UKR6R/trace-loss.png | Bin 0 -> 52754 bytes .../tebench-1695.6UKR6R/trace-windows.json | 1 + .../tebench-1695.6UKR6R/write-1.command.txt | 1 + .../results/tebench-1695.6UKR6R/write-1.jsonl | 15 + .../results/tebench-1695.6UKR6R/write-1.log | 59 + .../tebench-1695.6UKR6R/write-2.command.txt | 1 + .../results/tebench-1695.6UKR6R/write-2.jsonl | 15 + .../results/tebench-1695.6UKR6R/write-2.log | 59 + .../tebench-1695.6UKR6R/write-3.command.txt | 1 + .../results/tebench-1695.6UKR6R/write-3.jsonl | 15 + .../results/tebench-1695.6UKR6R/write-3.log | 59 + .../scripts/analyze_traces.py | 117 ++ .../scripts/extract_trace_windows.py | 112 ++ .../scripts/requirements.txt | 1 + .../validation/ctest.log | 283 ++++ .../validation/package-checks.txt | 4 + .../validation/rdma-smoke-job-1693.log | 7 + .../validation/regression-job-1694.log | 197 +++ 62 files changed, 6067 insertions(+) create mode 100644 experimental/mooncake-transfer-traces/LICENSE-Mooncake create mode 100644 experimental/mooncake-transfer-traces/README.md create mode 100644 experimental/mooncake-transfer-traces/README_zh.md create mode 100644 experimental/mooncake-transfer-traces/SHA256SUMS create mode 100644 experimental/mooncake-transfer-traces/manifest.json create mode 100644 experimental/mooncake-transfer-traces/patches/0001-tent-transfer-event-recorder.patch create mode 100644 experimental/mooncake-transfer-traces/patches/0002-tebench-slurm.patch create mode 100644 experimental/mooncake-transfer-traces/patches/0003-store-benchmark-slurm.patch create mode 100644 experimental/mooncake-transfer-traces/patches/series create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/commit.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-1.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-2.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-3.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/summary.csv create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/topology.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-1.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-2.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-3.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/analysis-job.out create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-comparison.png create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.csv create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.json create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/commit.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/endpoint.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-025.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-035.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/job.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/slurm-job.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target-command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/topology.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-analysis.json create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-loss.png create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-windows.json create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.log create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.command.txt create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.jsonl create mode 100644 experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.log create mode 100644 experimental/mooncake-transfer-traces/scripts/analyze_traces.py create mode 100644 experimental/mooncake-transfer-traces/scripts/extract_trace_windows.py create mode 100644 experimental/mooncake-transfer-traces/scripts/requirements.txt create mode 100644 experimental/mooncake-transfer-traces/validation/ctest.log create mode 100644 experimental/mooncake-transfer-traces/validation/package-checks.txt create mode 100644 experimental/mooncake-transfer-traces/validation/rdma-smoke-job-1693.log create mode 100644 experimental/mooncake-transfer-traces/validation/regression-job-1694.log diff --git a/docs/index.md b/docs/index.md index a57795adc7..536b3991f7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,6 +26,7 @@ This is the mandatory low-context router for InferenceX work. Pick the one page | [`documentation-procedures.md`](./documentation-procedures.md) / [`documentation-procedures_zh.md`](./documentation-procedures_zh.md) | Adding, translating, indexing, reviewing, and maintaining documentation | | [`PR_REVIEW_CHECKLIST.md`](./PR_REVIEW_CHECKLIST.md) / [`PR_REVIEW_CHECKLIST_zh.md`](./PR_REVIEW_CHECKLIST_zh.md) | CODEOWNER review and exact sign-off requirements | | [`DOCUMENTATION_PLAN.md`](./DOCUMENTATION_PLAN.md) / [`DOCUMENTATION_PLAN_zh.md`](./DOCUMENTATION_PLAN_zh.md) | Remaining documentation gaps, target information architecture, and rollout | +| [Mooncake transfer tracing](../experimental/mooncake-transfer-traces/README.md) / [中文](../experimental/mooncake-transfer-traces/README_zh.md) | Reproducing the experimental two-node H100 transfer sweep and interpreting trace loss | ## Authoritative references diff --git a/docs/index_zh.md b/docs/index_zh.md index 5c1b3da8bb..38a09646e0 100644 --- a/docs/index_zh.md +++ b/docs/index_zh.md @@ -26,6 +26,7 @@ | [`documentation-procedures.md`](./documentation-procedures.md) / [`documentation-procedures_zh.md`](./documentation-procedures_zh.md) | 新增、翻译、索引、审阅与维护文档 | | [`PR_REVIEW_CHECKLIST.md`](./PR_REVIEW_CHECKLIST.md) / [`PR_REVIEW_CHECKLIST_zh.md`](./PR_REVIEW_CHECKLIST_zh.md) | CODEOWNER 审阅与精确签署要求 | | [`DOCUMENTATION_PLAN.md`](./DOCUMENTATION_PLAN.md) / [`DOCUMENTATION_PLAN_zh.md`](./DOCUMENTATION_PLAN_zh.md) | 剩余文档缺口、目标信息架构与落地计划 | +| [Mooncake 传输跟踪](../experimental/mooncake-transfer-traces/README_zh.md) / [English](../experimental/mooncake-transfer-traces/README.md) | 复现双节点 H100 传输实验并理解跟踪事件丢失 | ## 权威参考 diff --git a/experimental/README.md b/experimental/README.md index f39dfc4afe..972a0447e4 100644 --- a/experimental/README.md +++ b/experimental/README.md @@ -3,3 +3,5 @@ This folder contains experimental WIP code that is mostly Claude Code generated. **Warning:** Code in this directory is very basic and likely contains errors or incomplete implementations. It is not intended for production use or as part of the official InferenceMAX results. + +- [Mooncake two-node H100 transfer tracing](mooncake-transfer-traces/README.md): patches, benchmark results, and trace-integrity analysis. diff --git a/experimental/mooncake-transfer-traces/LICENSE-Mooncake b/experimental/mooncake-transfer-traces/LICENSE-Mooncake new file mode 100644 index 0000000000..7a4a3ea242 --- /dev/null +++ b/experimental/mooncake-transfer-traces/LICENSE-Mooncake @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/experimental/mooncake-transfer-traces/README.md b/experimental/mooncake-transfer-traces/README.md new file mode 100644 index 0000000000..11cdbe66f3 --- /dev/null +++ b/experimental/mooncake-transfer-traces/README.md @@ -0,0 +1,190 @@ +# Mooncake two-node H100 transfer tracing + +**English** | [中文](README_zh.md) + +An experimental snapshot of Mooncake transfer-engine instrumentation, Slurm benchmark launchers, +and the two-node H100 measurements from 24 September 2026. This is a standalone Mooncake +experiment, not an official InferenceX serving benchmark or a registered InferenceX runner/recipe. + +## Results + +Job **1695** completed all **90 cases** (15 message sizes × read/write × 3 repeats) in **9 min 32 s**. +At **64 MiB**, average payload bandwidth was **41.751 GB/s read** and **40.152 GB/s write**. +The traces are **incomplete**: **2,483,420 event records were dropped (4.50%)** by the recorder queue. +Do not use them as a lossless replay or an exact outstanding-request timeline. + +![Bandwidth comparison](results/tebench-1695.6UKR6R/bandwidth-comparison.png) + +| Message size | Read GB/s | Write GB/s | Read vs untraced | Write vs untraced | +| --- | ---: | ---: | ---: | ---: | +| 4 KiB | 0.465 | 0.468 | -7.04% | -6.77% | +| 8 KiB | 0.930 | 0.940 | -5.24% | -5.07% | +| 16 KiB | 1.778 | 1.826 | -4.87% | -5.17% | +| 32 KiB | 3.387 | 3.451 | -5.40% | -5.63% | +| 64 KiB | 6.195 | 6.315 | -6.28% | -6.15% | +| 128 KiB | 10.190 | 10.183 | -7.73% | -8.29% | +| 256 KiB | 15.579 | 15.362 | -10.99% | -10.03% | +| 512 KiB | 20.957 | 21.272 | -8.74% | -7.57% | +| 1 MiB | 28.261 | 28.181 | -6.84% | -5.32% | +| 2 MiB | 29.451 | 29.845 | -3.52% | -3.63% | +| 4 MiB | 34.996 | 34.617 | -2.25% | -2.29% | +| 8 MiB | 38.708 | 37.683 | -1.24% | -1.43% | +| 16 MiB | 40.931 | 39.432 | -0.70% | -0.88% | +| 32 MiB | 41.468 | 40.072 | -0.20% | -0.75% | +| 64 MiB | 41.751 | 40.152 | -0.17% | -0.17% | + +Numbers are means of three repeats; plot whiskers show repeat minima/maxima, not confidence intervals. +The reference is untraced job **1612**, using the same message sizes and concurrency. This is an +observational comparison between separate runs and binaries, not an isolated measurement of tracing +cost. Small messages were roughly **4–11% slower**; at 64 MiB the decrease was about **0.17%**. +At 4 KiB, mean wall time was **8.81 µs read / 8.76 µs write**, versus **8.19 / 8.17 µs** previously. + +The workload uses registered **GPU memory**, one H100 80 GB per node, **one thread**, **batch size 1**, +and a 256 MiB buffer. Each case has 1 s warmup and 5 s measurement. Recording includes both phases. + +- Initiator: `slurm-h100-206-025` (`10.0.1.136`). +- Target: `slurm-h100-206-035` (`10.0.3.49:15309`). +- Read: payload moves **035 → 025**; write: payload moves **025 → 035**. +- Backend: TENT; transport: one-sided RDMA; discovery: p2p metadata. +- GB/s denotes decimal payload bytes/s; KiB and MiB denote binary message sizes. + +Large messages amortize per-request overhead and approach **40 GB/s**. Individual 64 KiB requests +achieved only **6.2–6.3 GB/s** at this concurrency. These are transport-level results, excluding +KV lookup, placement, packing, session affinity and prefill/decode compute. They do not establish +multi-NIC aggregate capacity, simultaneous bidirectional bandwidth, or actual PD serving performance. + +## Trace integrity and limitations + +![Trace loss](results/tebench-1695.6UKR6R/trace-loss.png) + +The full scan of six initiator files found: + +| Quantity | Count | +| --- | ---: | +| Retained event records | 52,712,290 | +| Dropped event records | 2,483,420 | +| Matched submit/completion pairs | 26,355,861 | +| Submit-only request IDs | 285 | +| Complete-only request IDs | 283 | +| Request IDs absent entirely | 1,241,426 | +| Attempted logical requests, including warmup | 27,597,855 | + +All retained completions report successful RDMA completion; no duplicate events were found. +Every initiator footer reconciles with its actual record count. The logs report `write_failed=0`: +loss came from the bounded recording queue. This does not prove the outcome of missing events. +The passive target contains only a start record and is killed during normal launcher cleanup. + +The recorder preserves original descriptors across merging, records completion once across retries, +and tags internal staging requests separately. It is disabled by default and works with aggregate +metrics compiled out. Enable it with `TENT_TRACE_DIR`; the Slurm launcher exposes `TRACE=1`. + +Timestamps describe admission and **observed** completion, including polling delay. They are not NIC +hardware timestamps, packets, CPU instructions, or target-side receive calls. IDs are local to a +trace file, not application/session IDs. Loss is not known to be random, so retained latency and +rate distributions may be biased. Exact replay needs zero-drop recording plus driver-level IDs and +phase markers. A larger buffer absorbs bursts but cannot fix sustained writer overload. + +## Package contents + +| Path | Contents | +| --- | --- | +| [patches/0001-tent-transfer-event-recorder.patch](patches/0001-tent-transfer-event-recorder.patch) | Recorder implementation, engine/peer lookup integration, nine tests, CMake registration, schema documentation | +| [patches/0002-tebench-slurm.patch](patches/0002-tebench-slurm.patch) | Two-node H100 launcher, optional tracing, compute-node Pyxis build, usage notes | +| [patches/0003-store-benchmark-slurm.patch](patches/0003-store-benchmark-slurm.patch) | Earlier Store KV/microbenchmark/local-storage Slurm adaptations; these are not the source of the reported RDMA measurements | +| [patches/series](patches/series) | Patch order | +| [results/tebench-1695.6UKR6R](results/tebench-1695.6UKR6R) | Traced summaries, logs, topology, plots, integrity analysis and 1,440 sampled request spans across 90 windows | +| [results/tebench-1612.XD95we](results/tebench-1612.XD95we) | Untraced baseline summaries and topology | +| [scripts/analyze_traces.py](scripts/analyze_traces.py) | Streaming full-file trace audit; explicit result/trace directories and worker count | +| [scripts/extract_trace_windows.py](scripts/extract_trace_windows.py) | Extract 16 matched request spans near each phase midpoint; assumes these sequential, single-thread traces | +| [validation/ctest.log](validation/ctest.log) | Original 64-test validation output | +| [manifest.json](manifest.json) | Base revision, patched-file hashes, provenance and external raw trace inventory | +| [SHA256SUMS](SHA256SUMS) | Checksums of packaged files, excluding this checksum file | +| [LICENSE-Mooncake](LICENSE-Mooncake) | Upstream Apache-2.0 license | + +The **16.39 GiB raw event files**, built binaries and container image are not included. On the +original cluster, raw traces remain at: + +```text +/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events +``` + +The package is usable for reading results without that directory. Re-running the full audit needs +the external traces. Historical command/log paths are preserved as provenance, not portable defaults. +The interactive conversation viewer is not bundled; its sampled data is in `trace-windows.json`. + +## Apply and reproduce + +The patches target Mooncake commit **`fe0d23e332f41c7d54b5c217194dce5f8e751b21`** and capture the +previously uncommitted changes. Use a fresh checkout; do not apply them again to the already-modified +original workspace. Replace the first two paths below: + +```bash +export PACKAGE=/path/to/InferenceX/experimental/mooncake-transfer-traces +export MOONCAKE_ROOT=/path/to/Mooncake-repro +git clone https://github.com/kvcache-ai/Mooncake.git "$MOONCAKE_ROOT" +git -C "$MOONCAKE_ROOT" checkout --detach fe0d23e332f41c7d54b5c217194dce5f8e751b21 +git -C "$MOONCAKE_ROOT" apply --check "$PACKAGE"/patches/*.patch +git -C "$MOONCAKE_ROOT" apply "$PACKAGE"/patches/*.patch +``` + +The archived Slurm scripts run inside Mooncake and retain its original configuration conventions. +They are not new InferenceX runner entrypoints. Patch 3 is optional for the transfer experiment; +patches 1 and 2 suffice for tebench. After application, the detailed recorder reference is +`mooncake-transfer-engine/tent/TRANSFER_EVENTS.md` in that checkout. + +On the original cluster, reproduce the full traced sweep with: + +```bash +export BUILD=1 TRACE=1 BUILD_ONLY=0 +export MIN_BYTES=4096 MAX_BYTES=67108864 BUFFER_BYTES=268435456 +export BATCH_SIZE=1 THREADS=1 DURATION=5 REPEATS=3 +export SEG_TYPE=VRAM OPS="read write" TENT_TRACE_BUFFER_RECORDS=65536 +sbatch --partition=h100 --account=cw-sup --nodes=2 \ + --nodelist=slurm-h100-206-025,slurm-h100-206-035 --gpus-per-node=1 \ + "$MOONCAKE_ROOT/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch" +``` + +The launcher builds in a lightweight writable `ubuntu:22.04` Pyxis container on a compute node, +using host CUDA at `/usr/local/cuda-13.0`; the login node does not need CMake. It installs build +packages in the container, caches `build/tebench-pyxis/runtime.sqsh`, and runs both peers in that +image. Prerequisites are Slurm with the named partition/account/nodes, Pyxis/Enroot, shared writable +storage, CUDA, RDMA devices and package-download access. Adapt these paths/resources on another +cluster. The image tag and apt packages are not digest/version pinned, so rebuilds are not +bit-for-bit reproducible. `BUILD=0` is valid only after a successful build with the recorder. + +Results appear under `$MOONCAKE_ROOT/benchmark-results/tebench-JOBID.XXXXXX/`; traces are in `events/`. +The script's outer log is `tebench-JOBID.out` in the submission directory unless `--output` overrides +it. The queue size above reproduces the observed overflow; it does **not** guarantee a complete trace. +For a controlled overhead comparison, run `TRACE=0` and `TRACE=1` back-to-back with the same binary +and settings. Avoid drawing lossless-replay conclusions until all initiator footers report zero drops. + +## Re-analyze and validation + +The streaming audit requires Python 3.10+ and `orjson`; run it on a compute node for the full dataset. +Change `EVENTS_DIR` if the raw files have moved. These commands replace the derived JSON files in +`RUN_DIR`, so use a copied run directory if you want to retain the packaged versions unchanged. + +```bash +export RUN_DIR="$PACKAGE/results/tebench-1695.6UKR6R" +export EVENTS_DIR=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events +python3 -m pip install -r "$PACKAGE/scripts/requirements.txt" +python3 "$PACKAGE/scripts/analyze_traces.py" \ + --run-dir "$RUN_DIR" --events-dir "$EVENTS_DIR" --workers 3 +python3 "$PACKAGE/scripts/extract_trace_windows.py" \ + --run-dir "$RUN_DIR" --events-dir "$EVENTS_DIR" +``` + +The full audit joins file-local request IDs and checks footer counts. The window extractor uses +monotonic event order for seeking; it is not a general extractor for arbitrarily interleaved +multi-thread traces. The scripts do not regenerate the Markdown or plots. + +Recorded runtime evidence: + +- Job **1694**: **64 tests passed**: 9 recorder, 8 merge, 30 failover, 14 queue-dispatch, 3 causal-chain tests; metrics compiled out. +- Job **1693**: two-node H100 smoke test, **16,442 matched pairs**, zero drops. +- Job **1695**: all 90 full-sweep cases completed, with **4.50% event loss**. +- Job **1696**: full scan of all six initiator traces completed. + +Packaging validation applies all patches to the pinned base files, compares the resulting file +hashes with the tested working tree, and runs the packaged analysis tools on controlled trace +fixtures. This packaging step does not claim another GPU performance run. diff --git a/experimental/mooncake-transfer-traces/README_zh.md b/experimental/mooncake-transfer-traces/README_zh.md new file mode 100644 index 0000000000..cad762d2fd --- /dev/null +++ b/experimental/mooncake-transfer-traces/README_zh.md @@ -0,0 +1,177 @@ +# Mooncake 双节点 H100 传输跟踪实验 + +[English](README.md) | **中文** + +本目录保存了 Mooncake 传输引擎插桩、Slurm 基准启动脚本，以及 2026 年 9 月 24 日的双节点 H100 +测试结果。这是独立的 Mooncake 实验，不是 InferenceX 官方推理服务基准，也没有注册新的 +InferenceX runner 或 recipe。 + +## 测试结果 + +作业 **1695** 用时 **9 分 32 秒**，完成了全部 **90 个测试项**（15 种消息大小 × 读写 × 3 次重复）。 +**64 MiB** 消息的平均有效载荷带宽为：**读 41.751 GB/s，写 40.152 GB/s**。 +跟踪数据**不完整**：记录队列丢弃了 **2,483,420 条事件（4.50%）**。 +不能将这些文件用于无损回放，也不能据此精确重建未完成请求数量的时间线。 + +![带宽对比](results/tebench-1695.6UKR6R/bandwidth-comparison.png) + +| 消息大小 | 读 GB/s | 写 GB/s | 读相对未跟踪基线 | 写相对未跟踪基线 | +| --- | ---: | ---: | ---: | ---: | +| 4 KiB | 0.465 | 0.468 | -7.04% | -6.77% | +| 8 KiB | 0.930 | 0.940 | -5.24% | -5.07% | +| 16 KiB | 1.778 | 1.826 | -4.87% | -5.17% | +| 32 KiB | 3.387 | 3.451 | -5.40% | -5.63% | +| 64 KiB | 6.195 | 6.315 | -6.28% | -6.15% | +| 128 KiB | 10.190 | 10.183 | -7.73% | -8.29% | +| 256 KiB | 15.579 | 15.362 | -10.99% | -10.03% | +| 512 KiB | 20.957 | 21.272 | -8.74% | -7.57% | +| 1 MiB | 28.261 | 28.181 | -6.84% | -5.32% | +| 2 MiB | 29.451 | 29.845 | -3.52% | -3.63% | +| 4 MiB | 34.996 | 34.617 | -2.25% | -2.29% | +| 8 MiB | 38.708 | 37.683 | -1.24% | -1.43% | +| 16 MiB | 40.931 | 39.432 | -0.70% | -0.88% | +| 32 MiB | 41.468 | 40.072 | -0.20% | -0.75% | +| 64 MiB | 41.751 | 40.152 | -0.17% | -0.17% | + +表中数据为三次重复的算术平均值；图中的误差线表示最小值和最大值，不是置信区间。 +基线为未启用跟踪的作业 **1612**，消息大小和并发设置相同。两次运行的时间和二进制文件不同， +因此这只是观测对比，不能单独归因于跟踪开销。小消息带宽下降约 **4–11%**，64 MiB 时下降约 **0.17%**。 +4 KiB 的平均墙钟耗时为**读 8.81 µs、写 8.76 µs**，此前为 **8.19 / 8.17 µs**。 + +工作负载使用已注册的 **GPU 显存**，每个节点一张 H100 80 GB，**单线程**、**batch size 1**， +缓冲区为 256 MiB。每个测试项预热 1 秒、测量 5 秒。事件跟踪包含这两个阶段。 + +- 发起端：`slurm-h100-206-025`（`10.0.1.136`）。 +- 目标端：`slurm-h100-206-035`（`10.0.3.49:15309`）。 +- 读操作的数据方向为 **035 → 025**；写操作为 **025 → 035**。 +- 后端为 TENT，传输为单边 RDMA，使用 p2p 元数据发现。 +- GB/s 表示十进制有效载荷字节率，KiB 和 MiB 表示二进制消息大小。 + +大消息摊薄了逐请求开销，带宽接近 **40 GB/s**；当前并发设置下，独立 64 KiB 请求仅达到 +**6.2–6.3 GB/s**。这些是传输层结果，不包括 KV 查询、放置、打包、会话亲和性，以及 prefill/decode +计算，不能据此判断多网卡聚合带宽、同时双向吞吐量或实际 PD 推理服务性能。 + +## 跟踪完整性与限制 + +![跟踪事件丢失率](results/tebench-1695.6UKR6R/trace-loss.png) + +对六个发起端文件进行完整扫描后得到： + +| 指标 | 数量 | +| --- | ---: | +| 保留的事件记录 | 52,712,290 | +| 丢弃的事件记录 | 2,483,420 | +| 提交与完成均匹配的请求 | 26,355,861 | +| 仅保留提交事件的请求 ID | 285 | +| 仅保留完成事件的请求 ID | 283 | +| 完全缺失的请求 ID | 1,241,426 | +| 尝试执行的逻辑请求，含预热 | 27,597,855 | + +所有保留下来的完成事件都报告 RDMA 成功，未发现重复事件。每个发起端文件的尾部计数均与实际 +记录数相符。日志中的 `write_failed=0` 表明记录丢失来自有界队列，而不是报告的文件写入失败； +这不能证明缺失事件对应请求的状态。被动目标端只有开始记录，启动脚本在正常清理时终止其等待进程。 + +记录器在请求合并后仍保留原始描述符，重试不会重复记录逻辑完成，并单独标记内部 staging 请求。 +默认关闭记录；即使聚合指标在编译时关闭也可工作。通过 `TENT_TRACE_DIR` 启用，Slurm 启动脚本 +提供 `TRACE=1` 开关。 + +时间戳表示请求接纳及**观察到的**完成时间，包含轮询延迟；它们不是网卡硬件时间戳、网络包、CPU +指令或目标端 receive 调用。请求 ID 仅在单个跟踪文件内有效，不是应用请求或会话 ID。丢失不一定 +随机，因此保留记录的延迟和速率分布可能有偏。精确回放需要零丢失记录，以及驱动层 ID 和阶段标记。 +增大缓冲区可以吸收突发，但不能解决持续的写出吞吐不足。 + +## 包内容 + +| 路径 | 内容 | +| --- | --- | +| [patches/0001-tent-transfer-event-recorder.patch](patches/0001-tent-transfer-event-recorder.patch) | 记录器实现、引擎与对端名称查询集成、九个测试、CMake 注册和数据格式说明 | +| [patches/0002-tebench-slurm.patch](patches/0002-tebench-slurm.patch) | 双节点 H100 启动脚本、可选跟踪、计算节点 Pyxis 构建及使用说明 | +| [patches/0003-store-benchmark-slurm.patch](patches/0003-store-benchmark-slurm.patch) | 此前的 Store KV、微基准和本地存储 Slurm 适配；本报告的 RDMA 数据并非由这些脚本生成 | +| [patches/series](patches/series) | 补丁应用顺序 | +| [results/tebench-1695.6UKR6R](results/tebench-1695.6UKR6R) | 跟踪运行的汇总、日志、拓扑、图表、完整性分析，以及 90 个窗口中的 1,440 个采样请求区间 | +| [results/tebench-1612.XD95we](results/tebench-1612.XD95we) | 未跟踪基线的汇总和拓扑 | +| [scripts/analyze_traces.py](scripts/analyze_traces.py) | 流式完整文件审计，显式指定结果目录、跟踪目录和进程数 | +| [scripts/extract_trace_windows.py](scripts/extract_trace_windows.py) | 在每个阶段中点附近提取 16 个匹配请求区间，适用于本实验的顺序单线程记录 | +| [validation/ctest.log](validation/ctest.log) | 原始 64 项测试的输出 | +| [manifest.json](manifest.json) | 基础版本、补丁结果文件哈希、来源及外部原始文件清单 | +| [SHA256SUMS](SHA256SUMS) | 包内文件校验和，不包含校验和文件自身 | +| [LICENSE-Mooncake](LICENSE-Mooncake) | 上游 Apache-2.0 许可证 | + +**16.39 GiB 原始事件文件**、构建后的二进制文件及容器镜像不包含在本目录中。原集群上的原始跟踪仍位于： + +```text +/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events +``` + +无需此目录即可阅读本包的结果；重新执行完整审计则需要这些外部文件。历史命令和日志中的路径用于 +保留来源，不是可移植的默认配置。对话中的交互式视图未打包，其采样数据保存在 `trace-windows.json`。 + +## 应用补丁与复现 + +补丁基于 Mooncake 提交 **`fe0d23e332f41c7d54b5c217194dce5f8e751b21`**，包含此前尚未提交的修改。 +请使用新的 checkout，不要在已修改的原工作目录重复应用。先替换下方两个路径： + +```bash +export PACKAGE=/path/to/InferenceX/experimental/mooncake-transfer-traces +export MOONCAKE_ROOT=/path/to/Mooncake-repro +git clone https://github.com/kvcache-ai/Mooncake.git "$MOONCAKE_ROOT" +git -C "$MOONCAKE_ROOT" checkout --detach fe0d23e332f41c7d54b5c217194dce5f8e751b21 +git -C "$MOONCAKE_ROOT" apply --check "$PACKAGE"/patches/*.patch +git -C "$MOONCAKE_ROOT" apply "$PACKAGE"/patches/*.patch +``` + +归档的 Slurm 脚本在 Mooncake 内运行，并保留其原有配置约定；它们不是新的 InferenceX runner +入口。传输实验只需要补丁 1 和 2，补丁 3 可选。应用后，完整记录器说明位于该 checkout 的 +`mooncake-transfer-engine/tent/TRANSFER_EVENTS.md`。 + +在原集群上，可使用以下命令复现完整跟踪扫描： + +```bash +export BUILD=1 TRACE=1 BUILD_ONLY=0 +export MIN_BYTES=4096 MAX_BYTES=67108864 BUFFER_BYTES=268435456 +export BATCH_SIZE=1 THREADS=1 DURATION=5 REPEATS=3 +export SEG_TYPE=VRAM OPS="read write" TENT_TRACE_BUFFER_RECORDS=65536 +sbatch --partition=h100 --account=cw-sup --nodes=2 \ + --nodelist=slurm-h100-206-025,slurm-h100-206-035 --gpus-per-node=1 \ + "$MOONCAKE_ROOT/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch" +``` + +启动脚本在计算节点的轻量可写 `ubuntu:22.04` Pyxis 容器中构建，使用宿主机 +`/usr/local/cuda-13.0` 下的 CUDA，因此登录节点不需要 CMake。构建依赖安装在容器内，镜像缓存为 +`build/tebench-pyxis/runtime.sqsh`，两个节点都使用该镜像运行。前提是具备所列 Slurm 分区、账号和 +节点，以及 Pyxis/Enroot、共享可写存储、CUDA、RDMA 设备和软件包下载权限。在其他集群上需调整 +路径和资源。镜像标签及 apt 软件包没有固定到摘要或版本，因此无法保证逐位相同的重建结果。 +只有成功构建了包含记录器的程序后，才可使用 `BUILD=0`。 + +结果位于 `$MOONCAKE_ROOT/benchmark-results/tebench-JOBID.XXXXXX/`，跟踪位于其 `events/` 子目录。 +外层日志默认为提交目录下的 `tebench-JOBID.out`，可用 `--output` 覆盖。上面的队列大小复现了本次 +溢出设置，**不保证**跟踪完整。要控制跟踪开销对比，应在同一二进制和相同设置下连续运行 `TRACE=0` +与 `TRACE=1`；只有全部发起端尾部计数均为零丢失时，才能讨论无损回放。 + +## 重新分析与验证 + +流式审计需要 Python 3.10+ 和 `orjson`；完整数据集建议在计算节点处理。若原始文件已迁移，请修改 +`EVENTS_DIR`。以下命令会替换 `RUN_DIR` 中派生的 JSON 文件；如需保持包内结果不变，请先复制运行目录。 + +```bash +export RUN_DIR="$PACKAGE/results/tebench-1695.6UKR6R" +export EVENTS_DIR=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events +python3 -m pip install -r "$PACKAGE/scripts/requirements.txt" +python3 "$PACKAGE/scripts/analyze_traces.py" \ + --run-dir "$RUN_DIR" --events-dir "$EVENTS_DIR" --workers 3 +python3 "$PACKAGE/scripts/extract_trace_windows.py" \ + --run-dir "$RUN_DIR" --events-dir "$EVENTS_DIR" +``` + +完整审计按文件内请求 ID 配对并核对尾部计数。窗口提取器利用单调的事件顺序定位，不适用于任意 +交错的多线程跟踪。这些脚本不会重新生成 Markdown 或图表。 + +已保存的运行证据： + +- 作业 **1694**：**64 项测试通过**，包括 9 项记录器、8 项合并、30 项故障切换、14 项队列派发、3 项因果链测试；编译时关闭聚合指标。 +- 作业 **1693**：双节点 H100 冒烟测试，**16,442 对匹配事件**，零丢失。 +- 作业 **1695**：完整扫描的 90 项全部完成，**事件丢失率 4.50%**。 +- 作业 **1696**：完成六个发起端跟踪文件的全量扫描。 + +打包验证会将全部补丁应用到指定基础版本的文件上，比较所得文件哈希与已测试工作目录是否一致， +并在受控跟踪样本上运行包内分析工具。本次打包不声称再次运行了 GPU 性能测试。 diff --git a/experimental/mooncake-transfer-traces/SHA256SUMS b/experimental/mooncake-transfer-traces/SHA256SUMS new file mode 100644 index 0000000000..3fe66632e1 --- /dev/null +++ b/experimental/mooncake-transfer-traces/SHA256SUMS @@ -0,0 +1,58 @@ +58d1e17ffe5109a7ae296caafcadfdbe6a7d176f0bc4ab01e12a689b0499d8bd LICENSE-Mooncake +c64aa625f3178b76699e8164c4c27635f082c8ae9ef26d56fd5b1221b22e26d9 README.md +232e1307898115dd3998407fd8ca1910526a9b4a3620f89809cb9abd316e5ed1 README_zh.md +f12c477de908b0ee64589ba9434522c57adc0e7a237347540b3cc227dd587b93 manifest.json +62a5f0ca444553a66c71cfe5951ebaaa3856b81c8eb559e5437ad82f9e0eac97 patches/0001-tent-transfer-event-recorder.patch +7ba12e62c510b50b2d734964aca0ea1fc448f81fbef4bf2c26ef9fd61b5f2d21 patches/0002-tebench-slurm.patch +393532df6f7cba064ab97af20af153e2bca7eb3647a5f02ddec76b1186533103 patches/0003-store-benchmark-slurm.patch +b386c64116b95c3b56644dc044ddd2510f4200c4fb2c557503143a7743e0c995 patches/series +86decd4df4b9fccae163368d30ba39598b84f9035bdb667a9ac988c6a448da9e results/tebench-1612.XD95we/commit.txt +870b18bbadc536ddba617243972fe9f7031449074302e892fd0f1164f4b42234 results/tebench-1612.XD95we/read-1.jsonl +ee92656aa9046e73b0a6c582c995b58821799514b0e462af9085c378cdbf6e26 results/tebench-1612.XD95we/read-2.jsonl +073b2c0c9c1520b8ce92de997395e26e3ab25561b9856b22caa97dd40cabd5c3 results/tebench-1612.XD95we/read-3.jsonl +92e196e04b74f340a4f532ecd6c930732b7b4c33383164b1d4b3c94e13f80f90 results/tebench-1612.XD95we/summary.csv +d2f7bf23875e9b3d1629e8c865d0e5dc4f52879512ccb3b1a4f7a72e5c0528b7 results/tebench-1612.XD95we/topology.txt +e7fa148a2d47922667363f8a15e61024460bead0828de0c3280010b84e4e2118 results/tebench-1612.XD95we/write-1.jsonl +ef15b9604e35638c09561f6e69375336360e57bd7f762af7ab6980431e642f81 results/tebench-1612.XD95we/write-2.jsonl +2fc8b62c41c30eee423ae5b6b26091bf696e8f1a06fe91ab0c7a678492121803 results/tebench-1612.XD95we/write-3.jsonl +d280fdb5153743cc4277040f10254910d93ae4b5f0b7931100faa17f47a20e64 results/tebench-1695.6UKR6R/analysis-job.out +b9b1021eb4249986d645dcf04282d5636f199653c10e02ff0ccbb8a77a22bd40 results/tebench-1695.6UKR6R/bandwidth-comparison.png +5a3b2c8007c5825d598518df020cc4faae8e439d2cda9b54a4a4ca5cd63a0438 results/tebench-1695.6UKR6R/bandwidth-summary.csv +a22335426ccfa1f2dbad871b90256f550cb33ab98c1148e789f0af9f10e2d836 results/tebench-1695.6UKR6R/bandwidth-summary.json +86decd4df4b9fccae163368d30ba39598b84f9035bdb667a9ac988c6a448da9e results/tebench-1695.6UKR6R/commit.txt +1e4f6810575d07379b5760f2e8afe714136782645aa1024ba9b9a0ee1017636b results/tebench-1695.6UKR6R/endpoint.txt +f1ccf982ac6c879ee8d0c79b5af3d2d8195d86af3f4230f7e8b9cea4a246545a results/tebench-1695.6UKR6R/host-slurm-h100-206-025.txt +378d7d11cd71292ea7f485bcd050249fe85d34ae52e3adefcf563e96d4043a7e results/tebench-1695.6UKR6R/host-slurm-h100-206-035.txt +a49dde32c01afe80315baabed381d9ad663f3b23ef3f959a580a62e816dae364 results/tebench-1695.6UKR6R/job.log +f9b3fdb356be852574633ea843232b37dc2bf1f73d3171b656852da2a33b6fff results/tebench-1695.6UKR6R/read-1.command.txt +2299aa0e0ec8bc303cf44c7ba86c94af7db8e3ab3b4fe9f7870128177398be74 results/tebench-1695.6UKR6R/read-1.jsonl +64f4b4a59a5b3371602ce2b393d67ad9502bdfd1a15085cf233c8cb17098cd92 results/tebench-1695.6UKR6R/read-1.log +66932b2ef77499ebd04ccc9f0aa970b1e12ea208ba234bdd38aa1e0f7e90ea85 results/tebench-1695.6UKR6R/read-2.command.txt +27d387618f044ef4a550696939ebbd3c9faa5ae941aba7eecda9adc8f6729f0a results/tebench-1695.6UKR6R/read-2.jsonl +1b5a7d7bc71f7c368178d40051499f84076c6af941c4316c997122c12d5f25dd results/tebench-1695.6UKR6R/read-2.log +41b71a692b2c3792a984f6614328a1257796a265ed85be11d43c155ea5db24cf results/tebench-1695.6UKR6R/read-3.command.txt +088d9ba6a26f836b1a054fa33a73ac9d5896b8d95f341fc9a70c44a8aacbd120 results/tebench-1695.6UKR6R/read-3.jsonl +ee0c945656342fe9dcc0c104fb51238dd54589c8cf295d7921354ec5c7bc1e9f results/tebench-1695.6UKR6R/read-3.log +782f6ecf53961c3db75ae9b0862eaf6f918953d3e4e9978d26bcf6572d0951c2 results/tebench-1695.6UKR6R/slurm-job.txt +9232724fa7bff1b5bdc3af7ef682788e021a7e45d6a643fefa92618bb74c9e3c results/tebench-1695.6UKR6R/target-command.txt +9e29e16aaae3cc944d2de8191385c77b94d72cbb87620fdf7cad4854295f1344 results/tebench-1695.6UKR6R/target.log +d2f7bf23875e9b3d1629e8c865d0e5dc4f52879512ccb3b1a4f7a72e5c0528b7 results/tebench-1695.6UKR6R/topology.txt +6048729abc0151c01ac607d7409d7efef75a249be32c7ff37231957d691e251f results/tebench-1695.6UKR6R/trace-analysis.json +2ab6d4036e18cc17bdec5854c81c60af5d9a0efafacb0f4206ac4daa24f2e05d results/tebench-1695.6UKR6R/trace-loss.png +f899b70d8384a3b2519115cafe973b7f11835f061feeb5419bc27d6a077a442b results/tebench-1695.6UKR6R/trace-windows.json +d5f79dc4aa1006e7be26c23e90eea2d6929e9d598cd879ab32fbf9911f973ee6 results/tebench-1695.6UKR6R/write-1.command.txt +fa019b39e2717f9eb1e006e57ecbaa23133e8256bdebd9fb15c034b996a6c948 results/tebench-1695.6UKR6R/write-1.jsonl +f96019974a0790a2b090860a995f8e0adfa9656a89a65b7b5789c6529c4ecaf3 results/tebench-1695.6UKR6R/write-1.log +fe0f9a09b5397113d74b4a96835c463132b403e3bb8344707223a814b45f0770 results/tebench-1695.6UKR6R/write-2.command.txt +8ce6e93c39bf2889ed49167592ee1bbcadc9497584bad4259e23be4edafd3cf5 results/tebench-1695.6UKR6R/write-2.jsonl +f399dcc091f6531900552984cbe7ccf7b944c9dc650568b9502728bc139932a7 results/tebench-1695.6UKR6R/write-2.log +71004bda65971a73404872008f2ccf12b58c4e9b1f6afcbcc3b340330b021442 results/tebench-1695.6UKR6R/write-3.command.txt +e7b7b1ef37cc2d375a88e4c1130bd525836df01c712ccd2608fd5055ec150cce results/tebench-1695.6UKR6R/write-3.jsonl +87af48bcd903452c73d9597279fb6aaa50f564e4b5010bfca53fc54686470e4d results/tebench-1695.6UKR6R/write-3.log +78538e6fc03fb327123e16ca37faeede1c38227d0545e7ca5ff3c2ef6f22eeb7 scripts/analyze_traces.py +b47b6cae4b34a39c5f7e47b069f5db180e243724bbab36598c25e49623474e19 scripts/extract_trace_windows.py +4446b5d59ae272063d7a7bb62c7d2077a99b89aa2c109c4b1c3f47c4d4b1f5e5 scripts/requirements.txt +2b6ea3819079f0d27a6659431957e6d97e00c37cff21292dc3fcf9bd5889519d validation/ctest.log +a0cdcaca892923571963d4dfd56f598f8ad915d2732cb809726308d8bf0fac78 validation/package-checks.txt +2c0f78bc6fe3793112c3567c3725b4568fddff712f3d4fb86be7820305c42962 validation/rdma-smoke-job-1693.log +c81b7502187f41dadf4c9c072bdf3354067817c99cd9734d5d001a8df57e7664 validation/regression-job-1694.log diff --git a/experimental/mooncake-transfer-traces/manifest.json b/experimental/mooncake-transfer-traces/manifest.json new file mode 100644 index 0000000000..208b339e72 --- /dev/null +++ b/experimental/mooncake-transfer-traces/manifest.json @@ -0,0 +1,129 @@ +{ + "schema_version": 1, + "upstream": "https://github.com/kvcache-ai/Mooncake", + "base_commit": "fe0d23e332f41c7d54b5c217194dce5f8e751b21", + "package_date": "2026-09-25", + "patch_order": [ + "0001-tent-transfer-event-recorder.patch", + "0002-tebench-slurm.patch", + "0003-store-benchmark-slurm.patch" + ], + "patched_files": [ + { + "path": "mooncake-transfer-engine/tent/include/tent/runtime/segment_manager.h", + "sha256": "43ab3d3cc0b8172a49201d2812d19343119c535eafd3675962a58a778fdd2942" + }, + { + "path": "mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h", + "sha256": "ea937bd0d304e65c1764edefc673e87ef38d9bc8351689945aa637885329f533" + }, + { + "path": "mooncake-transfer-engine/tent/src/runtime/segment_manager.cpp", + "sha256": "49ce72f2f2a8dbaa19ff95a2d92f4996fad00f981095fa7087a62d939c7da0cf" + }, + { + "path": "mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp", + "sha256": "a930b15225f2ba9a28e44ef058f35d94308cb19509b13bf4288a4f8cf6d120b5" + }, + { + "path": "mooncake-transfer-engine/tent/tests/CMakeLists.txt", + "sha256": "8b8d233e8f5dc3ee553bf82d0b230c39c0ee9547af8d9b46b890e8d5418123e6" + }, + { + "path": "mooncake-transfer-engine/tent/TRANSFER_EVENTS.md", + "sha256": "bc73d64ebff9a1052d998ee3662d4be45ec1a1ba9ce234ad4acd93bf3cb84760" + }, + { + "path": "mooncake-transfer-engine/tent/include/tent/runtime/transfer_event_recorder.h", + "sha256": "a034b2db5ea3d572427e8807bc45b0859c17e5c030671be9235e6f960a289835" + }, + { + "path": "mooncake-transfer-engine/tent/src/runtime/transfer_event_recorder.cpp", + "sha256": "b827e4c3208922b19c527af8389b661bfbb34f4757e9941c10de611be473c12c" + }, + { + "path": "mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp", + "sha256": "c6705a82f02e89c3f88773c025e7cd6f6332c0749585289e7159e05811edb7a4" + }, + { + "path": "mooncake-transfer-engine/benchmark/slurm/README.md", + "sha256": "ed6380df8e487be3a1326bf65550e6ab1d2c0d83839e694303d957c929d25b96" + }, + { + "path": "mooncake-transfer-engine/benchmark/slurm/tebench.sbatch", + "sha256": "66683b706a7829eeffa706a3095e35d6357ffe9e7f351691255f601d542e1d1f" + }, + { + "path": "mooncake-store/benchmarks/slurm/README.md", + "sha256": "b6f4d76e8c61283165011700b62c7d79801ebbc95be65ed204690a9d1d84382c" + }, + { + "path": "mooncake-store/benchmarks/slurm/common.sh", + "sha256": "c26feb9edbbe8b6b8d0c24a31b0d9b72f7740308d2f58eb6ad2035f03787d5d7" + }, + { + "path": "mooncake-store/benchmarks/slurm/storage_trace.sbatch", + "sha256": "964448d1157c049831ad6255fa04bdb9aeb50739578c3cd9ad7c75d80fef16b8" + }, + { + "path": "mooncake-store/benchmarks/slurm/store_kv.sbatch", + "sha256": "47862dd2eec4fb23f1c98487735dae9a33e091ee83659e40f5a63fac0b060a5e" + }, + { + "path": "mooncake-store/benchmarks/slurm/store_micro.sbatch", + "sha256": "cb3c2af9a1a1d4b50a63cc2d2cc76a0f6a14bd34be16392722d52eb3d555a854" + }, + { + "path": "mooncake-store/benchmarks/slurm/worker.sh", + "sha256": "09942d545b794bb3232856dffdc7945345332b0507ab15729720b6a05c421c8a" + } + ], + "source_result_directory": "/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R", + "raw_traces_included": false, + "raw_trace_directory": "/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events", + "raw_trace_files": [ + { + "file": "tent-events-slurm-h100-206-025-2231481-gziMb7.jsonl", + "bytes": 2951125300, + "written_records": 8853876, + "dropped_records": 312506 + }, + { + "file": "tent-events-slurm-h100-206-025-2232757-0Y5JRP.jsonl", + "bytes": 2926127648, + "written_records": 8778252, + "dropped_records": 417592 + }, + { + "file": "tent-events-slurm-h100-206-025-2234022-WL8pRq.jsonl", + "bytes": 2938446459, + "written_records": 8815626, + "dropped_records": 276006 + }, + { + "file": "tent-events-slurm-h100-206-025-2232113-2zgoGf.jsonl", + "bytes": 2916445523, + "written_records": 8722968, + "dropped_records": 505420 + }, + { + "file": "tent-events-slurm-h100-206-025-2233390-uky9P4.jsonl", + "bytes": 2906519707, + "written_records": 8693000, + "dropped_records": 593004 + }, + { + "file": "tent-events-slurm-h100-206-025-2234663-oNUhoa.jsonl", + "bytes": 2958300924, + "written_records": 8848568, + "dropped_records": 378892 + } + ], + "validation": { + "regression_job": 1694, + "passed_tests": 64, + "trace_smoke_job": 1693, + "traced_sweep_job": 1695, + "analysis_job": 1696 + } +} diff --git a/experimental/mooncake-transfer-traces/patches/0001-tent-transfer-event-recorder.patch b/experimental/mooncake-transfer-traces/patches/0001-tent-transfer-event-recorder.patch new file mode 100644 index 0000000000..a914e2cefc --- /dev/null +++ b/experimental/mooncake-transfer-traces/patches/0001-tent-transfer-event-recorder.patch @@ -0,0 +1,1283 @@ +diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/segment_manager.h b/mooncake-transfer-engine/tent/include/tent/runtime/segment_manager.h +index 26fccd23..c2fdbea9 100644 +--- a/mooncake-transfer-engine/tent/include/tent/runtime/segment_manager.h ++++ b/mooncake-transfer-engine/tent/include/tent/runtime/segment_manager.h +@@ -55,6 +55,9 @@ class SegmentManager { + + Status closeRemote(SegmentID handle); + ++ // Local handle lookup only; never contacts the metadata service. ++ std::string lookupSegmentName(SegmentID handle); ++ + // Use `withCachedSegment()` when you want automatic + // cache invalidation and retry on stale segment cache. + Status getRemoteCached(SegmentDesc *&desc, SegmentID handle); +diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h +index bb1417fe..a4f97c04 100644 +--- a/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h ++++ b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h +@@ -33,6 +33,7 @@ + #include "tent/common/types.h" + #include "tent/runtime/admission_queue.h" + #include "tent/runtime/transport.h" ++#include "tent/runtime/transfer_event_recorder.h" + #include "tent/runtime/transport_selector.h" + #include "tent/runtime/hp_tcp_transport_config.h" + +@@ -105,6 +106,7 @@ struct TaskInfo { + // attempt, so a task that recovers and later fails at poll still + // attributes its root cause to submit. + int8_t failure_stage{-1}; ++ std::shared_ptr trace_span; + + TaskInfo() = default; + +@@ -131,7 +133,8 @@ struct TaskInfo { + attempt_post_time(other.attempt_post_time), + attempt_type(other.attempt_type), + attempt_active(other.attempt_active), +- failure_stage(other.failure_stage) {} ++ failure_stage(other.failure_stage), ++ trace_span(other.trace_span) {} + + TaskInfo(TaskInfo&& other) noexcept + : type(other.type), +@@ -156,7 +159,8 @@ struct TaskInfo { + attempt_post_time(other.attempt_post_time), + attempt_type(other.attempt_type), + attempt_active(other.attempt_active), +- failure_stage(other.failure_stage) {} ++ failure_stage(other.failure_stage), ++ trace_span(other.trace_span) {} + + TaskInfo& operator=(const TaskInfo& other) { + if (this != &other) { +@@ -185,6 +189,7 @@ struct TaskInfo { + attempt_type = other.attempt_type; + attempt_active = other.attempt_active; + failure_stage = other.failure_stage; ++ trace_span = other.trace_span; + } + return *this; + } +@@ -216,6 +221,7 @@ struct TaskInfo { + attempt_type = other.attempt_type; + attempt_active = other.attempt_active; + failure_stage = other.failure_stage; ++ trace_span = other.trace_span; + } + return *this; + } +@@ -456,6 +462,9 @@ class TransferEngineImpl { + + Status commitPreparedSubmit(Batch* batch, const PreparedSubmit& prepared); + ++ void recordPreparedSubmit(Batch* batch, const PreparedSubmit& prepared); ++ void recordTraceCompletion(TaskInfo& task, TransferStatusEnum status); ++ + void attachProgressNotifier(Batch* batch, Transport::SubBatchRef sub_batch); + + uint64_t nextBatchToken(); +@@ -585,6 +594,7 @@ class TransferEngineImpl { + uint16_t port_; + bool ipv6_; + std::string local_segment_name_; ++ std::unique_ptr event_recorder_; + + std::unique_ptr staging_proxy_; + bool merge_requests_; +diff --git a/mooncake-transfer-engine/tent/src/runtime/segment_manager.cpp b/mooncake-transfer-engine/tent/src/runtime/segment_manager.cpp +index 61811719..598e6e2c 100644 +--- a/mooncake-transfer-engine/tent/src/runtime/segment_manager.cpp ++++ b/mooncake-transfer-engine/tent/src/runtime/segment_manager.cpp +@@ -59,6 +59,12 @@ Status SegmentManager::openRemote(SegmentID &handle, + return Status::OK(); + } + ++std::string SegmentManager::lookupSegmentName(SegmentID handle) { ++ RWSpinlock::ReadGuard guard(lock_); ++ auto it = id_to_name_map_.find(handle); ++ return it == id_to_name_map_.end() ? std::string() : it->second; ++} ++ + Status SegmentManager::closeRemote(SegmentID handle) { + RWSpinlock::WriteGuard guard(lock_); + if (!id_to_name_map_.count(handle)) +diff --git a/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp b/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp +index 7371be09..5c4f4f49 100644 +--- a/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp ++++ b/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp +@@ -70,6 +70,7 @@ struct Batch { + size_t runtime_refs{0}; + bool free_requested{false}; + uint64_t queue_token{0}; ++ uint64_t trace_batch_id{0}; + // Consecutive lazyFreeBatch passes that failed to reclaim this batch + // (poll error or queue retire error). Reset when a pass observes the + // batch healthy (PENDING). At kMaxReclaimAttempts the batch is marked +@@ -530,6 +531,9 @@ Status TransferEngineImpl::construct() { + } + } + ++ event_recorder_ = ++ TransferEventRecorder::fromEnvironment(local_segment_name_); ++ + if (enable_progress_worker_) { + progress_worker_ = std::make_unique( + this, runtime_queue_config_.enabled +@@ -629,6 +633,11 @@ Status TransferEngineImpl::deconstruct() { + std::unordered_set released_batches; + auto release_batch = [&](Batch* batch) { + if (!released_batches.insert(batch).second) return; ++ if (event_recorder_) { ++ for (auto& task : batch->task_list) ++ event_recorder_->finish(task.trace_span, "unobserved", ++ transportTypeName(task.type)); ++ } + for (size_t type = 0; type < kSupportedTransportTypes; ++type) { + auto& transport = transport_list_[type]; + auto& sub_batch = batch->sub_batch[type]; +@@ -686,6 +695,7 @@ Status TransferEngineImpl::deconstruct() { + } + } + ++ event_recorder_.reset(); + progress_worker_.reset(); + local_segment_tracker_.reset(); + if (metadata_) { +@@ -1086,6 +1096,7 @@ BatchID TransferEngineImpl::allocateBatch(size_t batch_size) { + Batch* batch = Slab::Get().allocate(); + if (!batch) return (BatchID)0; + batch->max_size = batch_size; ++ if (event_recorder_) batch->trace_batch_id = event_recorder_->nextBatchId(); + batch->task_list.reserve(batch_size); + BatchID batch_id = (BatchID)batch; + // Registry insert under the batch's own shard; no global lock. +@@ -1615,6 +1626,7 @@ struct TransferEngineImpl::PreparedSubmit { + std::vector staging_params; + }; + ++ bool trace_internal{false}; + std::chrono::steady_clock::time_point submit_time{}; + LogicalTransferRuntimePolicy runtime_policy; + std::vector tasks; +@@ -2096,6 +2108,8 @@ Status TransferEngineImpl::commitPreparedSubmit( + batch->task_list.insert(batch->task_list.end(), prepared.tasks.size(), + TaskInfo{}); + ++ recordPreparedSubmit(batch, prepared); ++ + std::unordered_map owner_task_id_by_merged_task; + std::unordered_map> + public_tasks_by_physical_owner; +@@ -2341,6 +2355,8 @@ Status TransferEngineImpl::enqueuePreparedSubmit(Batch* batch, + task.derived = task_plan.task_id != owner.owner_task_id; + } + ++ recordPreparedSubmit(batch, prepared); ++ + for (size_t i = 0; i < admitted_owner_ids.size(); ++i) { + QueuedOwnerState queued; + queued.batch = batch; +@@ -2377,6 +2393,8 @@ Status TransferEngineImpl::finishQueuedOwner( + queued.in_dispatch_window = false; + } + for (const auto task_id : queued.public_task_ids) { ++ recordTraceCompletion(queued.batch->task_list[task_id], ++ terminal_status); + queued.batch->task_list[task_id].status = terminal_status; + } + queued_owners_.erase(queued_it); +@@ -2396,6 +2414,7 @@ Status TransferEngineImpl::cancelQueuedOwner(QueueOwnerId owner_id) { + for (const auto task_id : queued_it->second.public_task_ids) { + auto& task = queued_it->second.batch->task_list[task_id]; + task.cancel_requested = true; ++ recordTraceCompletion(task, CANCELED); + task.status = CANCELED; + } + queued_owners_.erase(queued_it); +@@ -2582,6 +2601,7 @@ Status TransferEngineImpl::submitTransfer( + const size_t start_task_id = batch_ref.get()->task_list.size(); + PreparedSubmit prepared; + CHECK_STATUS(prepareSubmit(batch_ref.get(), request_list, prepared)); ++ prepared.trace_internal = owner_kind == QueueOwnerKind::StagingInternal; + + if (shouldQueueSubmit(prepared, owner_kind)) { + CHECK_STATUS( +@@ -2723,6 +2743,7 @@ Status TransferEngineImpl::cancelTransfer(BatchID batch_id, size_t task_id) { + } + if (owner.type == UNSPEC) { + owner.cancel_requested = true; ++ recordTraceCompletion(owner, CANCELED); + owner.status = CANCELED; + return Status::OK(); + } +@@ -3418,9 +3439,60 @@ Status TransferEngineImpl::unlockStageBuffer(uint64_t addr) { + return staging_proxy_->unpinStageBuffer(addr); + } + ++void TransferEngineImpl::recordPreparedSubmit(Batch* batch, ++ const PreparedSubmit& prepared) { ++ if (!event_recorder_) return; ++ std::vector> requests( ++ prepared.owners.size()); ++ for (const auto& task : prepared.tasks) ++ requests[task.merged_task_index].push_back( ++ {task.task_id, task.public_length}); ++ for (size_t i = 0; i < prepared.owners.size(); ++i) { ++ const auto& owner = prepared.owners[i]; ++ const auto target = owner.request.target_id; ++ // Handle-to-name lookup only: recording must never fetch metadata. ++ const auto peer = ++ target == LOCAL_SEGMENT_ID ++ ? local_segment_name_ ++ : metadata_->segmentManager().lookupSegmentName(target); ++ auto span = event_recorder_->begin( ++ batch->trace_batch_id, target, peer, ++ owner.request.opcode == Request::READ ? "read" : "write", ++ prepared.trace_internal, std::move(requests[i])); ++ batch->task_list[owner.owner_task_id].trace_span = span; ++ for (auto id : owner.derived_task_ids) ++ batch->task_list[id].trace_span = span; ++ } ++} ++ ++void TransferEngineImpl::recordTraceCompletion(TaskInfo& task, ++ TransferStatusEnum status) { ++ if (!event_recorder_ || status == PENDING) return; ++ const char* name = "unknown"; ++ switch (status) { ++ case COMPLETED: ++ name = "completed"; ++ break; ++ case FAILED: ++ name = "failed"; ++ break; ++ case CANCELED: ++ name = "canceled"; ++ break; ++ case TIMEOUT: ++ name = "timeout"; ++ break; ++ default: ++ break; ++ } ++ event_recorder_->finish(task.trace_span, name, ++ transportTypeName(task.type)); ++} ++ + void TransferEngineImpl::recordTaskCompletionMetrics( + TaskInfo& task, TransferStatusEnum prev_status, + TransferStatusEnum new_status) { ++ recordTraceCompletion(task, new_status); + #if TENT_METRICS_ENABLED + if (prev_status == PENDING && new_status != PENDING && !task.derived) { + auto end_time = std::chrono::steady_clock::now(); +diff --git a/mooncake-transfer-engine/tent/tests/CMakeLists.txt b/mooncake-transfer-engine/tent/tests/CMakeLists.txt +index b7aef776..e0eef001 100644 +--- a/mooncake-transfer-engine/tent/tests/CMakeLists.txt ++++ b/mooncake-transfer-engine/tent/tests/CMakeLists.txt +@@ -808,3 +808,16 @@ target_include_directories(tent_segment_desc_retry_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include) + add_test(NAME tent_segment_desc_retry_test COMMAND tent_segment_desc_retry_test) + set_tests_properties(tent_segment_desc_retry_test PROPERTIES TIMEOUT 60) ++ ++# Optional event recorder: exercise actual engine merge/queue/terminal paths ++# even when aggregate metrics are compiled out. ++add_executable(tent_transfer_event_recorder_test transfer_event_recorder_test.cpp) ++target_link_libraries(tent_transfer_event_recorder_test ++ PRIVATE gtest gtest_main tent_link_group) ++if(TARGET asio_shared) ++ target_link_libraries(tent_transfer_event_recorder_test PRIVATE asio_shared) ++endif() ++target_include_directories(tent_transfer_event_recorder_test ++ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include) ++add_test(NAME tent_transfer_event_recorder_test ++ COMMAND tent_transfer_event_recorder_test) +diff --git a/mooncake-transfer-engine/tent/TRANSFER_EVENTS.md b/mooncake-transfer-engine/tent/TRANSFER_EVENTS.md +new file mode 100644 +index 00000000..88cf42dd +--- /dev/null ++++ b/mooncake-transfer-engine/tent/TRANSFER_EVENTS.md +@@ -0,0 +1,128 @@ ++# Transfer event recording ++ ++The TENT backend can record original Transfer Engine request descriptors at ++admission and observed completion. Recording is disabled by default and works ++with `TENT_METRICS_ENABLED=OFF`. The classic Transfer Engine backend is not ++instrumented by this recorder. ++ ++## Enable ++ ++Build TENT normally, create a directory, and set the environment before creating ++the engine: ++ ++```bash ++mkdir -p /path/to/events ++export TENT_TRACE_DIR=/path/to/events ++export TENT_TRACE_BUFFER_RECORDS=65536 # optional ++# Run your existing application or trace replay driver here. ++``` ++ ++Set this on every process that initiates transfers. No replay-driver changes ++are required. The directory must exist and be writable in the process/container. ++Each engine creates a private, unique ++`tent-events-HOST-PID-RANDOM.jsonl` file and prints its path to stderr. Multiple ++engines, processes and hosts can share the directory without overwriting files. ++An invalid setting or file-open failure prints a diagnostic and disables ++recording without preventing engine startup. ++ ++For the supplied two-node Slurm launcher, `TRACE=1` creates and exports a shared ++`events/` directory under the benchmark result directory. Rebuild before using ++this with previously built binaries. ++ ++## JSONL schema (version 1) ++ ++Each file starts with `trace_start` (initiator segment name, PID, buffer size). ++Every admitted original descriptor produces a `submit` record and, when its ++terminal state is observed, a `complete` record: ++ ++| Field | Meaning | ++| --- | --- | ++| `timestamp_ns` | Unix wall-clock nanoseconds, captured at the engine hook | ++| `monotonic_ns` | Local steady-clock nanoseconds; use for local ordering/durations | ++| `initiator`, `peer` | Local and target segment names, usually host:port with p2p metadata | ++| `target_id` | Engine-local segment handle; peer is empty if the handle is unknown | ++| `batch_id` | Recorder-assigned batch ID, never a memory address | ++| `request_id` | Recorder-assigned original descriptor ID | ++| `task_index` | Original descriptor index within the batch, including appended submissions | ++| `operation` | `read` or `write`, from the initiator's perspective | ++| `bytes` | Original requested payload bytes, even if merged with adjacent requests | ++| `scope` | `user` or `staging_internal` | ++| `merged_request_count` | Number of original descriptors sharing this owner; repeated on each descriptor | ++| `status` | Completion only: `completed`, `failed`, `canceled`, `timeout`, or `unknown` | ++| `transport` | Completion only: final engine transport attribution | ++| `duration_ns` | Completion only: local monotonic time since the admission record | ++ ++Join records by **file + request_id**; batch/request IDs are unique only within ++one engine's file. A read pulls bytes **peer → initiator**; a write pushes bytes ++**initiator → peer**. One-sided RDMA does not generate a matching target-side ++receive event at this boundary. The recorder does not capture NIC work requests, ++packet counts, transport attempts, or CPU instructions. ++ ++Requests rejected before admission produce no records. Submission timestamps ++are captured after validation/route preparation and before dispatch, so they ++exclude preparation time and include any admission-queue wait after acceptance. ++Completion timestamps indicate when the engine observes terminal status, ++including its polling delay, rather than NIC hardware completion time. ++ ++Merging preserves one record per original descriptor. Retries do not create ++additional logical submissions or completions. Polling a request repeatedly, ++polling a merged alias, and freeing a batch do not duplicate completions. ++Staging's internal child requests are tagged separately: filter `scope == ++"user"` for logical workload counts. Request/session IDs owned by the application ++are not available at this engine boundary; recorder request IDs are not replay ++request IDs. ++ ++On engine teardown, descriptors with no observed terminal state get `abandoned` ++records with `status: "unobserved"`; these are **not successful completions**. ++A clean recorder shutdown drains the queue and writes `trace_end` with ++`written_records` (submit/complete/abandoned records) and `dropped_records`. ++Require that footer and zero drops when checking a complete trace. A process ++killed abruptly can lose queued/buffered records and have no footer. ++ ++## Analyze alongside a separate replay driver ++ ++Keep the workload's scheduling, session affinity and timing in the driver. This ++recorder observes whatever that driver actually submits. For each time window: ++ ++- Count `submit` records with `scope == "user"` for the number of requested ++ transfer descriptors, grouped by initiator, peer and operation. ++- Sum their `bytes` for offered payload volume. Do not sum ++ `merged_request_count`: it repeats on each original descriptor. ++- Sum `bytes` on `complete` records with `status == "completed"` and user scope ++ for completed payload volume, then divide by the window duration for GB/s. ++ This assigns each entire request to its completion window; it does not measure ++ continuous wire utilization or partial bytes from failed requests. ++- Join submit/complete records to plot outstanding requests and local durations. ++ File order across submitting threads need not equal timestamp order; sort by ++ the clock appropriate to the analysis. ++ ++Use synchronized host wall clocks to compare events across nodes; the recorder ++does not synchronize them. Steady-clock values should only be compared locally. ++ ++## Overhead and limits ++ ++JSON serialization and file writes run on a background thread. Producers enqueue ++in-memory events through a short mutex-protected queue. There are no per-request ++file writes or metadata network lookups on the submission/polling threads. ++Tracing still adds allocations, clock reads, locks and CPU/I/O load; run without ++tracing when measuring baseline bandwidth. ++ ++The queue is bounded in original descriptor records (default 65536). The writer ++may hold one additional drained buffer of that size while formatting it. An ++event group that will not fit is dropped, without waiting for disk I/O. Drops ++are counted in the footer and reported to stderr on shutdown; write errors also ++emit diagnostics. Increase the buffer or reduce the trace workload if records ++are dropped. Files are flushed after draining and at least every 250 ms while ++idle, but are not fsynced and have no rotation or size limit. ++ ++## Tests ++ ++With `BUILD_UNIT_TESTS=ON`, build `tent_transfer_event_recorder_test` and run: ++ ++```bash ++ctest --test-dir /path/to/build -R '^tent_transfer_event_recorder_test$' --output-on-failure ++``` ++ ++The tests exercise the real engine with a fake transport (no RDMA/GPU hardware ++required), plus concurrent recording, overflow accounting and disabled/error ++configuration. They also run with aggregate metrics compiled out. +diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/transfer_event_recorder.h b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_event_recorder.h +new file mode 100644 +index 00000000..9b70c292 +--- /dev/null ++++ b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_event_recorder.h +@@ -0,0 +1,91 @@ ++// Copyright 2026 KVCache.AI ++// SPDX-License-Identifier: Apache-2.0 ++ ++#pragma once ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++namespace mooncake { ++namespace tent { ++ ++// One span per merged owner; each original API descriptor keeps its own ID. ++// Shared by derived tasks so polling an alias cannot duplicate completions. ++struct TransferEventSpan { ++ struct Request { ++ uint64_t task_index; ++ uint64_t bytes; ++ uint64_t request_id{0}; ++ }; ++ uint64_t batch_id{0}; ++ uint64_t target_id{0}; ++ uint64_t submit_wall_ns{0}; ++ uint64_t submit_mono_ns{0}; ++ std::string peer; ++ std::string operation; ++ bool internal{false}; ++ std::vector requests; ++ std::atomic finished{false}; ++}; ++ ++class TransferEventRecorder { ++ public: ++ // Unset/empty TENT_TRACE_DIR disables recording. Directory must exist. ++ // Failure to open a trace is reported but never fails engine startup. ++ static std::unique_ptr fromEnvironment( ++ const std::string& initiator); ++ static std::unique_ptr create( ++ const std::string& directory, const std::string& initiator, ++ size_t capacity = 65536); ++ ~TransferEventRecorder(); ++ ++ uint64_t nextBatchId() { return next_batch_id_.fetch_add(1); } ++ std::shared_ptr begin( ++ uint64_t batch_id, uint64_t target_id, std::string peer, ++ std::string operation, bool internal, ++ std::vector requests); ++ void finish(const std::shared_ptr& span, ++ const std::string& status, const std::string& transport); ++ const std::string& path() const { return path_; } ++ ++ private: ++ struct Event { ++ std::shared_ptr span; ++ uint64_t wall_ns; ++ uint64_t mono_ns; ++ std::string status; ++ std::string transport; ++ }; ++ TransferEventRecorder(FILE* file, std::string path, std::string initiator, ++ size_t capacity); ++ void enqueue(Event event); ++ void run(); ++ void writeLine(const std::string& line); ++ ++ FILE* file_; ++ const std::string path_; ++ const std::string initiator_; ++ const size_t capacity_; ++ std::atomic next_batch_id_{1}; ++ std::atomic next_request_id_{1}; ++ std::mutex mutex_; ++ std::condition_variable ready_; ++ std::deque queue_; ++ size_t queued_records_{0}; ++ uint64_t dropped_records_{0}; ++ uint64_t written_records_{0}; ++ bool stopping_{false}; ++ bool write_failed_{false}; ++ std::thread writer_; ++}; ++ ++} // namespace tent ++} // namespace mooncake +diff --git a/mooncake-transfer-engine/tent/src/runtime/transfer_event_recorder.cpp b/mooncake-transfer-engine/tent/src/runtime/transfer_event_recorder.cpp +new file mode 100644 +index 00000000..adc31471 +--- /dev/null ++++ b/mooncake-transfer-engine/tent/src/runtime/transfer_event_recorder.cpp +@@ -0,0 +1,234 @@ ++// Copyright 2026 KVCache.AI ++// SPDX-License-Identifier: Apache-2.0 ++ ++#include "tent/runtime/transfer_event_recorder.h" ++ ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "tent/thirdparty/nlohmann/json.h" ++ ++namespace mooncake { ++namespace tent { ++namespace { ++template ++uint64_t nowNs() { ++ return std::chrono::duration_cast( ++ Clock::now().time_since_epoch()) ++ .count(); ++} ++} // namespace ++ ++std::unique_ptr TransferEventRecorder::fromEnvironment( ++ const std::string& initiator) { ++ const char* directory = std::getenv("TENT_TRACE_DIR"); ++ if (!directory || !*directory) return nullptr; ++ size_t capacity = 65536; ++ const char* value = std::getenv("TENT_TRACE_BUFFER_RECORDS"); ++ if (value) { ++ try { ++ const std::string text(value); ++ size_t end = 0; ++ auto parsed = std::stoull(text, &end); ++ if (text.empty() || ++ text.find_first_not_of("0123456789") != std::string::npos || ++ end != text.size() || parsed == 0 || ++ parsed > std::numeric_limits::max()) ++ throw std::invalid_argument("invalid capacity"); ++ capacity = static_cast(parsed); ++ } catch (...) { ++ std::fprintf(stderr, ++ "TENT trace disabled: invalid " ++ "TENT_TRACE_BUFFER_RECORDS\n"); ++ return nullptr; ++ } ++ } ++ return create(directory, initiator, capacity); ++} ++ ++std::unique_ptr TransferEventRecorder::create( ++ const std::string& directory, const std::string& initiator, ++ size_t capacity) { ++ if (!capacity) return nullptr; ++ char host[256]{}; ++ if (gethostname(host, sizeof(host) - 1) != 0) host[0] = '\0'; ++ std::string pattern = directory + "/tent-events-" + host + "-" + ++ std::to_string(getpid()) + "-XXXXXX.jsonl"; ++ std::vector filename(pattern.begin(), pattern.end()); ++ filename.push_back('\0'); ++ int fd = mkstemps(filename.data(), 6); ++ if (fd < 0) { ++ std::fprintf(stderr, "TENT trace disabled: cannot create %s\n", ++ pattern.c_str()); ++ return nullptr; ++ } ++ FILE* file = fdopen(fd, "w"); ++ if (!file) { ++ close(fd); ++ unlink(filename.data()); ++ std::fprintf(stderr, "TENT trace disabled: fdopen failed\n"); ++ return nullptr; ++ } ++ try { ++ auto recorder = ++ std::unique_ptr(new TransferEventRecorder( ++ file, filename.data(), initiator, capacity)); ++ std::fprintf(stderr, "TENT transfer events: %s\n", filename.data()); ++ return recorder; ++ } catch (const std::exception& e) { ++ fclose(file); ++ std::fprintf(stderr, "TENT trace disabled: %s\n", e.what()); ++ return nullptr; ++ } ++} ++ ++TransferEventRecorder::TransferEventRecorder(FILE* file, std::string path, ++ std::string initiator, ++ size_t capacity) ++ : file_(file), ++ path_(std::move(path)), ++ initiator_(std::move(initiator)), ++ capacity_(capacity), ++ writer_(&TransferEventRecorder::run, this) {} ++ ++TransferEventRecorder::~TransferEventRecorder() { ++ { ++ std::lock_guard lock(mutex_); ++ stopping_ = true; ++ } ++ ready_.notify_one(); ++ writer_.join(); ++ if (fclose(file_) != 0) write_failed_ = true; ++ if (dropped_records_ || write_failed_) ++ std::fprintf(stderr, ++ "TENT trace incomplete: %llu dropped records, " ++ "write_failed=%d, file=%s\n", ++ static_cast(dropped_records_), ++ write_failed_, path_.c_str()); ++} ++ ++std::shared_ptr TransferEventRecorder::begin( ++ uint64_t batch_id, uint64_t target_id, std::string peer, ++ std::string operation, bool internal, ++ std::vector requests) { ++ auto span = std::make_shared(); ++ span->batch_id = batch_id; ++ span->target_id = target_id; ++ span->peer = std::move(peer); ++ span->operation = std::move(operation); ++ span->internal = internal; ++ span->requests = std::move(requests); ++ for (auto& request : span->requests) ++ request.request_id = next_request_id_.fetch_add(1); ++ span->submit_wall_ns = nowNs(); ++ span->submit_mono_ns = nowNs(); ++ enqueue({span, span->submit_wall_ns, span->submit_mono_ns, "", ""}); ++ return span; ++} ++ ++void TransferEventRecorder::finish( ++ const std::shared_ptr& span, const std::string& status, ++ const std::string& transport) { ++ if (!span || span->finished.exchange(true)) return; ++ enqueue({span, nowNs(), ++ nowNs(), status, transport}); ++} ++ ++void TransferEventRecorder::enqueue(Event event) { ++ { ++ std::lock_guard lock(mutex_); ++ const auto count = event.span->requests.size(); ++ if (count > capacity_ - queued_records_) { ++ dropped_records_ += count; ++ return; ++ } ++ queued_records_ += count; ++ queue_.push_back(std::move(event)); ++ } ++ ready_.notify_one(); ++} ++ ++void TransferEventRecorder::writeLine(const std::string& line) { ++ if (write_failed_) return; ++ if (fwrite(line.data(), 1, line.size(), file_) != line.size() || ++ fputc('\n', file_) == EOF) { ++ write_failed_ = true; ++ std::fprintf(stderr, "TENT trace write failed: %s\n", path_.c_str()); ++ } ++} ++ ++void TransferEventRecorder::run() { ++ using nlohmann::json; ++ try { ++ writeLine(json({{"event", "trace_start"}, ++ {"schema_version", 1}, ++ {"initiator", initiator_}, ++ {"pid", getpid()}, ++ {"buffer_records", capacity_}}) ++ .dump()); ++ for (;;) { ++ std::deque events; ++ bool stop; ++ { ++ std::unique_lock lock(mutex_); ++ ready_.wait_for(lock, std::chrono::milliseconds(250), [this] { ++ return stopping_ || !queue_.empty(); ++ }); ++ events.swap(queue_); ++ queued_records_ = 0; ++ stop = stopping_; ++ } ++ for (const auto& event : events) { ++ const auto& span = *event.span; ++ const bool complete = !event.status.empty(); ++ for (const auto& request : span.requests) { ++ json record = { ++ {"schema_version", 1}, ++ {"event", ++ complete ? (event.status == "unobserved" ? "abandoned" ++ : "complete") ++ : "submit"}, ++ {"timestamp_ns", event.wall_ns}, ++ {"monotonic_ns", event.mono_ns}, ++ {"initiator", initiator_}, ++ {"peer", span.peer}, ++ {"target_id", span.target_id}, ++ {"batch_id", span.batch_id}, ++ {"request_id", request.request_id}, ++ {"task_index", request.task_index}, ++ {"operation", span.operation}, ++ {"bytes", request.bytes}, ++ {"scope", span.internal ? "staging_internal" : "user"}, ++ {"merged_request_count", span.requests.size()}}; ++ if (complete) { ++ record["status"] = event.status; ++ record["transport"] = event.transport; ++ record["duration_ns"] = ++ event.mono_ns - span.submit_mono_ns; ++ } ++ writeLine(record.dump(-1, ' ', false, ++ json::error_handler_t::replace)); ++ if (!write_failed_) ++written_records_; ++ } ++ } ++ if (fflush(file_) != 0) write_failed_ = true; ++ if (stop) break; ++ } ++ writeLine(json({{"event", "trace_end"}, ++ {"schema_version", 1}, ++ {"written_records", written_records_}, ++ {"dropped_records", dropped_records_}}) ++ .dump()); ++ } catch (const std::exception& e) { ++ write_failed_ = true; ++ std::fprintf(stderr, "TENT trace writer failed: %s\n", e.what()); ++ } ++} ++ ++} // namespace tent ++} // namespace mooncake +diff --git a/mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp b/mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp +new file mode 100644 +index 00000000..51f64fcd +--- /dev/null ++++ b/mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp +@@ -0,0 +1,510 @@ ++// Copyright 2026 KVCache.AI ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include ++#include ++#include ++#include ++#include ++#include "tent/thirdparty/nlohmann/json.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "tent/common/config.h" ++#include "tent/common/types.h" ++#if TENT_METRICS_ENABLED ++#include "tent/metrics/tent_metrics.h" ++#endif ++#include "tent/runtime/transfer_engine_impl.h" ++#include "tent/runtime/transport.h" ++ ++namespace mooncake { ++namespace tent { ++namespace { ++ ++class FakeSubBatch : public Transport::SubBatch { ++ public: ++ size_t size() const override { return task_count; } ++ ++ size_t task_count = 0; ++ std::vector requests; ++ std::vector statuses; ++}; ++ ++class FakeTransport : public Transport { ++ public: ++ explicit FakeTransport(TransportType self_type) : self_type_(self_type) { ++ caps.dram_to_dram = true; ++ } ++ ++ std::atomic submit_calls{0}; ++ std::atomic outcome{COMPLETED}; ++ bool fail_submit{false}; ++ ++ Status install(std::string&, std::shared_ptr, ++ std::shared_ptr, ++ std::shared_ptr) override { ++ return Status::OK(); ++ } ++ ++ Status allocateSubBatch(SubBatchRef& batch, size_t) override { ++ batch = new FakeSubBatch(); ++ return Status::OK(); ++ } ++ ++ Status freeSubBatch(SubBatchRef& batch) override { ++ delete static_cast(batch); ++ batch = nullptr; ++ return Status::OK(); ++ } ++ ++ Status submitTransferTasks(SubBatchRef batch, ++ const std::vector& requests) override { ++ ++submit_calls; ++ if (fail_submit) ++ return Status::InternalError("injected submit failure"); ++ auto* fake = static_cast(batch); ++ for (const auto& request : requests) { ++ fake->requests.push_back(request); ++ fake->statuses.push_back( ++ {TransferStatusEnum::COMPLETED, request.length}); ++ ++fake->task_count; ++ } ++ batch->notifyProgress(); ++ return Status::OK(); ++ } ++ ++ Status getTransferStatus(SubBatchRef batch, int task_id, ++ TransferStatus& status) override { ++ auto* fake = static_cast(batch); ++ if (task_id < 0 || task_id >= (int)fake->statuses.size()) { ++ return Status::InvalidArgument("bad task_id" LOC_MARK); ++ } ++ status = fake->statuses[task_id]; ++ status.s = outcome.load(); ++ return Status::OK(); ++ } ++ ++ Status addMemoryBuffer(BufferDesc& desc, const MemoryOptions&) override { ++ desc.transports.push_back(self_type_); ++ return Status::OK(); ++ } ++ ++ Status addMemoryBuffer(std::vector& desc_list, ++ const MemoryOptions& options) override { ++ for (auto& desc : desc_list) { ++ auto s = addMemoryBuffer(desc, options); ++ if (!s.ok()) return s; ++ } ++ return Status::OK(); ++ } ++ ++ Status removeMemoryBuffer(BufferDesc&) override { return Status::OK(); } ++ ++ Status allocateLocalMemory(void** addr, size_t size, ++ MemoryOptions&) override { ++ *addr = std::malloc(size); ++ return *addr ? Status::OK() ++ : Status::InternalError("malloc failed" LOC_MARK); ++ } ++ ++ Status freeLocalMemory(void* addr, size_t) override { ++ std::free(addr); ++ return Status::OK(); ++ } ++ ++ bool warmupMemory(void*, size_t) override { return false; } ++ ++ const char* getName() const override { return ""; } ++ ++ private: ++ TransportType self_type_; ++}; ++ ++std::shared_ptr makeTraceConfig(size_t max_dispatch_owners, ++ size_t max_dispatch_bytes) { ++ auto cfg = std::make_shared(); ++ cfg->set("metadata_type", "p2p"); ++ cfg->set("metadata_servers", ""); ++ cfg->set("rpc_server_hostname", "127.0.0.1"); ++ cfg->set("rpc_server_port", "0"); ++ cfg->set("log_level", "warning"); ++ cfg->set("merge_requests", false); ++ cfg->set("enable_runtime_queue", true); ++ cfg->set("runtime_queue/max_outstanding_owners", 16UL); ++ cfg->set("runtime_queue/max_outstanding_bytes", 1UL << 20); ++ cfg->set("runtime_queue/max_dispatch_owners", max_dispatch_owners); ++ cfg->set("runtime_queue/max_dispatch_bytes", max_dispatch_bytes); ++ cfg->set("runtime_queue/staging_owner_reserve", 0UL); ++ cfg->set("runtime_queue/staging_byte_reserve", 0UL); ++ cfg->set("runtime_queue/progress_fallback_interval_us", 50000UL); ++ ++ cfg->set("transports/tcp/enable", false); ++ cfg->set("transports/shm/enable", false); ++ cfg->set("transports/rdma/enable", false); ++ cfg->set("transports/io_uring/enable", false); ++ cfg->set("transports/nvlink/enable", false); ++ cfg->set("transports/mnnvl/enable", false); ++ cfg->set("transports/gds/enable", false); ++ cfg->set("transports/ascend_direct/enable", false); ++ return cfg; ++} ++ ++void installFakeRdma(TransferEngineImpl& engine, ++ const std::shared_ptr& fake) { ++ std::string seg_name = engine.getSegmentName(); ++ ASSERT_TRUE(fake->install(seg_name, nullptr, nullptr, nullptr).ok()); ++ engine.swapTransportForTest(RDMA, fake); ++} ++ ++Request makeLocalWrite(uint8_t* ptr, size_t length) { ++ Request request; ++ request.opcode = Request::WRITE; ++ request.source = ptr; ++ request.target_id = LOCAL_SEGMENT_ID; ++ request.target_offset = reinterpret_cast(ptr); ++ request.length = length; ++ request.transport_hint = RDMA; ++ return request; ++} ++ ++using nlohmann::json; ++ ++class TransferEventTest : public ::testing::Test { ++ protected: ++ std::string directory; ++ std::string old_directory; ++ std::string old_capacity; ++ bool had_directory{false}; ++ bool had_capacity{false}; ++ ++ void SetUp() override { ++ char path[] = "/tmp/tent-event-test-XXXXXX"; ++ ASSERT_NE(mkdtemp(path), nullptr); ++ directory = path; ++ if (auto value = getenv("TENT_TRACE_DIR")) { ++ had_directory = true; ++ old_directory = value; ++ } ++ if (auto value = getenv("TENT_TRACE_BUFFER_RECORDS")) { ++ had_capacity = true; ++ old_capacity = value; ++ } ++ setenv("TENT_TRACE_DIR", directory.c_str(), 1); ++ unsetenv("TENT_TRACE_BUFFER_RECORDS"); ++ } ++ void TearDown() override { ++ if (had_directory) ++ setenv("TENT_TRACE_DIR", old_directory.c_str(), 1); ++ else ++ unsetenv("TENT_TRACE_DIR"); ++ if (had_capacity) ++ setenv("TENT_TRACE_BUFFER_RECORDS", old_capacity.c_str(), 1); ++ else ++ unsetenv("TENT_TRACE_BUFFER_RECORDS"); ++ std::filesystem::remove_all(directory); ++ } ++ std::vector records() { ++ std::vector result; ++ for (const auto& file : ++ std::filesystem::directory_iterator(directory)) { ++ std::ifstream input(file.path()); ++ std::string line; ++ while (std::getline(input, line)) ++ result.push_back(json::parse(line)); ++ } ++ return result; ++ } ++ void checkPairs(const std::vector& rows, size_t count, ++ const std::string& status) { ++ std::map submitted, completed; ++ for (const auto& row : rows) { ++ if (row["event"] == "submit") { ++ EXPECT_TRUE(submitted.emplace(row["request_id"], row).second); ++ } else if (row["event"] == "complete") { ++ EXPECT_TRUE(completed.emplace(row["request_id"], row).second); ++ EXPECT_EQ(row["status"], status); ++ } ++ } ++ ASSERT_EQ(submitted.size(), count); ++ ASSERT_EQ(completed.size(), count); ++ for (const auto& [id, begin] : submitted) { ++ ASSERT_EQ(completed.count(id), 1); ++ const auto& end = completed.at(id); ++ EXPECT_EQ(begin["bytes"], end["bytes"]); ++ EXPECT_EQ(begin["batch_id"], end["batch_id"]); ++ EXPECT_EQ(begin["task_index"], end["task_index"]); ++ EXPECT_GE(end["monotonic_ns"].get(), ++ begin["monotonic_ns"].get()); ++ EXPECT_EQ(end["duration_ns"].get(), ++ end["monotonic_ns"].get() - ++ begin["monotonic_ns"].get()); ++ } ++ } ++}; ++ ++TEST_F(TransferEventTest, DisabledAndInvalidConfiguration) { ++ unsetenv("TENT_TRACE_DIR"); ++ EXPECT_EQ(TransferEventRecorder::fromEnvironment("a"), nullptr); ++ setenv("TENT_TRACE_DIR", directory.c_str(), 1); ++ setenv("TENT_TRACE_BUFFER_RECORDS", "-1", 1); ++ EXPECT_EQ(TransferEventRecorder::fromEnvironment("a"), nullptr); ++ EXPECT_EQ(TransferEventRecorder::create(directory + "/missing", "a"), ++ nullptr); ++ EXPECT_TRUE(records().empty()); ++} ++ ++TEST_F(TransferEventTest, ConcurrentWritersAndDuplicateCompletion) { ++ auto recorder = TransferEventRecorder::create(directory, "node\"a\n"); ++ ASSERT_NE(recorder, nullptr); ++ std::vector threads; ++ for (int t = 0; t < 8; ++t) { ++ threads.emplace_back([&] { ++ auto batch = recorder->nextBatchId(); ++ for (int i = 0; i < 50; ++i) { ++ auto span = recorder->begin(batch, 7, "node-b", "read", false, ++ {{static_cast(i), 4096}}); ++ recorder->finish(span, "completed", "rdma"); ++ recorder->finish(span, "failed", "tcp"); ++ } ++ }); ++ } ++ for (auto& thread : threads) thread.join(); ++ recorder.reset(); ++ auto rows = records(); ++ checkPairs(rows, 400, "completed"); ++ EXPECT_EQ(rows.back()["dropped_records"], 0); ++ EXPECT_EQ(rows.back()["written_records"], 800); ++} ++ ++TEST_F(TransferEventTest, OverflowIsExplicitAndFilesAreUnique) { ++ auto recorder = TransferEventRecorder::create(directory, "a", 1); ++ auto other = TransferEventRecorder::create(directory, "a", 1); ++ ASSERT_NE(recorder, nullptr); ++ ASSERT_NE(other, nullptr); ++ EXPECT_NE(recorder->path(), other->path()); ++ auto path = recorder->path(); ++ auto span = recorder->begin(1, 2, "b", "write", false, {{0, 8}, {1, 8}}); ++ recorder->finish(span, "completed", "rdma"); ++ recorder.reset(); ++ std::ifstream input(path); ++ json end; ++ std::string line; ++ while (std::getline(input, line)) end = json::parse(line); ++ EXPECT_EQ(end["event"], "trace_end"); ++ EXPECT_EQ(end["dropped_records"], 4); ++ EXPECT_EQ(end["written_records"], 0); ++} ++ ++TEST_F(TransferEventTest, MergedDescriptorsAndAppendKeepOriginalSizes) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(4, 1UL << 20); ++ cfg->set("enable_runtime_queue", false); ++ cfg->set("merge_requests", true); ++ TransferEngineImpl engine(cfg); ++ ASSERT_TRUE(engine.available()); ++ auto fake = std::make_shared(RDMA); ++ installFakeRdma(engine, fake); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto batch = engine.allocateBatch(4); ++ auto a = makeLocalWrite(buf.data(), 128); ++ auto b = makeLocalWrite(buf.data() + 128, 256); ++ ASSERT_TRUE(engine.submitTransfer(batch, {a, b}).ok()); ++ TransferStatus status; ++ ASSERT_TRUE(engine.getTransferStatus(batch, 1, status).ok()); ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ ASSERT_EQ(status.s, COMPLETED); ++ a.opcode = Request::READ; ++ ASSERT_TRUE(engine.submitTransfer(batch, {a}).ok()); ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ ASSERT_EQ(status.s, COMPLETED); ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ ASSERT_TRUE(engine.freeBatch(batch).ok()); ++ ASSERT_TRUE(engine.unregisterLocalMemory(buf.data(), buf.size()).ok()); ++ } ++ auto rows = records(); ++ checkPairs(rows, 3, "completed"); ++ size_t merged = 0, reads = 0; ++ for (const auto& row : rows) { ++ if (row["event"] != "submit") continue; ++ EXPECT_EQ(row["scope"], "user"); ++ EXPECT_EQ(row["peer"], row["initiator"]); ++ if (row["operation"] == "read") ++reads; ++ if (row["merged_request_count"] == 2) ++merged; ++ } ++ EXPECT_EQ(merged, 2); ++ EXPECT_EQ(reads, 1); ++} ++ ++TEST_F(TransferEventTest, QueueCompletionAndQueuedCancellation) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(1, 1UL << 20); ++ TransferEngineImpl engine(cfg); ++ ASSERT_TRUE(engine.available()); ++ auto fake = std::make_shared(RDMA); ++ fake->outcome = PENDING; ++ installFakeRdma(engine, fake); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto batch = engine.allocateBatch(2); ++ ASSERT_TRUE( ++ engine ++ .submitTransfer(batch, {makeLocalWrite(buf.data(), 128), ++ makeLocalWrite(buf.data() + 128, 128)}) ++ .ok()); ++ ASSERT_TRUE(engine.cancelTransfer(batch, 1).ok()); ++ fake->outcome = COMPLETED; ++ TransferStatus status; ++ for (int i = 0; i < 1000; ++i) { ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ if (status.s != PENDING) break; ++ std::this_thread::sleep_for(std::chrono::milliseconds(1)); ++ } ++ EXPECT_EQ(status.s, CANCELED); ++ ASSERT_TRUE(engine.freeBatch(batch).ok()); ++ ASSERT_TRUE(engine.unregisterLocalMemory(buf.data(), buf.size()).ok()); ++ } ++ auto rows = records(); ++ size_t canceled = 0, completed = 0; ++ for (const auto& row : rows) { ++ if (row["event"] != "complete") continue; ++ if (row["status"] == "canceled") ++canceled; ++ if (row["status"] == "completed") ++completed; ++ } ++ EXPECT_EQ(canceled, 1); ++ EXPECT_EQ(completed, 1); ++} ++ ++TEST_F(TransferEventTest, SubmitFailureIsRecordedWithoutMetrics) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(4, 1UL << 20); ++ cfg->set("enable_runtime_queue", false); ++ TransferEngineImpl engine(cfg); ++ auto fake = std::make_shared(RDMA); ++ fake->fail_submit = true; ++ installFakeRdma(engine, fake); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto batch = engine.allocateBatch(1); ++ ASSERT_TRUE( ++ engine.submitTransfer(batch, {makeLocalWrite(buf.data(), 128)}) ++ .ok()); ++ TransferStatus status; ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ EXPECT_EQ(status.s, FAILED); ++ ASSERT_TRUE(engine.freeBatch(batch).ok()); ++ ASSERT_TRUE(engine.unregisterLocalMemory(buf.data(), buf.size()).ok()); ++ } ++ checkPairs(records(), 1, "failed"); ++} ++ ++TEST_F(TransferEventTest, RejectionAndUnobservedTeardown) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(4, 1UL << 20); ++ cfg->set("enable_runtime_queue", false); ++ TransferEngineImpl engine(cfg); ++ auto fake = std::make_shared(RDMA); ++ installFakeRdma(engine, fake); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto rejected = engine.allocateBatch(0); ++ auto request = makeLocalWrite(buf.data(), 128); ++ EXPECT_FALSE(engine.submitTransfer(rejected, {request}).ok()); ++ EXPECT_TRUE(engine.freeBatch(rejected).ok()); ++ auto unpolled = engine.allocateBatch(1); ++ ASSERT_TRUE(engine.submitTransfer(unpolled, {request}).ok()); ++ // Fake transport completed, but the engine has never observed it. ++ // Teardown must not claim success on its behalf. ++ } ++ auto rows = records(); ++ size_t submissions = 0, abandoned = 0, completed = 0; ++ for (const auto& row : rows) { ++ if (row["event"] == "submit") ++submissions; ++ if (row["event"] == "abandoned") { ++ ++abandoned; ++ EXPECT_EQ(row["status"], "unobserved"); ++ } ++ if (row["event"] == "complete") ++completed; ++ } ++ EXPECT_EQ(submissions, 1); ++ EXPECT_EQ(abandoned, 1); ++ EXPECT_EQ(completed, 0); ++} ++ ++TEST_F(TransferEventTest, QueuedSubmitFailureRecordedOnce) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(4, 1UL << 20); ++ TransferEngineImpl engine(cfg); ++ auto fake = std::make_shared(RDMA); ++ fake->fail_submit = true; ++ installFakeRdma(engine, fake); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto batch = engine.allocateBatch(1); ++ ASSERT_TRUE( ++ engine.submitTransfer(batch, {makeLocalWrite(buf.data(), 128)}) ++ .ok()); ++ TransferStatus status; ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ EXPECT_EQ(status.s, FAILED); ++ ASSERT_TRUE(engine.freeBatch(batch).ok()); ++ ASSERT_TRUE(engine.unregisterLocalMemory(buf.data(), buf.size()).ok()); ++ } ++ checkPairs(records(), 1, "failed"); ++} ++ ++TEST_F(TransferEventTest, FailoverKeepsOneLogicalRequest) { ++ std::vector buf(4096); ++ { ++ auto cfg = makeTraceConfig(4, 1UL << 20); ++ cfg->set("enable_runtime_queue", false); ++ TransferEngineImpl engine(cfg); ++ auto rdma = std::make_shared(RDMA); ++ rdma->outcome = FAILED; ++ installFakeRdma(engine, rdma); ++ auto tcp = std::make_shared(TCP); ++ engine.swapTransportForTest(TCP, tcp); ++ ASSERT_TRUE(engine.registerLocalMemory(buf.data(), buf.size()).ok()); ++ auto batch = engine.allocateBatch(1); ++ auto request = makeLocalWrite(buf.data(), 128); ++ request.transport_hint = UNSPEC; ++ ASSERT_TRUE(engine.submitTransfer(batch, {request}).ok()); ++ TransferStatus status; ++ for (int i = 0; i < 1000; ++i) { ++ ASSERT_TRUE(engine.getTransferStatus(batch, status).ok()); ++ if (status.s != PENDING) break; ++ } ++ EXPECT_EQ(status.s, COMPLETED); ++ EXPECT_EQ(rdma->submit_calls.load(), 1); ++ EXPECT_EQ(tcp->submit_calls.load(), 1); ++ ASSERT_TRUE(engine.freeBatch(batch).ok()); ++ ASSERT_TRUE(engine.unregisterLocalMemory(buf.data(), buf.size()).ok()); ++ } ++ auto rows = records(); ++ checkPairs(rows, 1, "completed"); ++ for (const auto& row : rows) ++ if (row["event"] == "complete") EXPECT_EQ(row["transport"], "tcp"); ++} ++ ++} // namespace ++} // namespace tent ++} // namespace mooncake diff --git a/experimental/mooncake-transfer-traces/patches/0002-tebench-slurm.patch b/experimental/mooncake-transfer-traces/patches/0002-tebench-slurm.patch new file mode 100644 index 0000000000..b1975d212f --- /dev/null +++ b/experimental/mooncake-transfer-traces/patches/0002-tebench-slurm.patch @@ -0,0 +1,506 @@ +diff --git a/mooncake-transfer-engine/benchmark/slurm/README.md b/mooncake-transfer-engine/benchmark/slurm/README.md +new file mode 100644 +index 00000000..b03949c6 +--- /dev/null ++++ b/mooncake-transfer-engine/benchmark/slurm/README.md +@@ -0,0 +1,200 @@ ++# Two-node tebench RDMA sweep ++ ++`tebench.sbatch` runs a target on one node and an initiator on another, using ++TENT RDMA with peer-to-peer metadata (no etcd or Store master). It discovers the ++endpoint from the target's startup output, waits up to 120 seconds, runs the ++sweep, and terminates its own target step when finished or interrupted. ++ ++## Node selection ++ ++The default allocation uses **two H100 nodes**: `slurm-h100-206-025` and ++`slurm-h100-206-035`, in partition `h100`, account `cw-sup`. The first node is ++the initiator; the second is the target. Each node reserves one GPU, eight CPUs ++and 64 GiB host RAM for up to two hours (including compilation). Busy nodes queue normally; the script ++does not touch existing jobs. ++ ++Override the node list and partition using sbatch options before the script ++path. `INITIATOR_NODE` can select either allocated node to reverse the roles. ++The inspected cluster has only one H200 node, so it cannot currently provide an ++H200-to-H200 comparison. ++ ++## Build and submit ++ ++Submit directly from the Mooncake root; no CMake is needed on the login node: ++ ++```bash ++sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++``` ++ ++The script builds on the CPU cores of the initiator node inside a writable ++Pyxis container. The base image is `ubuntu:22.04`; it binds the compute node's ++existing CUDA 13 toolkit into the container instead of downloading a large CUDA ++development image. It installs CMake, GCC and the required development packages ++inside the container, initializes the pybind11 submodule if missing, and builds ++only `tebench` with CUDA/TENT enabled. Store, EP, examples and tests are disabled. ++No host system packages are changed. First-time setup needs registry, Ubuntu ++package-mirror and GitHub access from the compute node. ++ ++The compiled files go into shared `build/tebench-pyxis/`. The populated container ++is saved as `runtime.sqsh` there, and **both benchmark peers run in that image** ++so their libraries match the build. The launcher supplies the CUDA and TENT ++shared-library paths and binds `/dev/infiniband` directly. It disables this ++cluster's optional Mellanox remount hook, which fails on absent uverbs devices; ++GPU access still follows Slurm cgroups and `CUDA_VISIBLE_DEVICES`. Subsequent builds reuse that image and the ++CMake build tree. A lock protects the build directory throughout the benchmark; ++use separate `BUILD_DIR` values for concurrent independent jobs. ++ ++To build first without reserving GPUs or a second node: ++ ++```bash ++BUILD_ONLY=1 sbatch --nodes=1 --nodelist=slurm-h100-206-025 --gpus-per-node=0 \ ++ mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++ ++# After the build job completes, run the two-node benchmark without rebuilding. ++BUILD=0 sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++``` ++ ++This cluster exposes no CPU-only partition; the build-only command uses CPU ++resources on an H100 host and requests zero GPUs. The default combined job ++reserves its two benchmark GPUs for the whole allocation, including build time. ++ ++Export an absolute `MOONCAKE_ROOT` when submitting from elsewhere. The checkout, ++build directory, saved image and results must be shared between nodes. Override ++`CUDA_HOST_PATH` if the toolkit lives elsewhere; `BUILD_IMAGE` can be another ++Ubuntu-compatible base or an existing squashfs image. A custom `TEBENCH_BIN` ++requires `BUILD=0` and a compatible `RUNTIME_IMAGE`. ++ ++Examples (from the repository root): ++ ++```bash ++# Default: H100 initiator, H100 target, GPU-memory RDMA. ++sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++ ++# Reverse initiator and target on the same two nodes. ++INITIATOR_NODE=slurm-h100-206-035 sbatch \ ++ mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++ ++# Host-memory RDMA; allocation still reserves one GPU per node. ++SEG_TYPE=DRAM sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++ ++# Longer write-only sweep: 1 KiB through 128 MiB, fixed batching/concurrency. ++MIN_BYTES=1024 MAX_BYTES=134217728 BUFFER_BYTES=1073741824 \ ++BATCH_SIZE=4 THREADS=2 DURATION=10 OPS=write \ ++ sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++ ++# Short launch check: two sizes, one repetition, one second per size. ++MIN_BYTES=4096 MAX_BYTES=8192 REPEATS=1 DURATION=1 OPS=write \ ++ sbatch mooncake-transfer-engine/benchmark/slurm/tebench.sbatch ++``` ++ ++## Sweep and controls ++ ++The default is **4 KiB, 8 KiB, ... 64 MiB** (15 message sizes), fixed batch size ++1 and one sending thread. Read and write sweeps each run three times; each size ++has five seconds of measurement plus tebench's warmup. Expected benchmark time ++is roughly nine minutes plus startup, excluding the build. No automatic batch/concurrency sweep is ++mixed into the message-size comparison. ++ ++| Variable | Default | Meaning | ++| --- | --- | --- | ++| `SEG_TYPE` | `VRAM` | GPU memory; use `DRAM` for host-memory RDMA | ++| `MIN_BYTES`, `MAX_BYTES` | `4096`, `67108864` | Doubling sweep endpoints, in bytes | ++| `BUFFER_BYTES` | `268435456` | Registered buffer size; must cover maximum message × batch × threads | ++| `BATCH_SIZE`, `THREADS` | `1`, `1` | Fixed submission batch and concurrency | ++| `OPS` | `read write` | One or both operations, space separated | ++| `DURATION` | `5` | Measured seconds per size | ++| `REPEATS` | `3` | Repetitions of each operation's sweep | ++| `READY_TIMEOUT` | `120` | Target startup limit in seconds | ++| `INITIATOR_NODE` | First allocated node | Initiator host | ++| `BUILD`, `BUILD_ONLY` | `1`, `0` | Build before running; optionally exit after building | ++| `BUILD_DIR` | `$MOONCAKE_ROOT/build/tebench-pyxis` | Shared incremental build and container cache | ++| `BUILD_JOBS` | Allocated CPUs (8) | Compiler parallelism, no more than allocated CPUs | ++| `BUILD_IMAGE` | `ubuntu:22.04` | Initial Pyxis build image; cached runtime takes precedence | ++| `RUNTIME_IMAGE` | `$BUILD_DIR/runtime.sqsh` | Saved build environment used by both peers | ++| `CUDA_HOST_PATH`, `CUDA_HOME` | `/usr/local/cuda-13.0`, `/usr/local/cuda` | Host toolkit path and container mount destination | ++| `TEBENCH_BIN` | `$BUILD_DIR/mooncake-transfer-engine/benchmark/tebench` | Executable | ++| `RESULT_ROOT` | `$MOONCAKE_ROOT/benchmark-results` | Shared result directory; do not use node-local `/tmp` | ++| `PYTHON` | `python3` | Standard-library-only result validation | ++ ++`MAX_BYTES / MIN_BYTES` must be a power of two. Numeric controls are positive ++decimal integers, capped at 2^31-1 by the launcher. Increase Slurm's `--time` ++when expanding repetitions/durations. For controlled placement, configure the ++TENT engine with `MC_TENT_CONF` before submission; Slurm GPU visibility is ++preserved and each process uses its visible GPU 0. GPU RDMA requires working ++GPUDirect registration; the launcher reports failures instead of falling back ++to CPU memory or another transport. ++ ++## Results ++ ++A unique `benchmark-results/tebench-JOBID.XXXXXX/` directory contains: ++ ++- `build.log` when compilation is enabled. ++- `target.log`, `endpoint.txt` and the exact target command. ++- `read-N.log` / `write-N.log`, matching command files and JSONL measurements. ++- Allocation, commit, role mapping, GPU/NIC topology, locked-memory limit and ++ per-node shared-library checks. ++ ++Each JSONL file contains one record per size, including `block_size`, ++`aggregate_throughput_gbps` (decimal **GB/s**, despite the field spelling), and ++per-target mean/p99/p999 transfer latency. The launcher rejects missing sizes, ++empty measurements and failed initiator exits. Compare repeated runs rather ++than treating a single observation as peak fabric throughput. ++ ++A read pulls target memory into the initiator; a write pushes initiator memory ++to the target. These are separate one-direction measurements, not simultaneous ++bidirectional bandwidth. With a single thread and batch size 1, small messages ++primarily expose latency and submission overhead; increase fixed batching or ++concurrency in a separate run to explore saturation. A file supplied through `MC_TENT_CONF` must be inside a mounted directory. ++GPU choice, NIC selection ++and other jobs on either host can affect results. ++ ++## Validation performed ++ ++- Job 1608 built `tebench` successfully in Pyxis on one H100 host using eight ++ CPUs and **zero GPUs**, in 4 minutes 13 seconds. It produced the executable ++ under `build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench` and a ++ roughly 488 MiB `runtime.sqsh` image. ++- Job 1610 reused that image on both H100 nodes and completed GPU-memory RDMA ++ reads and writes at 4 KiB and 8 KiB, one repetition, one measured second per ++ size. Job status was `COMPLETED`, exit `0:0`; all four JSON records were ++ validated. The long-lived target step is intentionally cancelled at cleanup. ++ This smoke test validates the build/runtime path, not peak bandwidth or the ++ full default size sweep. ++- Logs for these validation jobs are in the shared workspace's ++ `mooncake-build-results/` directory. ++- Shell syntax and whitespace checks passed. Temporary stand-ins also verified ++ build-only success/failure, preservation of the old image after a failed ++ build, sweep completion, initiator exit-code propagation, startup timeout, ++ incomplete-result rejection and target cleanup. ++- `pre-commit` was unavailable in the environment. ++ ++## Optional per-request event traces ++ ++After rebuilding the modified TENT backend, use `TRACE=1` to record original ++request submissions and observed completions from both processes under the ++run's `events/` directory. For a short two-node H100 example: ++ ++```bash ++export MOONCAKE_ROOT=/mnt/home/kimbo/networkx/Mooncake ++TRACE=1 REPEATS=1 DURATION=1 MIN_BYTES=1048576 MAX_BYTES=4194304 \ ++ sbatch "$MOONCAKE_ROOT/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch" ++``` ++ ++The default `BUILD=1` rebuilds the binary first. An existing binary built before ++this recorder was added will not produce traces with `BUILD=0`. ++ ++Each process/engine gets a unique JSONL file. The initiator log prints its trace ++filename, which associates the file with the operation and repeat. The target ++normally submits no requests for one-sided RDMA, so its file has no data events; ++the launcher terminates that waiting target and its final footer may be absent. ++Initiator files should end with `trace_end` and `dropped_records: 0`. ++ ++Tracing also records warmup transfers. Unlike the tebench per-case summary, ++event files contain every observed request, and can be large. JSON serialization ++and tracing overhead can change measured bandwidth; retain untraced sweeps as ++the performance baseline. `TENT_TRACE_BUFFER_RECORDS` sets the bounded queue size ++(default 65536). Overflow drops trace records and is reported at shutdown. ++ ++See [TENT event recorder](../../tent/TRANSFER_EVENTS.md) for the schema, ++completion semantics, clock interpretation, and use with an independent replay ++driver. +diff --git a/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch b/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch +new file mode 100644 +index 00000000..77e606ec +--- /dev/null ++++ b/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch +@@ -0,0 +1,294 @@ ++#!/usr/bin/env bash ++#SBATCH --job-name=tebench-rdma ++#SBATCH --partition=h100 ++#SBATCH --account=cw-sup ++#SBATCH --nodes=2 ++#SBATCH --nodelist=slurm-h100-206-025,slurm-h100-206-035 ++#SBATCH --ntasks-per-node=1 ++#SBATCH --cpus-per-task=8 ++#SBATCH --gpus-per-node=1 ++#SBATCH --mem=64G ++#SBATCH --time=02:00:00 ++#SBATCH --output=tebench-%j.out ++ ++# Submit from the Mooncake root or export an absolute MOONCAKE_ROOT. ++set -euo pipefail ++: "${SLURM_JOB_ID:?Submit with sbatch}" ++MOONCAKE_ROOT=${MOONCAKE_ROOT:-$SLURM_SUBMIT_DIR} ++BUILD=${BUILD:-1} ++BUILD_ONLY=${BUILD_ONLY:-0} ++TRACE=${TRACE:-0} ++BUILD_IMAGE=${BUILD_IMAGE:-ubuntu:22.04} ++CUDA_HOST_PATH=${CUDA_HOST_PATH:-/usr/local/cuda-13.0} ++BUILD_DIR=${BUILD_DIR:-$MOONCAKE_ROOT/build/tebench-pyxis} ++RUNTIME_IMAGE=${RUNTIME_IMAGE:-$BUILD_DIR/runtime.sqsh} ++BUILD_JOBS=${BUILD_JOBS:-${SLURM_CPUS_PER_TASK:-8}} ++CUDA_HOME=${CUDA_HOME:-/usr/local/cuda} ++TEBENCH_BIN=${TEBENCH_BIN:-$BUILD_DIR/mooncake-transfer-engine/benchmark/tebench} ++SEG_TYPE=${SEG_TYPE:-VRAM} ++MIN_BYTES=${MIN_BYTES:-4096} ++MAX_BYTES=${MAX_BYTES:-67108864} ++BUFFER_BYTES=${BUFFER_BYTES:-268435456} ++BATCH_SIZE=${BATCH_SIZE:-1} ++THREADS=${THREADS:-1} ++DURATION=${DURATION:-5} ++REPEATS=${REPEATS:-3} ++READY_TIMEOUT=${READY_TIMEOUT:-120} ++OPS=${OPS:-read write} ++PYTHON=${PYTHON:-python3} ++export TEBENCH_BIN MOONCAKE_ROOT BUILD_DIR BUILD_JOBS CUDA_HOME PYTHON ++ ++fail() { echo "$*" >&2; exit 1; } ++[[ $# == 0 ]] || fail 'Configure the benchmark via environment variables; see README.md.' ++[[ $TEBENCH_BIN = /* && $BUILD_DIR = /* && $MOONCAKE_ROOT = /* && $RUNTIME_IMAGE = /* ]] || fail 'MOONCAKE_ROOT, BUILD_DIR, TEBENCH_BIN and RUNTIME_IMAGE must be absolute paths' ++for flag in BUILD BUILD_ONLY TRACE; do ++ [[ ${!flag} == 0 || ${!flag} == 1 ]] || fail "$flag must be 0 or 1" ++done ++if [[ $BUILD == 1 && $TEBENCH_BIN != "$BUILD_DIR/mooncake-transfer-engine/benchmark/tebench" ]]; then ++ fail 'Use BUILD=0 for an external TEBENCH_BIN, or select its BUILD_DIR' ++fi ++[[ $BUILD_JOBS =~ ^[1-9][0-9]*$ ]] || fail 'BUILD_JOBS must be a positive integer' ++((BUILD_JOBS <= ${SLURM_CPUS_PER_TASK:-8})) || fail 'BUILD_JOBS exceeds allocated CPUs' ++[[ $SEG_TYPE == VRAM || $SEG_TYPE == DRAM ]] || fail 'SEG_TYPE must be VRAM or DRAM' ++for var in MIN_BYTES MAX_BYTES BUFFER_BYTES BATCH_SIZE THREADS DURATION REPEATS READY_TIMEOUT; do ++ [[ ${!var} =~ ^[1-9][0-9]*$ ]] || fail "$var must be a positive decimal integer" ++done ++# Validate bounds in Python to avoid overflowing shell arithmetic on user inputs. ++"$PYTHON" - "$MIN_BYTES" "$MAX_BYTES" "$BUFFER_BYTES" "$BATCH_SIZE" "$THREADS" \ ++ "$DURATION" "$REPEATS" "$READY_TIMEOUT" <<'PY' ++import sys ++lo, hi, buf, batch, threads, duration, repeats, ready = map(int, sys.argv[1:]) ++if max(lo, hi, buf, batch, threads, duration, repeats, ready) > 2**31 - 1: ++ raise SystemExit('Launcher parameters must be <= 2^31-1') ++if lo > hi or hi % lo or (hi // lo) & ((hi // lo) - 1): ++ raise SystemExit('MAX_BYTES must equal MIN_BYTES times a power of two') ++if hi * batch * threads > buf: ++ raise SystemExit('BUFFER_BYTES must cover MAX_BYTES * BATCH_SIZE * THREADS') ++PY ++read -r -a operations <<< "$OPS" ++((${#operations[@]})) || fail 'OPS must contain read and/or write' ++for op in "${operations[@]}"; do ++ [[ $op == read || $op == write ]] || fail "Unsupported operation: $op" ++done ++mapfile -t nodes < <(scontrol show hostnames "$SLURM_JOB_NODELIST") ++if [[ $BUILD_ONLY == 1 ]]; then ++ ((${#nodes[@]} >= 1)) || fail 'At least one allocated node is required' ++else ++ ((${#nodes[@]} == 2)) || fail 'Exactly two allocated nodes are required' ++fi ++# Use the first allocated node as initiator unless explicitly overridden. ++initiator=${INITIATOR_NODE:-${nodes[0]}} ++[[ $initiator == "${nodes[0]}" || $initiator == "${nodes[1]:-}" ]] || fail 'INITIATOR_NODE must be in the allocation' ++target=${nodes[0]} ++[[ $target != "$initiator" ]] || target=${nodes[1]:-$initiator} ++# Must be shared between both nodes. Never default to node-local /tmp. ++run_base=${RESULT_ROOT:-$MOONCAKE_ROOT/benchmark-results} ++mkdir -p "$run_base" ++run_dir=$(mktemp -d "$run_base/tebench-${SLURM_JOB_ID}.XXXXXX") ++run_dir=$(cd "$run_dir" && pwd) ++export run_dir ++# Hold the lock through the sweep so another job cannot relink a running binary. ++mkdir -p "$BUILD_DIR" ++exec 9> "$BUILD_DIR/.slurm-build.lock" ++flock -w 1800 9 || fail 'Another job is using BUILD_DIR; select a different directory' ++scontrol show job "$SLURM_JOB_ID" > "$run_dir/slurm-job.txt" ++git -C "$MOONCAKE_ROOT" rev-parse HEAD > "$run_dir/commit.txt" ++printf 'initiator=%s\ntarget=%s\nseg_type=%s\nCUDA_VISIBLE_DEVICES is assigned by Slurm per node\n' \ ++ "$initiator" "$target" "$SEG_TYPE" > "$run_dir/topology.txt" ++printf 'Results: %s\nInitiator: %s; target: %s; RDMA %s\n' "$run_dir" "$initiator" "$target" "$SEG_TYPE" ++ ++# --exact lets the two one-node steps coexist within the two-node allocation. ++step=(srun --exact --nodes=1 --ntasks=1 --cpus-per-task="${SLURM_CPUS_PER_TASK:-8}" ++ --gpus-per-node=1 --cpu-bind=cores --kill-on-bad-exit=1) ++pids=() ++cleanup() { ++ local rc=$? ++ trap - EXIT ++ for pid in "${pids[@]}"; do kill -TERM "$pid" 2>/dev/null || true; done ++ # Signalling srun forwards termination to its remote task. ++ for ((i=0; i<50; i++)); do ++ local alive=0 ++ for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && alive=1; done ++ ((alive)) || break ++ sleep 0.1 ++ done ++ for pid in "${pids[@]}"; do ++ kill -KILL "$pid" 2>/dev/null || true ++ wait "$pid" 2>/dev/null || true ++ done ++ exit "$rc" ++} ++trap cleanup EXIT ++trap 'exit 143' TERM ++trap 'exit 130' INT ++ ++# Runs inside a writable Pyxis container on a compute node. ++build_tebench() { ++ set -euo pipefail ++ export PATH="$CUDA_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ++ export DEBIAN_FRONTEND=noninteractive ++ echo "Building tebench in Pyxis on $(hostname), output: $BUILD_DIR" ++ if [[ ! -f /opt/mooncake-tebench-deps-v1 ]]; then ++ apt-get update ++ apt-get install -y --no-install-recommends \ ++ ca-certificates cmake make g++ git python3-dev pkg-config \ ++ libnuma-dev libibverbs-dev librdmacm-dev libgflags-dev \ ++ libgoogle-glog-dev libjsoncpp-dev libyaml-cpp-dev libssl-dev liburing-dev ++ apt-get clean ++ mkdir -p /opt ++ touch /opt/mooncake-tebench-deps-v1 ++ fi ++ if [[ ! -f $MOONCAKE_ROOT/extern/pybind11/CMakeLists.txt ]]; then ++ git -c safe.directory="$MOONCAKE_ROOT" -C "$MOONCAKE_ROOT" \ ++ submodule update --init --recursive extern/pybind11 ++ fi ++ [[ -x $CUDA_HOME/bin/nvcc ]] || { ++ echo "CUDA compiler missing: $CUDA_HOME/bin/nvcc; check CUDA_HOST_PATH" >&2; return 1; ++ } ++ cmake --version ++ "$CUDA_HOME/bin/nvcc" --version ++ cmake -S "$MOONCAKE_ROOT" -B "$BUILD_DIR" \ ++ -DCMAKE_BUILD_TYPE=Release -DENABLE_DEBUG_SYMBOLS=OFF \ ++ -DCUDAToolkit_ROOT="$CUDA_HOME" -DCMAKE_CUDA_COMPILER="$CUDA_HOME/bin/nvcc" \ ++ -DCMAKE_CUDA_ARCHITECTURES=90 -DPython3_EXECUTABLE=/usr/bin/python3 \ ++ -DWITH_TE=ON -DUSE_TENT=ON -DUSE_CUDA=ON \ ++ -DWITH_STORE=OFF -DWITH_STORE_RUST=OFF -DWITH_STORE_GO=OFF \ ++ -DWITH_EP=OFF -DWITH_CONDUCTOR=OFF -DWITH_P2P_STORE=OFF \ ++ -DWITH_RUST_EXAMPLE=OFF -DBUILD_UNIT_TESTS=OFF -DBUILD_EXAMPLES=OFF \ ++ -DBUILD_BENCHMARK=ON -DUSE_ETCD=OFF -DUSE_REDIS=OFF -DUSE_HTTP=OFF ++ cmake --build "$BUILD_DIR" --target tebench --parallel "$BUILD_JOBS" ++ test -x "$TEBENCH_BIN" ++ ldd "$TEBENCH_BIN" ++} ++container_mounts="$MOONCAKE_ROOT:$MOONCAKE_ROOT,$BUILD_DIR:$BUILD_DIR,$run_dir:$run_dir,$CUDA_HOST_PATH:$CUDA_HOME:ro" ++if [[ $BUILD == 1 ]]; then ++ echo "Compute-node container build log: $run_dir/build.log" ++ build_image=$BUILD_IMAGE ++ [[ ! -f $RUNTIME_IMAGE ]] || build_image=$RUNTIME_IMAGE ++ # Save to a new path; publish the image only after a successful build. ++ pending_image=$BUILD_DIR/runtime-${SLURM_JOB_ID}.sqsh ++ MELLANOX_VISIBLE_DEVICES=none srun --exact --nodes=1 --ntasks=1 --nodelist="$initiator" \ ++ --cpus-per-task="${SLURM_CPUS_PER_TASK:-8}" --gpus-per-node=0 \ ++ --container-image="$build_image" --container-mounts="$container_mounts" \ ++ --container-workdir="$MOONCAKE_ROOT" --no-container-mount-home \ ++ --container-remap-root --container-writable --container-save="$pending_image" \ ++ --kill-on-bad-exit=1 bash -c "$(declare -f build_tebench); build_tebench" \ ++ > "$run_dir/build.log" 2>&1 & ++ pids=("$!") ++ build_rc=0 ++ wait "${pids[0]}" || build_rc=$? ++ pids=() ++ if ((build_rc != 0)); then ++ tail -n 60 "$run_dir/build.log" >&2 ++ echo "Build failed; full log: $run_dir/build.log" >&2 ++ exit "$build_rc" ++ fi ++ mv -- "$pending_image" "$RUNTIME_IMAGE" ++fi ++[[ -x $TEBENCH_BIN ]] || fail "Missing executable: $TEBENCH_BIN" ++[[ -f $RUNTIME_IMAGE ]] || fail "Missing built runtime image: $RUNTIME_IMAGE" ++if [[ $BUILD_ONLY == 1 ]]; then ++ echo "Build complete: $TEBENCH_BIN; runtime image: $RUNTIME_IMAGE" ++ exit 0 ++fi ++# Run both peers in the saved image so runtime libraries match the build. ++# The cluster already exposes permitted RDMA devices; the optional Mellanox ++# hook tries to recreate absent uverbs devices and fails on this platform. ++if [[ $TRACE == 1 ]]; then ++ export TENT_TRACE_DIR="$run_dir/events" ++ mkdir -p "$TENT_TRACE_DIR" ++ export TENT_TRACE_BUFFER_RECORDS=${TENT_TRACE_BUFFER_RECORDS:-65536} ++ echo "Transfer event traces: $TENT_TRACE_DIR" ++fi ++export MELLANOX_VISIBLE_DEVICES=none ++export NVIDIA_VISIBLE_DEVICES=all ++export NVIDIA_DRIVER_CAPABILITIES=compute,utility ++# GPU access remains restricted by Slurm cgroups and CUDA_VISIBLE_DEVICES. ++export LD_LIBRARY_PATH="$CUDA_HOME/lib64:$BUILD_DIR/mooncake-transfer-engine/tent/src/metrics${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ++container_mounts+=",/dev/infiniband:/dev/infiniband" ++container_env=LD_LIBRARY_PATH ++if [[ $TRACE == 1 ]]; then ++ container_env+=,TENT_TRACE_DIR,TENT_TRACE_BUFFER_RECORDS ++fi ++container=(--container-image="$RUNTIME_IMAGE" --container-mounts="$container_mounts" ++ --container-workdir="$MOONCAKE_ROOT" --no-container-mount-home ++ --container-env="$container_env") ++step+=("${container[@]}") ++ ++# Preflight on both nodes, including shared-path visibility and allocated GPUs. ++srun --nodes=2 --ntasks=2 --ntasks-per-node=1 --gpus-per-node=1 \ ++ --kill-on-bad-exit=1 "${container[@]}" bash -c ' ++ set -eu ++ test -x "$TEBENCH_BIN" ++ test -d "$run_dir" ++ ldd "$TEBENCH_BIN" > "$run_dir/ldd-$(hostname).txt" ++ if grep -q "not found" "$run_dir/ldd-$(hostname).txt"; then ++ cat "$run_dir/ldd-$(hostname).txt" >&2 ++ exit 1 ++ fi ++ { hostname; printf "CUDA_VISIBLE_DEVICES=%s\n" "${CUDA_VISIBLE_DEVICES:-unset}" ++ nvidia-smi -L; nvidia-smi topo -m ++ ls /sys/class/infiniband; ulimit -l ++ } > "$run_dir/host-$(hostname).txt" ++' ++# Keep both roles on GPU 0 in their own Slurm-provided visible device list. ++common=(--backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 ++ --seg_type="$SEG_TYPE" --local_gpu_id=0 --target_gpu_id=0 ++ --total_buffer_size="$BUFFER_BYTES") ++printf '%q ' "$TEBENCH_BIN" "${common[@]}" > "$run_dir/target-command.txt" ++printf '\n' >> "$run_dir/target-command.txt" ++"${step[@]}" --nodelist="$target" "$TEBENCH_BIN" "${common[@]}" \ ++ > "$run_dir/target.log" 2>&1 & ++target_pid=$! ++pids+=("$target_pid") ++endpoint= ++deadline=$((SECONDS + READY_TIMEOUT)) ++while ((SECONDS < deadline)); do ++ kill -0 "$target_pid" 2>/dev/null || fail "Target exited; inspect $run_dir/target.log" ++ endpoint=$(sed -n 's/.*\.\/tebench --target_seg_name=\([^[:space:]]*\).*/\1/p' "$run_dir/target.log" | head -n 1) ++ [[ -z $endpoint ]] || break ++ sleep 0.5 ++done ++[[ -n $endpoint ]] || fail "Target readiness timed out; inspect $run_dir/target.log" ++printf '%s\n' "$endpoint" > "$run_dir/endpoint.txt" ++ ++for ((repeat=1; repeat<=REPEATS; repeat++)); do ++ for op in "${operations[@]}"; do ++ kill -0 "$target_pid" 2>/dev/null || fail 'Target exited during sweep' ++ stem=$run_dir/${op}-${repeat} ++ args=("${common[@]}" --target_seg_name="$endpoint" --op_type="$op" ++ --start_block_size="$MIN_BYTES" --max_block_size="$MAX_BYTES" ++ --start_batch_size="$BATCH_SIZE" --max_batch_size="$BATCH_SIZE" ++ --start_num_threads="$THREADS" --max_num_threads="$THREADS" ++ --duration="$DURATION" --result_output_jsonl="$stem.jsonl") ++ printf '%q ' "$TEBENCH_BIN" "${args[@]}" > "$stem.command.txt" ++ printf '\n' >> "$stem.command.txt" ++ echo "Sweep: $op, repeat $repeat/$REPEATS, $MIN_BYTES..$MAX_BYTES bytes" ++ "${step[@]}" --nodelist="$initiator" "$TEBENCH_BIN" "${args[@]}" > "$stem.log" 2>&1 & ++ initiator_pid=$! ++ pids+=("$initiator_pid") ++ rc=0 ++ wait "$initiator_pid" || rc=$? ++ pids=("$target_pid") ++ ((rc == 0)) || { echo "Initiator failed: $stem.log" >&2; exit "$rc"; } ++ # Detect skipped/empty results even if the executable exits successfully. ++ "$PYTHON" - "$stem.jsonl" "$MIN_BYTES" "$MAX_BYTES" "$BATCH_SIZE" "$THREADS" <<'PY' ++import json ++import sys ++path, lo, hi, batch, threads = sys.argv[1:] ++expected = [] ++size = int(lo) ++while size <= int(hi): ++ expected.append(size) ++ size *= 2 ++with open(path) as stream: ++ rows = [json.loads(line) for line in stream if line.strip()] ++if [row['block_size'] for row in rows] != expected: ++ raise SystemExit('Incomplete message-size sweep: ' + path) ++if any(row['batch_size'] != int(batch) or row['num_threads'] != int(threads) ++ or row['aggregate_operations'] <= 0 for row in rows): ++ raise SystemExit('Unexpected parameters or empty measurements: ' + path) ++PY ++ done ++done ++echo "Completed all sweeps. JSON results and logs: $run_dir" diff --git a/experimental/mooncake-transfer-traces/patches/0003-store-benchmark-slurm.patch b/experimental/mooncake-transfer-traces/patches/0003-store-benchmark-slurm.patch new file mode 100644 index 0000000000..e7b7fa28b2 --- /dev/null +++ b/experimental/mooncake-transfer-traces/patches/0003-store-benchmark-slurm.patch @@ -0,0 +1,440 @@ +diff --git a/mooncake-store/benchmarks/slurm/README.md b/mooncake-store/benchmarks/slurm/README.md +new file mode 100644 +index 00000000..61e69ca1 +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/README.md +@@ -0,0 +1,168 @@ ++# H200 Slurm benchmark launchers ++ ++These launchers target this checkout and the cluster inspected on 2026-09-21. ++Submit from the Mooncake repository root, or export an absolute `MOONCAKE_ROOT`. ++They run existing workloads without changing benchmark implementations. ++ ++## Workload inventory ++ ++The repository has `benchmarks/`, not a top-level `benchmark/` directory. ++ ++| Location | Workload and measurement | ++| --- | --- | ++| `benchmarks/storage_benchmark_v1` | Replays conversation, synthetic, and tool-agent JSONL traces from `FAST25-release/traces`. Repeated cache-page hashes drive reads; new pages drive writes. GLM-5 and Kimi-K2.6 MLA layouts determine object bytes. Measures QPS, hit rate, request wall/I/O latency, page-read/write bandwidth and percentiles. Supports replay pacing, multiple independent clients, single-file/per-page files, and fsync policies. This tests filesystem storage, not the Mooncake Store service. | ++| `benchmarks/xypd_benchmarks` | End-to-end vLLM disaggregated prefill/decode through MooncakeStoreConnector and a proxy. The shell demo sweeps 1P1D, 2P1D, 2P2D, 2P4D, 4P4D routing; random prompts of 1024/4096 input and 6/256 output tokens; concurrency 1/4/8/16 (1 only for 1P1D); 50-token prefix by default. Reports serving throughput and TTFT, TPOT, inter-token and end-to-end latency. It launches four prefill and four decode instances even for smaller routing configurations. | ++| `mooncake-store/benchmarks/store_kv_bench.py` | Real Store API workloads: verified writes, fill, write/read performance, mixed existing-object reads plus new-object writes; also metadata smoke/mix, remove performance, and recorded-operation replay in the current source. Plain and registered-buffer APIs; single/batch operations; worker/depth controls. Outputs request/KV rate, MiB/s, latency percentiles, misses/errors, JSON summary and journal. | ++| `allocation_strategy_bench` | Simulated allocation placement across segment/replica/strategy matrices: fill-up, scale-out convergence, DSA churn, weighted size-class churn. Measures allocation latency, throughput, utilization, partial/failing allocations and fragmentation. Virtual segment capacity is not physical cluster RAM. | ++| `allocator_bench`, `offset_allocator_concurrency_bench` | Low-level allocation patterns and concurrent allocator/PutStart stages; allocation/free cost and contention. | ++| `master_bench`, `batch_get_replica_bench` | Master metadata RPC throughput and batched replica-lookup latency, including dataset prefill and optional concurrent refill. Mounted segments are synthetic; these are not data-transfer bandwidth benchmarks. | ++| `storage_backend_bench` | Offset-allocator, bucket and file-per-key storage: initialization, offload, load, concurrent reads, existence, mixed reads/writes, churn and restart. Configurable object/batch sizes, key access patterns and cache modes; integrity checking. | ++| `file_interface_bench`, `uring_batch_read_bench` | File read/write bandwidth through POSIX/io_uring/direct-I/O variants; sequential versus batched aligned io_uring reads. Requires a build/kernel supporting the selected variant. | ++| `batch_evict_bench`, `batch_oplog_promotion_bench` | In-process eviction scale/mutex contention and synthetic standby object promotion in chunks. | ++| `batch_remove_benchmark.py` | Batch-remove API demonstrations, correctness checks and single/batch removal comparisons. Use `store_kv_bench.py --scenario remove_perf` for the supplied Slurm removal workflow. | ++| `oplog_batch_bench`, `run_oplog_batch_sweep.py`, `report_oplog_batch.py` | Etcd-backed operation-log batching: batch size, producer count, writer/backend throughput and latency; repeatable sweep and report tooling. Requires `STORE_USE_ETCD` and a dedicated etcd endpoint. | ++| `nof_worker_pool_bench` | SPDK NVMe-oF read/write/mixed I/O, sequential/random offsets, queue depth, request sizes, submission concurrency and endpoint scaling. Requires `USE_NOF` and configured storage targets. | ++ ++The Slurm scripts cover local Store KV, the listed supported C++ microbenchmarks, ++and filesystem trace replay. They do not provision vLLM models/servers, etcd or ++NVMe-oF targets. The upstream vLLM demo installs packages and manages processes ++by ports; it is not used as an unattended Slurm entry point here. ++ ++## Observed cluster ++ ++- Cluster `slurm`, Slurm 25.05.3; account `cw-sup`, allowed QoS `normal`. ++- `h200`: **one** node, `slurm-h200-205-005`, eight H200 GPUs. ++- `h100`: two nodes; not used by these launchers. ++- Each node advertises 128 logical CPUs (64 cores, two sockets) and 2,031,616 MiB ++ RAM (1,984 GiB), with 32,768 MiB reserved by Slurm. ++- Default partition is `hpc-mid`. Other shared partitions are `all`, `hpc-low`, ++ `hpc-high`, and `hpc-prod`; scripts explicitly select `h200`. ++- H200 probe found local XFS at `/tmp`; `/mnt/home` is shared NFS. RDMA device ++ names include `ibp0` through `ibp7` and `mlx5_0` through `mlx5_8`. No bandwidth ++ or link-speed claim was measured. ++- At inspection the H200 node had five GPUs, 80 CPUs, and 500 GiB allocated. ++ Availability changes; jobs queue normally. ++ ++Defaults request one node, one task, 16 CPUs/64 GiB for Store or 4 CPUs/16 GiB ++for trace replay, with a 30-minute limit. These are host-memory and disk tests: ++selecting H200 places them on that host but does not reserve or use a GPU. ++For isolated measurements add `--exclusive` (which may wait for other jobs). ++One H200 node cannot measure cross-node RDMA; KV defaults to local TCP with one ++memory replica. The single-process Store client owns its segment, so this can ++exercise local-copy paths and must not be labeled network throughput. ++ ++## Prerequisites ++ ++The inspected environment has Python 3 but no Mooncake build directory, ++`mooncake_master` on PATH, or importable `mooncake` Python package. Build this ++checkout with the dependencies in `docs/source/getting_started/build.md`, and ++install its matching Python bindings in a shared environment. No system package ++installation is performed by these scripts. ++ ++For an already provisioned build environment, enable benchmark targets: ++ ++```bash ++cmake -S . -B build -DWITH_STORE=ON -DBUILD_BENCHMARK=ON ++cmake --build build --parallel 16 --target mooncake_master allocation_strategy_bench \ ++ offset_allocator_concurrency_bench storage_backend_bench master_bench \ ++ batch_get_replica_bench allocator_bench batch_evict_bench \ ++ batch_oplog_promotion_bench file_interface_bench uring_batch_read_bench ++``` ++ ++Use `BUILD_DIR` for a different absolute build directory, `MASTER_BIN` for a ++master executable elsewhere, and `VENV_DIR` or `PYTHON` for Python bindings built ++from this revision. A generic released wheel may lack APIs in this checkout. ++Trace replay uses the standard Python library and does not require Mooncake. ++ ++## Submit ++ ++From the Mooncake repository root: ++ ++```bash ++# Count-bounded, read-after-write verification: 1024 x 128 KiB, batches of 16. ++sbatch mooncake-store/benchmarks/slurm/store_kv.sbatch ++ ++# Read performance with automatic dataset preparation and registered buffers. ++SCENARIO=read_perf IO_API=zcopy RUNTIME=30 \ ++ sbatch mooncake-store/benchmarks/slurm/store_kv.sbatch ++ ++# Existing-object reads and new-object writes, 70% reads by default. ++SCENARIO=mixed_rw sbatch mooncake-store/benchmarks/slurm/store_kv.sbatch ++ ++SCENARIO=remove_perf sbatch mooncake-store/benchmarks/slurm/store_kv.sbatch ++ ++# Allocation churn is the default microbenchmark. ++sbatch mooncake-store/benchmarks/slurm/store_micro.sbatch ++BENCHMARK=master_bench sbatch mooncake-store/benchmarks/slurm/store_micro.sbatch \ ++ --operation=BatchGet ++BENCHMARK=batch_get_replica_bench \ ++ sbatch mooncake-store/benchmarks/slurm/store_micro.sbatch ++BENCHMARK=storage_backend_bench \ ++ sbatch mooncake-store/benchmarks/slurm/store_micro.sbatch \ ++ --backend=all --test=mixed_rw --pattern=zipf ++ ++# A bounded trace sample; choose each trace or all. ++SCENARIO=all sbatch mooncake-store/benchmarks/slurm/storage_trace.sbatch ++FSYNC_MODE=always MAX_REQUESTS=1000 MAX_PAGES=512 \ ++ sbatch mooncake-store/benchmarks/slurm/storage_trace.sbatch ++``` ++ ++Arguments after the script path pass through to the benchmark; for C++ flags, ++later arguments override launcher defaults. Slurm options belong before the ++script path. Do not override service endpoints or output paths unless you also ++adapt the worker's readiness/summary handling. An existing replay journal can ++be supplied with `SCENARIO=replay` and `--replay-file /absolute/path`. ++ ++| Launcher | Environment controls (defaults) | ++| --- | --- | ++| All | `MOONCAKE_ROOT` (submission directory), `BUILD_DIR` (`$MOONCAKE_ROOT/build`), `PYTHON` (`python3`), optional `VENV_DIR`, `RESULT_ROOT` (`$MOONCAKE_ROOT/benchmark-results`), `SCRATCH_ROOT` (`$SLURM_TMPDIR` or `/tmp`), `KEEP_SCRATCH` (`0`) | ++| Store KV | `SCENARIO=verify_write`, `IO_API=plain`, `OBJECTS=1024`, `WRITE_OBJECTS=1024`, `VALUE_BYTES=131072`, `BATCH_SIZE=16`, `NUMJOBS=4`, `IODEPTH=1`, `RUNTIME=0`, `SEGMENT_BYTES=8589934592`, `BUFFER_BYTES=268435456` | ++| Micro | `BENCHMARK=allocation_strategy_bench`; supported choices are the ten C++ targets in the build command above, excluding `mooncake_master` | ++| Trace | `SCENARIO=toolagent`, `TRACE_DIR` (`$MOONCAKE_ROOT/FAST25-release/traces`), `MODEL=glm5`, `THREADS=1`, `MAX_PAGES=128`, `MAX_REQUESTS=100`, `FILE_MODE=single`, `FSYNC_MODE=none`, `REPLAY_SCALES=0` | ++ ++## Results and interpretation ++ ++`benchmark-results/JOBID/` contains `benchmark.log`, host/filesystem details, ++Slurm allocation details, commit and worktree status. Service-backed jobs also ++save `master.log`; KV jobs save `kv/summary.json` and a journal; replica lookup ++saves `replica.csv`. Commands and resolved arguments are recorded in the log. ++The Slurm output file is created in the submission directory. Each job uses a ++unique scratch directory and shuts down only its own master process on exit. ++`KEEP_SCRATCH=1` retains data at the path recorded in `host.txt`. ++ ++Verification/pattern generation, Python threads and service/client CPU sharing ++contribute to the measured cost. Write workloads are count-bounded by default; ++raise counts/runtime and segment size deliberately for capacity/eviction tests. ++KV summaries with `ok=false` fail the job even when the upstream performance ++scenario exits zero. Inspect misses, error counts and latency sample rate before ++comparing results. ++ ++Trace defaults are smoke-scale, not a full trace reproduction. GLM-5 at 512 ++tokens uses roughly 53.6 MiB per page, so 128 pages take about 6.7 GiB per client. ++Each extra thread replays a full independent trace and uses its own storage. ++If pages exceed the cap, modulo mapping reuses physical slots. `fsync=none` ++measures page-cache-backed I/O, not durable disk throughput. Per-file `batch` ++and `end` fsync modes have weaker durability than single-file mode; see the ++upstream trace README. No global cache dropping is performed. ++ ++## Validation ++ ++Run `bash -n` separately on each `.sh` and `.sbatch` file. `sbatch --test-only` ++checks scheduling without submitting a job. Full Store execution requires the ++matching binaries/bindings described above. GPU-memory, multi-node RDMA, ++NVMe-oF and etcd-backed tests need additional topology/services and are outside ++these single-node launchers. ++ ++Validation performed on 2026-09-21: ++ ++- Shell syntax checks passed for every launcher and helper. ++- `sbatch --test-only` accepted all three requests on `slurm-h200-205-005`. ++- Job 1592 ran trace replay on H200 with one request and two physical pages; ++ Slurm reported `COMPLETED`, exit `0:0`. It wrote 14 logical pages using modulo ++ mapping and removed its scratch directory. This is a launch smoke test, not ++ a performance baseline. ++- Temporary stand-in master/benchmark executables verified successful and failed ++ benchmark exits (0 and 17), master termination and scratch cleanup. ++- Whitespace checks passed. `pre-commit` is not installed in this environment. ++- Real Store binaries/bindings were unavailable, so Store execution remains ++ unverified; no Store performance results are claimed. +diff --git a/mooncake-store/benchmarks/slurm/common.sh b/mooncake-store/benchmarks/slurm/common.sh +new file mode 100644 +index 00000000..1468711c +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/common.sh +@@ -0,0 +1,22 @@ ++#!/usr/bin/env bash ++# Shared initialization. sbatch copies scripts, so resolve from MOONCAKE_ROOT. ++set -euo pipefail ++: "${SLURM_JOB_ID:?Submit this script with sbatch}" ++export MOONCAKE_ROOT=${MOONCAKE_ROOT:-${SLURM_SUBMIT_DIR}} ++[[ -f "$MOONCAKE_ROOT/mooncake-store/benchmarks/store_kv_bench.py" ]] || { ++ echo 'Set MOONCAKE_ROOT to the absolute Mooncake checkout path.' >&2; exit 1; ++} ++export BUILD_DIR=${BUILD_DIR:-$MOONCAKE_ROOT/build} ++export PYTHON=${PYTHON:-python3} ++if [[ -n ${VENV_DIR:-} ]]; then ++ source "$VENV_DIR/bin/activate" ++fi ++export PYTHONUNBUFFERED=1 ++export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} ++export RUN_DIR=${RESULT_ROOT:-$MOONCAKE_ROOT/benchmark-results}/${SLURM_JOB_ID}${SLURM_ARRAY_TASK_ID:+_$SLURM_ARRAY_TASK_ID} ++mkdir -p "$RUN_DIR" ++# Unique local scratch is created inside the srun task, never on the login node. ++export SLURM_BENCH_DIR=$MOONCAKE_ROOT/mooncake-store/benchmarks/slurm ++scontrol show job "$SLURM_JOB_ID" > "$RUN_DIR/slurm-job.txt" ++git -C "$MOONCAKE_ROOT" rev-parse HEAD > "$RUN_DIR/commit.txt" ++git -C "$MOONCAKE_ROOT" status --short > "$RUN_DIR/worktree.txt" +diff --git a/mooncake-store/benchmarks/slurm/storage_trace.sbatch b/mooncake-store/benchmarks/slurm/storage_trace.sbatch +new file mode 100644 +index 00000000..d936af39 +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/storage_trace.sbatch +@@ -0,0 +1,14 @@ ++#!/usr/bin/env bash ++#SBATCH --job-name=mc-trace ++#SBATCH --partition=h200 ++#SBATCH --account=cw-sup ++#SBATCH --nodes=1 ++#SBATCH --ntasks=1 ++#SBATCH --cpus-per-task=4 ++#SBATCH --mem=16G ++#SBATCH --time=00:30:00 ++#SBATCH --output=mooncake-%x-%j.out ++ ++set -euo pipefail ++source "${MOONCAKE_ROOT:-$SLURM_SUBMIT_DIR}/mooncake-store/benchmarks/slurm/common.sh" ++srun --kill-on-bad-exit=1 --cpu-bind=cores bash "$SLURM_BENCH_DIR/worker.sh" trace "$@" +diff --git a/mooncake-store/benchmarks/slurm/store_kv.sbatch b/mooncake-store/benchmarks/slurm/store_kv.sbatch +new file mode 100644 +index 00000000..a4064daa +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/store_kv.sbatch +@@ -0,0 +1,14 @@ ++#!/usr/bin/env bash ++#SBATCH --job-name=mc-kv ++#SBATCH --partition=h200 ++#SBATCH --account=cw-sup ++#SBATCH --nodes=1 ++#SBATCH --ntasks=1 ++#SBATCH --cpus-per-task=16 ++#SBATCH --mem=64G ++#SBATCH --time=00:30:00 ++#SBATCH --output=mooncake-%x-%j.out ++ ++set -euo pipefail ++source "${MOONCAKE_ROOT:-$SLURM_SUBMIT_DIR}/mooncake-store/benchmarks/slurm/common.sh" ++srun --kill-on-bad-exit=1 --cpu-bind=cores bash "$SLURM_BENCH_DIR/worker.sh" kv "$@" +diff --git a/mooncake-store/benchmarks/slurm/store_micro.sbatch b/mooncake-store/benchmarks/slurm/store_micro.sbatch +new file mode 100644 +index 00000000..4638fa1e +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/store_micro.sbatch +@@ -0,0 +1,14 @@ ++#!/usr/bin/env bash ++#SBATCH --job-name=mc-micro ++#SBATCH --partition=h200 ++#SBATCH --account=cw-sup ++#SBATCH --nodes=1 ++#SBATCH --ntasks=1 ++#SBATCH --cpus-per-task=16 ++#SBATCH --mem=64G ++#SBATCH --time=00:30:00 ++#SBATCH --output=mooncake-%x-%j.out ++ ++set -euo pipefail ++source "${MOONCAKE_ROOT:-$SLURM_SUBMIT_DIR}/mooncake-store/benchmarks/slurm/common.sh" ++srun --kill-on-bad-exit=1 --cpu-bind=cores bash "$SLURM_BENCH_DIR/worker.sh" micro "$@" +diff --git a/mooncake-store/benchmarks/slurm/worker.sh b/mooncake-store/benchmarks/slurm/worker.sh +new file mode 100644 +index 00000000..d37eb757 +--- /dev/null ++++ b/mooncake-store/benchmarks/slurm/worker.sh +@@ -0,0 +1,172 @@ ++#!/usr/bin/env bash ++set -euo pipefail ++mode=$1 ++shift ++master_pid= ++scratch=$(mktemp -d "${SCRATCH_ROOT:-${SLURM_TMPDIR:-/tmp}}/mooncake-${SLURM_JOB_ID}.XXXXXX") ++cleanup() { ++ local rc=$? ++ trap - EXIT ++ if [[ -n $master_pid ]]; then ++ kill "$master_pid" 2>/dev/null || true ++ # Bound shutdown; only kill the process started by this worker. ++ for ((i=0; i<20; i++)); do ++ kill -0 "$master_pid" 2>/dev/null || break ++ sleep 0.1 ++ done ++ kill -KILL "$master_pid" 2>/dev/null || true ++ wait "$master_pid" 2>/dev/null || true ++ fi ++ if [[ ${KEEP_SCRATCH:-0} != 1 ]]; then ++ rm -rf -- "$scratch" ++ fi ++ exit "$rc" ++} ++trap cleanup EXIT ++trap 'exit 143' TERM ++trap 'exit 130' INT ++exec > >(tee "$RUN_DIR/benchmark.log") 2>&1 ++{ ++ hostname ++ date -u ++ printf 'scratch=%s\nmode=%s\n' "$scratch" "$mode" ++ findmnt -T "$scratch" ++ lscpu ++} > "$RUN_DIR/host.txt" ++cd "$scratch" ++export MOONCAKE_HOST_ID ++MOONCAKE_HOST_ID=$(hostname) ++ ++run() { ++ printf '%q ' "$@" ++ printf '\n' ++ "$@" ++} ++require_binary() { ++ [[ -x $1 ]] || { echo "Missing executable: $1 (build this checkout first)" >&2; exit 1; } ++} ++start_master() { ++ local master=${MASTER_BIN:-$BUILD_DIR/mooncake-store/src/mooncake_master} ++ require_binary "$master" ++ # Allocate three currently free ports; startup detects collisions/races. ++ read -r master_port metadata_port metrics_port < <("$PYTHON" - <<'PY' ++import socket ++sockets = [socket.socket() for _ in range(3)] ++for sock in sockets: ++ sock.bind(('127.0.0.1', 0)) ++print(*(sock.getsockname()[1] for sock in sockets)) ++PY ++) ++ "$master" --rpc_port="$master_port" --rpc_thread_num=4 \ ++ --enable_http_metadata_server=true \ ++ --http_metadata_server_host=127.0.0.1 \ ++ --http_metadata_server_port="$metadata_port" \ ++ --metrics_host=127.0.0.1 --metrics_port="$metrics_port" \ ++ --enable_ha=false --enable_offload=false > "$RUN_DIR/master.log" 2>&1 & ++ master_pid=$! ++ "$PYTHON" - "$master_pid" "$master_port" "$metadata_port" <<'PY' ++import os ++import socket ++import sys ++import time ++pid, *ports = map(int, sys.argv[1:]) ++deadline = time.monotonic() + 60 ++while time.monotonic() < deadline: ++ os.kill(pid, 0) ++ try: ++ for port in ports: ++ with socket.create_connection(('127.0.0.1', port), timeout=0.5): ++ pass ++ break ++ except OSError: ++ time.sleep(0.2) ++else: ++ raise SystemExit('Master readiness timed out; inspect master.log') ++PY ++ kill -0 "$master_pid" ++} ++case "$mode" in ++ kv) ++ "$PYTHON" -c 'from mooncake.store import MooncakeDistributedStore, ReplicateConfig' ++ start_master ++ scenario=${SCENARIO:-verify_write} ++ # Count-bounded writes avoid accidentally filling node memory in a timed loop. ++ run "$PYTHON" "$MOONCAKE_ROOT/mooncake-store/benchmarks/store_kv_bench.py" \ ++ --scenario "$scenario" --io-api "${IO_API:-plain}" \ ++ --local-hostname 127.0.0.1 \ ++ --metadata-server "http://127.0.0.1:$metadata_port/metadata" \ ++ --master-server "127.0.0.1:$master_port" --protocol tcp \ ++ --global-segment-size "${SEGMENT_BYTES:-8589934592}" \ ++ --local-buffer-size "${BUFFER_BYTES:-268435456}" \ ++ --nr-objects "${OBJECTS:-1024}" --write-objects "${WRITE_OBJECTS:-1024}" \ ++ --value-size "${VALUE_BYTES:-131072}" --batch-size "${BATCH_SIZE:-16}" \ ++ --numjobs "${NUMJOBS:-4}" --iodepth "${IODEPTH:-1}" \ ++ --runtime "${RUNTIME:-0}" --memory-replica-num 1 --nof-replica-num 0 \ ++ --key-prefix "j${SLURM_JOB_ID}_" --key-size 64 \ ++ --verify --pattern 0xab --output-dir "$RUN_DIR/kv" "$@" ++ # Some performance scenarios return 0 even with failed KV operations. ++ "$PYTHON" - "$RUN_DIR/kv/summary.json" <<'PY' ++import json ++import sys ++with open(sys.argv[1]) as stream: ++ result = json.load(stream) ++if not result['ok']: ++ raise SystemExit('KV errors recorded; inspect summary.json and journal.jsonl') ++PY ++ ;; ++ micro) ++ benchmark=${BENCHMARK:-allocation_strategy_bench} ++ binary=$BUILD_DIR/mooncake-store/benchmarks/$benchmark ++ require_binary "$binary" ++ case "$benchmark" in ++ allocation_strategy_bench) ++ run "$binary" --workload=size_class_churn --segment_capacity=1024 \ ++ --num_allocations=10000 --prefill_pct=70 "$@" ;; ++ offset_allocator_concurrency_bench) ++ run "$binary" --threads="${SLURM_CPUS_PER_TASK:-16}" "$@" ;; ++ storage_backend_bench) ++ run "$binary" --storage_path="$scratch/backend" --capacity_gb=4 \ ++ --num_operations=100 --warmup_operations=10 \ ++ --num_threads="${SLURM_CPUS_PER_TASK:-16}" "$@" ;; ++ file_interface_bench) ++ run "$binary" --file "$scratch/file.dat" "$@" ;; ++ uring_batch_read_bench) ++ run "$binary" "$scratch/uring.dat" "$@" ;; ++ master_bench) ++ start_master ++ run "$binary" --master_server="127.0.0.1:$master_port" \ ++ --num_clients=4 --num_threads=1 --duration=30 "$@" ;; ++ batch_get_replica_bench) ++ start_master ++ run "$binary" --master_server="127.0.0.1:$master_port" \ ++ --num_objects=100000 --prefill_threads=4 --lookup_threads=8 \ ++ --batch_size=128 --duration=30 --cycles=1 \ ++ --csv_path="$RUN_DIR/replica.csv" "$@" ;; ++ allocator_bench|batch_evict_bench|batch_oplog_promotion_bench) ++ run "$binary" "$@" ;; ++ *) echo "Unsupported benchmark: $benchmark; see README.md" >&2; exit 2 ;; ++ esac ++ ;; ++ trace) ++ scenario=${SCENARIO:-toolagent} ++ trace_dir=${TRACE_DIR:-$MOONCAKE_ROOT/FAST25-release/traces} ++ case "$scenario" in ++ conversation|synthetic|toolagent) scenarios=("$scenario") ;; ++ all) scenarios=(conversation synthetic toolagent) ;; ++ *) echo "Unknown trace scenario: $scenario" >&2; exit 2 ;; ++ esac ++ for item in "${scenarios[@]}"; do ++ [[ -s "$trace_dir/${item}_trace.jsonl" ]] || { ++ echo "Missing trace: $trace_dir/${item}_trace.jsonl" >&2; exit 1; ++ } ++ done ++ cd "$MOONCAKE_ROOT/benchmarks/storage_benchmark_v1" ++ run "$PYTHON" benchmark.py --trace-dir "$trace_dir" --scenario "$scenario" \ ++ --storage-dir "$scratch/trace" --model "${MODEL:-glm5}" \ ++ --threads "${THREADS:-1}" --max-pages "${MAX_PAGES:-128}" \ ++ --max-requests "${MAX_REQUESTS:-100}" --file-mode "${FILE_MODE:-single}" \ ++ --fsync-mode "${FSYNC_MODE:-none}" --replay-scales "${REPLAY_SCALES:-0}" \ ++ --progress-interval 0 "$@" ++ ;; ++ *) echo "Unknown mode: $mode" >&2; exit 2 ;; ++esac diff --git a/experimental/mooncake-transfer-traces/patches/series b/experimental/mooncake-transfer-traces/patches/series new file mode 100644 index 0000000000..ae68ee7465 --- /dev/null +++ b/experimental/mooncake-transfer-traces/patches/series @@ -0,0 +1,3 @@ +0001-tent-transfer-event-recorder.patch +0002-tebench-slurm.patch +0003-store-benchmark-slurm.patch diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/commit.txt b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/commit.txt new file mode 100644 index 0000000000..f75798842c --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/commit.txt @@ -0,0 +1 @@ +fe0d23e332f41c7d54b5c217194dce5f8e751b21 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-1.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-1.jsonl new file mode 100644 index 0000000000..ede9154655 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-1.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":608505,"aggregate_throughput_gbps":0.4984865981187626,"aggregate_transferred_bytes":2492436480,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5808020575289865,"avg_latency_us":8.216870855621565,"avg_transfer_us":7.078438139374368,"index":0,"operations":608505,"p999_us":11.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.4984865981187626,"total_duration_us":5000007.0,"transferred_bytes":2492436480}]} +{"aggregate_operations":595848,"aggregate_throughput_gbps":0.9762367774579335,"aggregate_transferred_bytes":4881186816,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.115329457478969,"avg_latency_us":8.391406868865886,"avg_transfer_us":7.385589949114539,"index":0,"operations":595848,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.9762367774579335,"total_duration_us":5000003.0,"transferred_bytes":4881186816}]} +{"aggregate_operations":570425,"aggregate_throughput_gbps":1.869167892332843,"aggregate_transferred_bytes":9345843200,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.038545814829497,"avg_latency_us":8.765397729762896,"avg_transfer_us":8.051847306832626,"index":0,"operations":570425,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.869167892332843,"total_duration_us":5000002.0,"transferred_bytes":9345843200}]} +{"aggregate_operations":544105,"aggregate_throughput_gbps":3.565842962157038,"aggregate_transferred_bytes":17829232640,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.064820310312367,"avg_latency_us":9.189411970116062,"avg_transfer_us":8.07421361685704,"index":0,"operations":544105,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.565842962157038,"total_duration_us":5000005.0,"transferred_bytes":17829232640}]} +{"aggregate_operations":505116,"aggregate_throughput_gbps":6.620648490421812,"aggregate_transferred_bytes":33103282176,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.249220417918269,"avg_latency_us":9.898728212925349,"avg_transfer_us":9.0489451927874,"index":0,"operations":505116,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.620648490421812,"total_duration_us":5000006.0,"transferred_bytes":33103282176}]} +{"aggregate_operations":422979,"aggregate_throughput_gbps":11.0881406976,"aggregate_transferred_bytes":55440703488,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.91482890118237,"avg_latency_us":11.820917823343475,"avg_transfer_us":11.013442747748707,"index":0,"operations":422979,"p999_us":15.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":11.0881406976,"total_duration_us":5000000.0,"transferred_bytes":55440703488}]} +{"aggregate_operations":337678,"aggregate_throughput_gbps":17.704009836776393,"aggregate_transferred_bytes":88520261632,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":18.80822705417645,"avg_latency_us":14.807041027250813,"avg_transfer_us":13.951806158529724,"index":0,"operations":337678,"p999_us":18.0,"p99_us":15.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":17.704009836776393,"total_duration_us":5000012.0,"transferred_bytes":88520261632}]} +{"aggregate_operations":219462,"aggregate_throughput_gbps":23.01218497220809,"aggregate_transferred_bytes":115061293056,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.55427203119992,"avg_latency_us":22.78306039314323,"avg_transfer_us":21.383218962736144,"index":0,"operations":219462,"p999_us":26.0,"p99_us":24.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":23.01218497220809,"total_duration_us":5000016.0,"transferred_bytes":115061293056}]} +{"aggregate_operations":145721,"aggregate_throughput_gbps":30.55980475586383,"aggregate_transferred_bytes":152799543296,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":32.34920751588829,"avg_latency_us":34.31226110169433,"avg_transfer_us":32.49390959436183,"index":0,"operations":145721,"p999_us":51.0,"p99_us":49.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.55980475586383,"total_duration_us":5000017.0,"transferred_bytes":152799543296}]} +{"aggregate_operations":73146,"aggregate_throughput_gbps":30.679349244907552,"aggregate_transferred_bytes":153398280192,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.88805658315263,"avg_latency_us":68.35712137369097,"avg_transfer_us":65.84230169797391,"index":0,"operations":73146,"p999_us":90.0,"p99_us":84.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.679349244907552,"total_duration_us":5000050.0,"transferred_bytes":153398280192}]} +{"aggregate_operations":42825,"aggregate_throughput_gbps":35.9235743203771,"aggregate_transferred_bytes":179621068800,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.756890885450375,"avg_latency_us":116.75631056625802,"avg_transfer_us":114.25139521307648,"index":0,"operations":42825,"p999_us":160.0,"p99_us":147.00000000005093,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.9235743203771,"total_duration_us":5000089.0,"transferred_bytes":179621068800}]} +{"aggregate_operations":23360,"aggregate_throughput_gbps":39.19108276835712,"aggregate_transferred_bytes":195957882880,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.7055354409625,"avg_latency_us":214.04379280821917,"avg_transfer_us":211.53587328767122,"index":0,"operations":23360,"p999_us":300.0,"p99_us":247.47999999999593,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.19108276835712,"total_duration_us":5000063.0,"transferred_bytes":195957882880}]} +{"aggregate_operations":12284,"aggregate_throughput_gbps":41.217777899020795,"aggregate_transferred_bytes":206091321344,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.526475030291024,"avg_latency_us":407.0383425594269,"avg_transfer_us":404.54843699120806,"index":0,"operations":12284,"p999_us":581.0,"p99_us":444.1800000000112,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.217777899020795,"total_duration_us":5000059.0,"transferred_bytes":206091321344}]} +{"aggregate_operations":6245,"aggregate_throughput_gbps":41.90842109410421,"aggregate_transferred_bytes":209547427840,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":42.097483619706516,"avg_latency_us":800.6608486789431,"avg_transfer_us":798.1559647718175,"index":0,"operations":6245,"p999_us":1148.0,"p99_us":990.0399999998299,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.90842109410421,"total_duration_us":5000127.0,"transferred_bytes":209547427840}]} +{"aggregate_operations":3114,"aggregate_throughput_gbps":41.78853882112557,"aggregate_transferred_bytes":208977002496,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.91886099643007,"avg_latency_us":1605.9155427103403,"avg_transfer_us":1603.4001284521516,"index":0,"operations":3114,"p999_us":2363.887,"p99_us":1606.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.78853882112557,"total_duration_us":5000821.0,"transferred_bytes":208977002496}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-2.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-2.jsonl new file mode 100644 index 0000000000..9360b7acb9 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-2.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":612740,"aggregate_throughput_gbps":0.5019565076086985,"aggregate_transferred_bytes":2509783040,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5821138763770836,"avg_latency_us":8.160069523778438,"avg_transfer_us":7.056085778633678,"index":0,"operations":612740,"p999_us":11.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.5019565076086985,"total_duration_us":5000001.0,"transferred_bytes":2509783040}]} +{"aggregate_operations":603860,"aggregate_throughput_gbps":0.9893626410197743,"aggregate_transferred_bytes":4946821120,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.1440223579509647,"avg_latency_us":8.280078163812805,"avg_transfer_us":7.184075779154108,"index":0,"operations":603860,"p999_us":10.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.9893626410197743,"total_duration_us":5000008.0,"transferred_bytes":4946821120}]} +{"aggregate_operations":568191,"aggregate_throughput_gbps":1.8618482688000002,"aggregate_transferred_bytes":9309241344,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.0376776425065493,"avg_latency_us":8.799857794298044,"avg_transfer_us":8.055578141857227,"index":0,"operations":568191,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.8618482688000002,"total_duration_us":5000000.0,"transferred_bytes":9309241344}]} +{"aggregate_operations":547091,"aggregate_throughput_gbps":3.5854127092698325,"aggregate_transferred_bytes":17927077888,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.067341870624284,"avg_latency_us":9.139254712652923,"avg_transfer_us":8.07034295939798,"index":0,"operations":547091,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.5854127092698325,"total_duration_us":5000004.0,"transferred_bytes":17927077888}]} +{"aggregate_operations":506182,"aggregate_throughput_gbps":6.6346180950110485,"aggregate_transferred_bytes":33173143552,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.24268538630299,"avg_latency_us":9.877885819724922,"avg_transfer_us":9.057989023710839,"index":0,"operations":506182,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.6346180950110485,"total_duration_us":5000008.0,"transferred_bytes":33173143552}]} +{"aggregate_operations":422206,"aggregate_throughput_gbps":11.067872539250985,"aggregate_transferred_bytes":55339384832,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.949309537776239,"avg_latency_us":11.842565003813304,"avg_transfer_us":10.980563990090145,"index":0,"operations":422206,"p999_us":14.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":11.067872539250985,"total_duration_us":5000002.0,"transferred_bytes":55339384832}]} +{"aggregate_operations":330585,"aggregate_throughput_gbps":17.332157515842482,"aggregate_transferred_bytes":86660874240,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":18.552224125711824,"avg_latency_us":15.124718302403314,"avg_transfer_us":14.146313353600435,"index":0,"operations":330585,"p999_us":18.0,"p99_us":16.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":17.332157515842482,"total_duration_us":5000005.0,"transferred_bytes":86660874240}]} +{"aggregate_operations":219040,"aggregate_throughput_gbps":22.96792601946633,"aggregate_transferred_bytes":114840043520,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.523804883567642,"avg_latency_us":22.82696311176041,"avg_transfer_us":21.44759861212564,"index":0,"operations":219040,"p999_us":30.0,"p99_us":25.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":22.96792601946633,"total_duration_us":5000018.0,"transferred_bytes":114840043520}]} +{"aggregate_operations":142746,"aggregate_throughput_gbps":29.935940080131825,"aggregate_transferred_bytes":149680029696,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.605474151163627,"avg_latency_us":35.02732826138736,"avg_transfer_us":33.24952713210878,"index":0,"operations":142746,"p999_us":52.0,"p99_us":49.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":29.935940080131825,"total_duration_us":5000011.0,"transferred_bytes":149680029696}]} +{"aggregate_operations":72084,"aggregate_throughput_gbps":30.233979081767348,"aggregate_transferred_bytes":151171104768,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.395155108904042,"avg_latency_us":69.36407524554686,"avg_transfer_us":66.86496309860718,"index":0,"operations":72084,"p999_us":90.0,"p99_us":86.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.233979081767348,"total_duration_us":5000040.0,"transferred_bytes":151171104768}]} +{"aggregate_operations":42374,"aggregate_throughput_gbps":35.54519796235953,"aggregate_transferred_bytes":177729437696,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.355989855105044,"avg_latency_us":117.99917402180583,"avg_transfer_us":115.5013923632416,"index":0,"operations":42374,"p999_us":161.0,"p99_us":155.2699999999968,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.54519796235953,"total_duration_us":5000097.0,"transferred_bytes":177729437696}]} +{"aggregate_operations":23342,"aggregate_throughput_gbps":39.160375081597905,"aggregate_transferred_bytes":195806887936,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.67337463050737,"avg_latency_us":214.2116356781767,"avg_transfer_us":211.70833690343588,"index":0,"operations":23342,"p999_us":301.0,"p99_us":223.36000000000058,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.160375081597905,"total_duration_us":5000128.0,"transferred_bytes":195806887936}]} +{"aggregate_operations":12284,"aggregate_throughput_gbps":41.21693708342592,"aggregate_transferred_bytes":206091321344,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.5291233436027,"avg_latency_us":407.046646043634,"avg_transfer_us":404.5222240312602,"index":0,"operations":12284,"p999_us":581.0,"p99_us":439.20000000001164,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.21693708342592,"total_duration_us":5000161.0,"transferred_bytes":206091321344}]} +{"aggregate_operations":6166,"aggregate_throughput_gbps":41.37653676382212,"aggregate_transferred_bytes":206896627712,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.56898435377207,"avg_latency_us":810.9531300681155,"avg_transfer_us":808.4176127148882,"index":0,"operations":6166,"p999_us":1187.0,"p99_us":941.7500000001328,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.37653676382212,"total_duration_us":5000337.0,"transferred_bytes":206896627712}]} +{"aggregate_operations":3118,"aggregate_throughput_gbps":41.84249321346955,"aggregate_transferred_bytes":209245437952,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.97186654103933,"avg_latency_us":1603.8447722899296,"avg_transfer_us":1601.3345093008338,"index":0,"operations":3118,"p999_us":2360.0,"p99_us":1604.83,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.84249321346955,"total_duration_us":5000788.0,"transferred_bytes":209245437952}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-3.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-3.jsonl new file mode 100644 index 0000000000..19fa040a89 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/read-3.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":611115,"aggregate_throughput_gbps":0.5006253078749384,"aggregate_transferred_bytes":2503127040,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5827743553618878,"avg_latency_us":8.18176775238703,"avg_transfer_us":7.044346808702127,"index":0,"operations":611115,"p999_us":10.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.5006253078749384,"total_duration_us":5000001.0,"transferred_bytes":2503127040}]} +{"aggregate_operations":597038,"aggregate_throughput_gbps":0.978186081013919,"aggregate_transferred_bytes":4890935296,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.1223145067962839,"avg_latency_us":8.374684693436599,"avg_transfer_us":7.336757459324197,"index":0,"operations":597038,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.978186081013919,"total_duration_us":5000005.0,"transferred_bytes":4890935296}]} +{"aggregate_operations":572734,"aggregate_throughput_gbps":1.8767340205063918,"aggregate_transferred_bytes":9383673856,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.046218519015945,"avg_latency_us":8.730059678664093,"avg_transfer_us":8.01734487563162,"index":0,"operations":572734,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.8767340205063918,"total_duration_us":5000002.0,"transferred_bytes":9383673856}]} +{"aggregate_operations":547919,"aggregate_throughput_gbps":3.590836931228296,"aggregate_transferred_bytes":17954209792,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.085543926834862,"avg_latency_us":9.12544919960797,"avg_transfer_us":8.026069546776075,"index":0,"operations":547919,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.590836931228296,"total_duration_us":5000007.0,"transferred_bytes":17954209792}]} +{"aggregate_operations":501732,"aggregate_throughput_gbps":6.576293778847465,"aggregate_transferred_bytes":32881508352,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.205459320771748,"avg_latency_us":9.965491537314742,"avg_transfer_us":9.11039957586919,"index":0,"operations":501732,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.576293778847465,"total_duration_us":5000006.0,"transferred_bytes":32881508352}]} +{"aggregate_operations":418678,"aggregate_throughput_gbps":10.9753925632,"aggregate_transferred_bytes":54876962816,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.854646773629604,"avg_latency_us":11.942351879009644,"avg_transfer_us":11.068883485638128,"index":0,"operations":418678,"p999_us":15.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":10.9753925632,"total_duration_us":5000000.0,"transferred_bytes":54876962816}]} +{"aggregate_operations":333272,"aggregate_throughput_gbps":17.47303356056644,"aggregate_transferred_bytes":87365255168,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":18.5825467345672,"avg_latency_us":15.002775510693967,"avg_transfer_us":14.125384070669003,"index":0,"operations":333272,"p999_us":19.0,"p99_us":16.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":17.47303356056644,"total_duration_us":5000005.0,"transferred_bytes":87365255168}]} +{"aggregate_operations":218515,"aggregate_throughput_gbps":22.912953881409223,"aggregate_transferred_bytes":114564792320,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.420155086230597,"avg_latency_us":22.88172894309315,"avg_transfer_us":21.512697068851107,"index":0,"operations":218515,"p999_us":27.0,"p99_us":24.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":22.912953881409223,"total_duration_us":5000001.0,"transferred_bytes":114564792320}]} +{"aggregate_operations":145487,"aggregate_throughput_gbps":30.51078038299531,"aggregate_transferred_bytes":152554176512,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":32.3098484915999,"avg_latency_us":34.36739365029178,"avg_transfer_us":32.534638833710225,"index":0,"operations":145487,"p999_us":52.0,"p99_us":49.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.51078038299531,"total_duration_us":5000009.0,"transferred_bytes":152554176512}]} +{"aggregate_operations":73097,"aggregate_throughput_gbps":30.659048762512224,"aggregate_transferred_bytes":153295519744,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.86044290507602,"avg_latency_us":68.40238313473877,"avg_transfer_us":65.89546766625169,"index":0,"operations":73097,"p999_us":89.0,"p99_us":84.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.659048762512224,"total_duration_us":5000009.0,"transferred_bytes":153295519744}]} +{"aggregate_operations":42842,"aggregate_throughput_gbps":35.93792094961737,"aggregate_transferred_bytes":179692371968,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.76776139531133,"avg_latency_us":116.70970076093553,"avg_transfer_us":114.21509266607535,"index":0,"operations":42842,"p999_us":160.0,"p99_us":154.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.93792094961737,"total_duration_us":5000077.0,"transferred_bytes":179692371968}]} +{"aggregate_operations":23386,"aggregate_throughput_gbps":39.23412232264836,"aggregate_transferred_bytes":196175986688,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.74664346167114,"avg_latency_us":213.80898828358846,"avg_transfer_us":211.3181818181818,"index":0,"operations":23386,"p999_us":300.0,"p99_us":270.90000000003783,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.23412232264836,"total_duration_us":5000137.0,"transferred_bytes":196175986688}]} +{"aggregate_operations":12288,"aggregate_throughput_gbps":41.22992964659706,"aggregate_transferred_bytes":206158430208,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.537680265568575,"avg_latency_us":406.9183756510417,"avg_transfer_us":404.4379069010417,"index":0,"operations":12288,"p999_us":581.0,"p99_us":444.84999999988395,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.22992964659706,"total_duration_us":5000213.0,"transferred_bytes":206158430208}]} +{"aggregate_operations":6165,"aggregate_throughput_gbps":41.37043857093117,"aggregate_transferred_bytes":206863073280,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.56057809286164,"avg_latency_us":811.0726682887267,"avg_transfer_us":808.5826439578265,"index":0,"operations":6165,"p999_us":1187.0,"p99_us":947.5199999998831,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.37043857093117,"total_duration_us":5000263.0,"transferred_bytes":206863073280}]} +{"aggregate_operations":3118,"aggregate_throughput_gbps":41.83873668694365,"aggregate_transferred_bytes":209245437952,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.967804894300194,"avg_latency_us":1603.9887748556766,"avg_transfer_us":1601.5016035920462,"index":0,"operations":3118,"p999_us":2361.0,"p99_us":1605.83,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":41.83873668694365,"total_duration_us":5001237.0,"transferred_bytes":209245437952}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/summary.csv b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/summary.csv new file mode 100644 index 0000000000..6edc63e6a8 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/summary.csv @@ -0,0 +1,31 @@ +operation,message_bytes,repetitions,mean_GBps,min_GBps,max_GBps,mean_wall_us,mean_transfer_us,min_run_p99_us,max_run_p99_us,min_run_p999_us,max_run_p999_us +read,4096,3,0.5003561378674665,0.4984865981187626,0.5019565076086985,8.18623604392901,7.059623575570058,8.0,8.0,10.0,11.0 +read,8192,3,0.9812618331638756,0.9762367774579335,0.9893626410197743,8.34872324203843,7.302141062530948,8.0,9.0,10.0,11.0 +read,16384,3,1.8692500605464117,1.8618482688000002,1.8767340205063918,8.76510506757501,8.041590108107158,9.0,9.0,11.0,12.0 +read,32768,3,3.580697534218389,3.565842962157038,3.590836931228296,9.151371960792318,8.056875374343699,9.0,10.0,11.0,12.0 +read,65536,3,6.6105201214267755,6.576293778847465,6.6346180950110485,9.914035189988338,9.07244459745581,10.0,10.0,12.0,13.0 +read,131072,3,11.043801933350329,10.9753925632,11.0881406976,11.868611568722141,11.02096340782566,12.0,12.0,14.0,15.0 +read,262144,3,17.50306697106177,17.332157515842482,17.704009836776393,14.978178280116031,14.074501194266388,15.0,16.0,18.0,19.0 +read,524288,3,22.964354957694546,22.912953881409223,23.01218497220809,22.830584149332264,21.447838214570965,24.0,25.0,26.0,30.0 +read,1048576,3,30.335508406330323,29.935940080131825,30.55980475586383,34.56899433779116,32.75935852006028,49.0,49.0,51.0,52.0 +read,2097152,3,30.524125696395707,30.233979081767348,30.679349244907552,68.7078599179922,66.20091082094426,84.0,86.0,89.0,90.0 +read,4194304,3,35.80223107745133,35.54519796235953,35.93792094961737,117.1550617829998,114.65596008079781,147.00000000005093,155.2699999999968,160.0,161.0 +read,8388608,3,39.19519339086779,39.160375081597905,39.23412232264836,214.02147225666144,211.52079733642964,223.36000000000058,270.90000000003783,300.0,301.0 +read,16777216,3,41.22154820968126,41.21693708342592,41.22992964659706,407.0011214180342,404.5028559745033,439.20000000001164,444.84999999988395,581.0,581.0 +read,33554432,3,41.55179880961917,41.37043857093117,41.90842109410421,807.5622156785951,805.052073814844,941.7500000001328,990.0399999998299,1148.0,1187.0 +read,67108864,3,41.823256240512926,41.78853882112557,41.84249321346955,1604.583029951982,1602.0787471150106,1604.83,1606.0,2360.0,2363.887 +write,4096,3,0.5015755508396647,0.4986648629340548,0.5044724841385222,8.166449750191946,7.065704674982779,8.0,9.0,10.0,12.0 +write,8192,3,0.9906987270880243,0.9859524836285098,0.994896304882956,8.26902508580103,7.1468484983858955,8.0,9.0,10.0,12.0 +write,16384,3,1.9253342860587188,1.9081598885761561,1.9350388736000002,8.51003305492369,7.733591253398925,8.0,9.0,10.0,12.0 +write,32768,3,3.6564099802635868,3.641998631600274,3.667270727566836,8.961871590111139,8.041331150422923,9.0,9.0,11.0,12.0 +write,65536,3,6.728749591086833,6.7016327168,6.751615475661429,9.739791244426165,9.016499333684717,9.0,10.0,12.0,13.0 +write,131072,3,11.10367177332039,11.05551450627969,11.143044265338029,11.804509607726386,10.95960729504708,12.0,12.0,14.0,15.0 +write,262144,3,17.075179699210093,16.51841753214948,17.385309834704262,15.360676734985498,14.346883404089255,16.0,16.0,18.0,19.0 +write,524288,3,23.013176365697383,22.77099049303993,23.3445523456,22.784595622157617,21.38837925912564,24.0,24.0,26.0,27.0 +write,1048576,3,29.7657023669789,29.494372578090015,29.9918619350351,35.22934304381233,33.425933362761754,55.0,56.0,58.0,59.0 +write,2097152,3,30.969740273355836,30.70157883381166,31.117211533557743,67.71872161313895,65.18746230309162,96.11000000000058,98.0,100.0,101.0 +write,4194304,3,35.42831182874708,35.34138701500152,35.472090791511796,118.3888371709473,115.86012382661913,176.0,176.0,181.0,182.0 +write,8388608,3,38.23022561757633,38.14061343899004,38.28895469740906,219.42407286162637,216.93170450315958,230.0,290.0700000000288,340.0,342.0 +write,16777216,3,39.780574458980695,39.71437439945985,39.81753935141252,421.7445218848949,419.2401700152059,579.3200000000652,651.6499999999996,660.1340000000018,661.0 +write,33554432,3,40.37462545607061,40.36417338103253,40.38137358194148,831.0772614547418,828.5650754860786,942.6000000000167,1271.7199999999993,1283.0,1285.0 +write,67108864,3,40.218340715051085,40.19957947487343,40.23570663778332,1668.6136691054846,1666.1096095283538,1762.400000000176,2617.05,2631.003,2634.0150000000003 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/topology.txt b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/topology.txt new file mode 100644 index 0000000000..43dddc44bf --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/topology.txt @@ -0,0 +1,4 @@ +initiator=slurm-h100-206-025 +target=slurm-h100-206-035 +seg_type=VRAM +CUDA_VISIBLE_DEVICES is assigned by Slurm per node diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-1.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-1.jsonl new file mode 100644 index 0000000000..5dcd4fd4b7 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-1.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":608722,"aggregate_throughput_gbps":0.4986648629340548,"aggregate_transferred_bytes":2493325312,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5797104663059562,"avg_latency_us":8.213933454023348,"avg_transfer_us":7.090159383100989,"index":0,"operations":608722,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.4986648629340548,"total_duration_us":5000002.0,"transferred_bytes":2493325312}]} +{"aggregate_operations":607237,"aggregate_throughput_gbps":0.994896304882956,"aggregate_transferred_bytes":4974485504,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.1585230793497494,"avg_latency_us":8.234023947816091,"avg_transfer_us":7.0844612564781135,"index":0,"operations":607237,"p999_us":10.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.994896304882956,"total_duration_us":5000004.0,"transferred_bytes":4974485504}]} +{"aggregate_operations":590527,"aggregate_throughput_gbps":1.9350388736000002,"aggregate_transferred_bytes":9675194368,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.132143955748516,"avg_latency_us":8.46701336264049,"avg_transfer_us":7.717100149527456,"index":0,"operations":590527,"p999_us":10.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.9350388736000002,"total_duration_us":5000000.0,"transferred_bytes":9675194368}]} +{"aggregate_operations":555725,"aggregate_throughput_gbps":3.641998631600274,"aggregate_transferred_bytes":18209996800,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.0761624910181204,"avg_latency_us":8.997257636420892,"avg_transfer_us":8.047694453191777,"index":0,"operations":555725,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.641998631600274,"total_duration_us":5000001.0,"transferred_bytes":18209996800}]} +{"aggregate_operations":511294,"aggregate_throughput_gbps":6.7016327168,"aggregate_transferred_bytes":33508163584,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.269240628581399,"avg_latency_us":9.779109475174753,"avg_transfer_us":9.021365398381361,"index":0,"operations":511294,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.7016327168,"total_duration_us":5000000.0,"transferred_bytes":33508163584}]} +{"aggregate_operations":425074,"aggregate_throughput_gbps":11.143044265338029,"aggregate_transferred_bytes":55715299328,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":12.018346226619132,"avg_latency_us":11.76267426377525,"avg_transfer_us":10.923342759143114,"index":0,"operations":425074,"p999_us":15.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":11.143044265338029,"total_duration_us":5000007.0,"transferred_bytes":55715299328}]} +{"aggregate_operations":330388,"aggregate_throughput_gbps":17.32181173077654,"aggregate_transferred_bytes":86609231872,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":18.566171132478036,"avg_latency_us":15.133751831180309,"avg_transfer_us":14.133261498601644,"index":0,"operations":330388,"p999_us":18.0,"p99_us":16.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":17.32181173077654,"total_duration_us":5000010.0,"transferred_bytes":86609231872}]} +{"aggregate_operations":218621,"aggregate_throughput_gbps":22.923986258452217,"aggregate_transferred_bytes":114620366848,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.444556607305707,"avg_latency_us":22.870716902767803,"avg_transfer_us":21.48288590757521,"index":0,"operations":218621,"p999_us":26.0,"p99_us":24.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":22.923986258452217,"total_duration_us":5000019.0,"transferred_bytes":114620366848}]} +{"aggregate_operations":142150,"aggregate_throughput_gbps":29.81087258781158,"aggregate_transferred_bytes":149055078400,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.558826110118165,"avg_latency_us":35.17428068941259,"avg_transfer_us":33.3436299683433,"index":0,"operations":142150,"p999_us":58.0,"p99_us":56.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":29.81087258781158,"total_duration_us":5000024.0,"transferred_bytes":149055078400}]} +{"aggregate_operations":74126,"aggregate_throughput_gbps":31.090430452698108,"aggregate_transferred_bytes":155453489152,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":32.36989679780356,"avg_latency_us":67.45329573968648,"avg_transfer_us":64.92612578582414,"index":0,"operations":74126,"p999_us":101.0,"p99_us":98.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":31.090430452698108,"total_duration_us":5000043.0,"transferred_bytes":155453489152}]} +{"aggregate_operations":42287,"aggregate_throughput_gbps":35.472090791511796,"aggregate_transferred_bytes":177364533248,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.32938716941327,"avg_latency_us":118.24236763071393,"avg_transfer_us":115.71002908695344,"index":0,"operations":42287,"p999_us":182.0,"p99_us":176.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.472090791511796,"total_duration_us":5000115.0,"transferred_bytes":177364533248}]} +{"aggregate_operations":22806,"aggregate_throughput_gbps":38.26110871632989,"aggregate_transferred_bytes":191310594048,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.78812112357254,"avg_latency_us":219.24633868280276,"avg_transfer_us":216.74445321406648,"index":0,"operations":22806,"p999_us":341.19500000000335,"p99_us":230.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":38.26110871632989,"total_duration_us":5000132.0,"transferred_bytes":191310594048}]} +{"aggregate_operations":11865,"aggregate_throughput_gbps":39.80980962606971,"aggregate_transferred_bytes":199061667840,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.13872312330622,"avg_latency_us":421.43421828908555,"avg_transfer_us":418.9218710493047,"index":0,"operations":11865,"p999_us":661.0,"p99_us":579.3200000000652,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.80980962606971,"total_duration_us":5000317.0,"transferred_bytes":199061667840}]} +{"aggregate_operations":6018,"aggregate_throughput_gbps":40.38137358194148,"aggregate_transferred_bytes":201930571776,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.590480391957065,"avg_latency_us":830.9383516118312,"avg_transfer_us":828.419740777667,"index":0,"operations":6018,"p999_us":1285.0,"p99_us":1220.109999999977,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.38137358194148,"total_duration_us":5000587.0,"transferred_bytes":201930571776}]} +{"aggregate_operations":2997,"aggregate_throughput_gbps":40.2197360324965,"aggregate_transferred_bytes":201125265408,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.37790005612358,"avg_latency_us":1668.5555555555557,"avg_transfer_us":1666.0183516850184,"index":0,"operations":2997,"p999_us":2632.0040000000004,"p99_us":1896.9999999999727,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.2197360324965,"total_duration_us":5000661.0,"transferred_bytes":201125265408}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-2.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-2.jsonl new file mode 100644 index 0000000000..5de7e8c253 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-2.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":615812,"aggregate_throughput_gbps":0.5044724841385222,"aggregate_transferred_bytes":2522365952,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5838307093571846,"avg_latency_us":8.119372470819016,"avg_transfer_us":7.029127396023462,"index":0,"operations":615812,"p999_us":10.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.5044724841385222,"total_duration_us":5000007.0,"transferred_bytes":2522365952}]} +{"aggregate_operations":601778,"aggregate_throughput_gbps":0.9859524836285098,"aggregate_transferred_bytes":4929765376,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.1373633486310961,"avg_latency_us":8.308716835776648,"avg_transfer_us":7.235311028319414,"index":0,"operations":601778,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.9859524836285098,"total_duration_us":5000003.0,"transferred_bytes":4929765376}]} +{"aggregate_operations":582325,"aggregate_throughput_gbps":1.9081598885761561,"aggregate_transferred_bytes":9540812800,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.1124207881659984,"avg_latency_us":8.586282574163912,"avg_transfer_us":7.800460224101662,"index":0,"operations":582325,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.9081598885761561,"total_duration_us":5000007.0,"transferred_bytes":9540812800}]} +{"aggregate_operations":559582,"aggregate_throughput_gbps":3.667270727566836,"aggregate_transferred_bytes":18336382976,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.0870308393000165,"avg_latency_us":8.935255244092913,"avg_transfer_us":8.024734533991444,"index":0,"operations":559582,"p999_us":11.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.667270727566836,"total_duration_us":5000008.0,"transferred_bytes":18336382976}]} +{"aggregate_operations":515108,"aggregate_throughput_gbps":6.751615475661429,"aggregate_transferred_bytes":33758117888,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.2889973616155705,"avg_latency_us":9.706713931835655,"avg_transfer_us":8.99691521001421,"index":0,"operations":515108,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.751615475661429,"total_duration_us":5000006.0,"transferred_bytes":33758117888}]} +{"aggregate_operations":421735,"aggregate_throughput_gbps":11.05551450627969,"aggregate_transferred_bytes":55277649920,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.91950232187337,"avg_latency_us":11.855802814563647,"avg_transfer_us":11.01026236854897,"index":0,"operations":421735,"p999_us":14.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":11.05551450627969,"total_duration_us":5000007.0,"transferred_bytes":55277649920}]} +{"aggregate_operations":315064,"aggregate_throughput_gbps":16.51841753214948,"aggregate_transferred_bytes":82592137216,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":17.757841119986374,"avg_latency_us":15.869801056293324,"avg_transfer_us":14.782412462229896,"index":0,"operations":315064,"p999_us":19.0,"p99_us":16.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":16.51841753214948,"total_duration_us":5000003.0,"transferred_bytes":82592137216}]} +{"aggregate_operations":217162,"aggregate_throughput_gbps":22.77099049303993,"aggregate_transferred_bytes":113855430656,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.29738281791078,"avg_latency_us":23.024382718891886,"avg_transfer_us":21.615545076947164,"index":0,"operations":217162,"p999_us":27.0,"p99_us":24.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":22.77099049303993,"total_duration_us":5000021.0,"transferred_bytes":113855430656}]} +{"aggregate_operations":143013,"aggregate_throughput_gbps":29.9918619350351,"aggregate_transferred_bytes":149959999488,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.67915388029898,"avg_latency_us":34.96201743897408,"avg_transfer_us":33.21095285044017,"index":0,"operations":143013,"p999_us":58.0,"p99_us":55.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":29.9918619350351,"total_duration_us":5000023.0,"transferred_bytes":149959999488}]} +{"aggregate_operations":74190,"aggregate_throughput_gbps":31.117211533557743,"aggregate_transferred_bytes":155587706880,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":32.39214868940974,"avg_latency_us":67.39524194635396,"avg_transfer_us":64.87519881385631,"index":0,"operations":74190,"p999_us":100.0,"p99_us":96.11000000000058,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":31.117211533557743,"total_duration_us":5000053.0,"transferred_bytes":155587706880}]} +{"aggregate_operations":42286,"aggregate_throughput_gbps":35.47145767972791,"aggregate_transferred_bytes":177360338944,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.32337585284628,"avg_latency_us":118.24447807785083,"avg_transfer_us":115.72648157782717,"index":0,"operations":42286,"p999_us":181.0,"p99_us":176.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.47145767972791,"total_duration_us":5000086.0,"transferred_bytes":177360338944}]} +{"aggregate_operations":22822,"aggregate_throughput_gbps":38.28895469740906,"aggregate_transferred_bytes":191444811776,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.80568492759427,"avg_latency_us":219.08688984313383,"avg_transfer_us":216.63964595565682,"index":0,"operations":22822,"p999_us":340.0,"p99_us":290.0700000000288,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":38.28895469740906,"total_duration_us":5000001.0,"transferred_bytes":191444811776}]} +{"aggregate_operations":11867,"aggregate_throughput_gbps":39.81753935141252,"aggregate_transferred_bytes":199095222272,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.14638588633502,"avg_latency_us":421.3524058312969,"avg_transfer_us":418.8809303109463,"index":0,"operations":11867,"p999_us":660.1340000000018,"p99_us":650.0200000000004,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.81753935141252,"total_duration_us":5000189.0,"transferred_bytes":199095222272}]} +{"aggregate_operations":6015,"aggregate_throughput_gbps":40.36417338103253,"aggregate_transferred_bytes":201829908480,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.577292404243344,"avg_latency_us":831.2924355777224,"avg_transfer_us":828.7986699916875,"index":0,"operations":6015,"p999_us":1284.9860000000008,"p99_us":1271.7199999999993,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.36417338103253,"total_duration_us":5000224.0,"transferred_bytes":201829908480}]} +{"aggregate_operations":2998,"aggregate_throughput_gbps":40.23570663778332,"aggregate_transferred_bytes":201192374272,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.388220056212994,"avg_latency_us":1667.893262174783,"avg_transfer_us":1665.405603735824,"index":0,"operations":2998,"p999_us":2631.003,"p99_us":1762.400000000176,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.23570663778332,"total_duration_us":5000344.0,"transferred_bytes":201192374272}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-3.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-3.jsonl new file mode 100644 index 0000000000..e505c11491 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1612.XD95we/write-3.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":612292,"aggregate_throughput_gbps":0.5015893054464168,"aggregate_transferred_bytes":2507948032,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5807087722913884,"avg_latency_us":8.166043325733474,"avg_transfer_us":7.077827245823888,"index":0,"operations":612292,"p999_us":12.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.5015893054464168,"total_duration_us":5000003.0,"transferred_bytes":2507948032}]} +{"aggregate_operations":605010,"aggregate_throughput_gbps":0.9912473927526072,"aggregate_transferred_bytes":4956241920,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.1540828767389224,"avg_latency_us":8.26433447381035,"avg_transfer_us":7.120773210360159,"index":0,"operations":605010,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":0.9912473927526072,"total_duration_us":5000005.0,"transferred_bytes":4956241920}]} +{"aggregate_operations":589845,"aggregate_throughput_gbps":1.9328040960000001,"aggregate_transferred_bytes":9664020480,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.1426115809968493,"avg_latency_us":8.47680322796667,"avg_transfer_us":7.683213386567657,"index":0,"operations":589845,"p999_us":11.0,"p99_us":8.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":1.9328040960000001,"total_duration_us":5000000.0,"transferred_bytes":9664020480}]} +{"aggregate_operations":558466,"aggregate_throughput_gbps":3.659960581623651,"aggregate_transferred_bytes":18299813888,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":4.07494539373725,"avg_latency_us":8.953101889819614,"avg_transfer_us":8.05156446408555,"index":0,"operations":558466,"p999_us":12.0,"p99_us":9.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":3.659960581623651,"total_duration_us":5000003.0,"transferred_bytes":18299813888}]} +{"aggregate_operations":513688,"aggregate_throughput_gbps":6.733000580799071,"aggregate_transferred_bytes":33665056768,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.263414447838044,"avg_latency_us":9.733550326268086,"avg_transfer_us":9.03121739265858,"index":0,"operations":513688,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":6.733000580799071,"total_duration_us":5000008.0,"transferred_bytes":33665056768}]} +{"aggregate_operations":423907,"aggregate_throughput_gbps":11.11245654834345,"aggregate_transferred_bytes":55562338304,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.991888119237997,"avg_latency_us":11.79505174484026,"avg_transfer_us":10.945216757449158,"index":0,"operations":423907,"p999_us":14.0,"p99_us":12.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":11.11245654834345,"total_duration_us":5000005.0,"transferred_bytes":55562338304}]} +{"aggregate_operations":331599,"aggregate_throughput_gbps":17.385309834704262,"aggregate_transferred_bytes":86926688256,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":18.577741048972666,"avg_latency_us":15.078477317482863,"avg_transfer_us":14.124976251436223,"index":0,"operations":331599,"p999_us":18.0,"p99_us":16.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":17.385309834704262,"total_duration_us":5000008.0,"transferred_bytes":86926688256}]} +{"aggregate_operations":222631,"aggregate_throughput_gbps":23.3445523456,"aggregate_transferred_bytes":116722761728,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":24.942879793798536,"avg_latency_us":22.458687244813166,"avg_transfer_us":21.066706792854543,"index":0,"operations":222631,"p999_us":26.0,"p99_us":24.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":23.3445523456,"total_duration_us":5000000.0,"transferred_bytes":116722761728}]} +{"aggregate_operations":140641,"aggregate_throughput_gbps":29.494372578090015,"aggregate_transferred_bytes":147472777216,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.201896552517216,"avg_latency_us":35.55173100305032,"avg_transfer_us":33.72321726950178,"index":0,"operations":140641,"p999_us":59.0,"p99_us":56.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":29.494372578090015,"total_duration_us":5000031.0,"transferred_bytes":147472777216}]} +{"aggregate_operations":73199,"aggregate_throughput_gbps":30.70157883381166,"aggregate_transferred_bytes":153509429248,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.955625441010636,"avg_latency_us":68.30762715337642,"avg_transfer_us":65.7610623095944,"index":0,"operations":73199,"p999_us":101.0,"p99_us":97.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":30.70157883381166,"total_duration_us":5000050.0,"transferred_bytes":153509429248}]} +{"aggregate_operations":42131,"aggregate_throughput_gbps":35.34138701500152,"aggregate_transferred_bytes":176710221824,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":36.191893456564806,"avg_latency_us":118.67966580427714,"avg_transfer_us":116.14386081507678,"index":0,"operations":42131,"p999_us":182.0,"p99_us":176.0,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":35.34138701500152,"total_duration_us":5000093.0,"transferred_bytes":176710221824}]} +{"aggregate_operations":22734,"aggregate_throughput_gbps":38.14061343899004,"aggregate_transferred_bytes":190706614272,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.66824670691537,"avg_latency_us":219.93899005894255,"avg_transfer_us":217.41101433975544,"index":0,"operations":22734,"p999_us":342.0,"p99_us":270.08999999995285,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":38.14061343899004,"total_duration_us":5000093.0,"transferred_bytes":190706614272}]} +{"aggregate_operations":11836,"aggregate_throughput_gbps":39.71437439945985,"aggregate_transferred_bytes":198575128576,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.05095653046539,"avg_latency_us":422.44694153430214,"avg_transfer_us":419.9177086853667,"index":0,"operations":11836,"p999_us":661.0,"p99_us":651.6499999999996,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":39.71437439945985,"total_duration_us":5000082.0,"transferred_bytes":198575128576}]} +{"aggregate_operations":6017,"aggregate_throughput_gbps":40.37832940523782,"aggregate_transferred_bytes":201897017344,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.58605130096636,"avg_latency_us":831.0009971746717,"avg_transfer_us":828.4768156888815,"index":0,"operations":6017,"p999_us":1283.0,"p99_us":942.6000000000167,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.37832940523782,"total_duration_us":5000133.0,"transferred_bytes":201897017344}]} +{"aggregate_operations":2996,"aggregate_throughput_gbps":40.19957947487343,"aggregate_transferred_bytes":201058156544,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.36002631896842,"avg_latency_us":1669.392189586115,"avg_transfer_us":1666.904873164219,"index":0,"operations":2996,"p999_us":2634.0150000000003,"p99_us":2617.05,"segment_name":"10.0.3.49:16985","threads":1,"throughput_gbps":40.19957947487343,"total_duration_us":5001499.0,"transferred_bytes":201058156544}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/analysis-job.out b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/analysis-job.out new file mode 100644 index 0000000000..4bd14cabd6 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/analysis-job.out @@ -0,0 +1,6 @@ +read-3 done Counter({'submit': 4407813, 'complete': 4407813}) drops 276006 +write-1 done Counter({'submit': 4361484, 'complete': 4361484}) drops 505420 +read-2 done Counter({'submit': 4389126, 'complete': 4389126}) drops 417592 +read-1 done Counter({'submit': 4426938, 'complete': 4426938}) drops 312506 +write-2 done Counter({'submit': 4346500, 'complete': 4346500}) drops 593004 +write-3 done Counter({'submit': 4424285, 'complete': 4424283}) drops 378892 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-comparison.png b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..6bceed3d985f22171da64cb989aff4ff254dab6b GIT binary patch literal 167024 zcmce;cRbho|33aiArX~IgrrkeNF@;oAq`nkMn=dEQMM9VqB2s+-Wef#OGHs3du3_Zw)K{BYxwQ)(tfqlN&b;?2L(X1~;x-+T5@-Gu-WHY-ev~V|_^Qps?UE zf!)_`+_-KpE+l02Kd%t9u`?CgC;#mdK4rso#Y^@C!TJ~Z4~6e=P7gsr5U1r%s^5Aw z)_GHB>kuc+l=7C{mP~T35qDg?c($#-shg{9waPp3^|EGX_JldRgGI`Nwl?!o=~#!z zJ7P@RPBP`(qs%xddVBBUGWX!w>fI-q8Yh?Mf1Jv7naCRwys2D1TQuY(MqeSk`!16V zE!Y44E9SxVv2Ok!Ri^Z)!vF(}HD`M*A& z{Jca}=72r_&sS(Xpj7?mySz4U*+RL=mui!*S3p3(Pr*%i_n!Ux_tR|LAt?87`vHAR z%TMkmOG`@{H^0`*a|ON2sI02WtSa&Nv&QJy-QXH5sQdf;Qnh=xbN7%JlT3A$-`}5H zO(U}q-gI+m>B67HKuPy=Y)Xe)<$0L*@bS^PER2=)e|*+EI7lriSwu%8`QGs0rd?v5 z6P*PWZEdW-fB(LIt!;yyAsp;w4&z^04^ypDUTAF{p!)TP-^1}1uP9l_)loxb#-iRgBxFAkRt)Q0c}YMPnx ziin6zjkO+nb;5%K*C0RHRn$8+_UM3iwx@=B8V zzO1b*+5h_eJIZ17yH>X4PQfe1k3xj5M5U)QQQjuZ%*=9~X7@Rcw|SR&(U6biOo-qq z1B1wm%TJ1Fnp1Tv9&F?Pl55+?xX?>6WBZgL;G68`r16%wRE!+Q31Vf^%xYH zhD=+xdTAtI)UY3^j(_Ydxf!vz;YQLw@&5HuSQtBAqncz2Q4#Rs#e=ogWo}W??VC1j z8W_Ae(Q&M$t&K8Z?nCXzk0<5jsja(<{SKMd?G_Q)vSGu9=TdIsh=c1t-fwsw8k+c7 z?aCF-%F4=nj~~mct21BDu}%@O!kgBXCr=Qbw{KHCc<|t`*v2hbK{2=Ahl$db78dLr zF}r@+Y|BAd)x`S7%nXa0cP!d1Cr#uuYS=pkerE;E@aG6}JbO1F$pLpqJpEnX-OVX3-qiiBty+T@IxU=Bs z`7rTHQMtT=0wV+#xidNoZoah}P$D0_S7xSt(15IzprfNB-J(N?psuj{%B{JdO&htm zxT8QtO@hHuA%SB zv$wZh`DOcP*D=*T*G8w6Uk6>?+@8FAxsiB&$aE8OR?^{~resZ_voA#%PMEsTj&<3G57Yz`GxJ?6Ye+%o@YiSxMatxc zh$kWqy}yfBX3k74xu(?Y>s-jJm#V;Tzeb#-B7!6=yEU)4RZ>R>3tf5q&Z0TEeYvKt zZqv)7HuBf5@t9-Z_PV8A%Jh-kpJUT4kL~3>yXS7<6|Al2imc2(#oz#LkqnBrNRAms zm7K?KT`+rrhxOJh|K?=s;m=+)j4g*<7AG`b#O3C);8IWF`Cf=W&yE|3L{tp7z2nC^ zeh_VNee_z&t!5zs0nYa7)vLg`(ZAzoC!>tefp{rW`^KYsj}8qevDI_X6%EG+ElW5VV~etu=W*O|6$^QNGp_N?eFZER#-zkdBd@W=?E zjap2tt*wpFDyy!h#od#uUmKRgCnfc|bEqNq3i~s~zd<15Bz|a6EVVbXmFg#;=7=_9zRt zD|&n0-`{HP|L!=hhvk^v-d*hMk!fj+xOmt3Ce2*?p~r3GNYJ+Z)tk@065DLmp6#8N zckG6PgMono1tTLP`L!QEel(xe%D=(uGV!CYj|&;+=A_;&C#SN8hAlomKGzyzHjvP! zIqgb_ivWs>R-+>_W{n zJ&hL=wL&H9d@EE~E^99Q)>yoL3fMu^s_iZosSFQ|e8^2n+fZ8WudKKRwV$4y#FHwp z{Pg@FU30=*m&yPT3X9nF-Ii8XWka*QJv{`X^tk(q*0>{b$Gu%XB(UXN59CNqa4$~_ zZhyG*%$YOP#39Eqt=M+tqsN_frKO&mY1h%2W!rSu#W^jEg_i$DaTcB*Z8rVJvutZ? zYkS*6mzAY%%yHDF>uQ>&@d?05)4K2z zS0-H4R8`B*zBqip_qv#Q?IE|{H{|zj;k67853j#5-lj30O^_cMyWE{)!}-$WtBjzERD72C>!TTMePv{3k?mexe=}4Z}LsjV@)So7CYbP`%|96combexaTTdwY8;<%H;kcXf4rs(iR(=b=M~Vx3sn z*bbpO7}ba7<>fWppr)q&1c;&$oD&lxH=(Dt5FH<11x%o3%#mR%p9y3uB1^?xmg-O5c)~ zLxm0QG>E@6=$}l(Llth%vC(wu93CDPR(QNi<%P1665ThT9lP3wH+qS~jGo zr*E|G>+7okVB$NFoRf2u%G1`)&Jg*$OzYAmUQ13UrVDE?1!NaE78e&am-14hqoWsC zgM-g?`rp65VYb`jPvR%pg#s;0OM&IqsIhn~{aJJKq?$sqLWxO8BzK_XESyzW4{R6> zHDEEB9sE3&4>-)%{|E2=IFRGIWEVS-pccQ0%5$mv&)WVR9qgd_yl1ap*JYV~lOI2J zYx;+Ghg*bzhr@}C_G-Q6<%Myh?E+d6b`nNLMuRm${CxE%$hPXC^FQVFAo?vE8yliD z_Dl%#_o2bT8p(mn>gspX(}lfnl$Dhc1Rhyq+LbHTy5{D5uCA^v`-^Ed?Yg&h|M?!2 zHZj{D>i}mc6kks*OiwTI_=e|u{E=GDH~SW6{!P@nLuL=h#2m+3$`f{Pxn!e`#(h6O<13mAu7*fx)|;YhLxe&K{IZ(A;p-n_r{p>8d2(+&CDCd!;px*Y zyTotM5?@L@Jf6RNNk#l@PV)zJqr4^Ys=FXS@L_^Va@WY)p*)%CG5Mhtciwk9XY%WHqXPElGAosOhbGR#*p? z6YhmItDnLocHk}E7kK1Gzo)!4s@g;XcRZtC%abLo7PHu$Wp_7hZ46!l+GG*6WI228 z+*{Pb3ji^x>XJ%&)+Vv|Qcl*suMNRJspeSn7^@#1Lbq^5F^JDh=@EWuh zWZ4f44CLvoN4aSoO-IuNXt^Ixbu%mL15Wo}6!P-&JiNSRcxF>W;T}l`X0c+^O`2D3 z*AMl@hr5>t@oO@1a6G=^vAS*h_U$B;Z~rEvF=?DgMYBnI?-mNrHhaB`*mPv$hSz4) zuB@(Z-nx~0Q7yabS!1Eo?De%3r@8U=9q2JCfxd0gVW2a+dybo#&cNFG@e^)^zZ(+Q zC6S4aJTLp9I)mEa1J_#8xd?qUDq2O(4td5>Hz#(*oqOp6X<5$Jn2ZWoY7(@KT_Hhocf_y!dfmFI`ewg?r8RN~o_Cn=C8KQ2_StjyMvjfUAFO=o_r==h00 zD;05_?<_Qf`lETPk9W4^J6P=F=Rd=B4oT#t$?v%ly3x#x3>JW9YQo6cx(*-46Z5gI z&g62|wX#9`k%qs#4|B^u(r>nDNk3^&q$OZd{QD<=o(R8-&BA1NJ)w_%G>eVFCutpd zp?bo z>?}AQ8x|P&eh`^Qv)487D_(-EGVuCVh=!W8B89%x^4OuchzMyeKTF0#{QUeuBR&jl zAL1T9dZeyBf$l@t@U50v=wMw~ZQRBU8&tG}mRpB^{;Y|KjpdD50{<{j^M)@h>n3MJ z;n3*|7nll)&%G2qd;a`mnHLDQ^hMvqMBesq${L~QXqvjxkg|-wN!^-~HWoKiR%m@K z*%pxEU$d+u(B5YTZpi%G{(bxSisWPD_G~_c#6%r1m!#sTSGz!rfU8jhD;6+OdYj7f zMTVru!v_yqIM7xwsA`*N%$HbBifRDuyvJ5PNIbVGUXnMi@5P=>H|{L)C@FVt)lD8c@BA@UpRPbne00;b5$|6&{o1_pm2AVcvf9Tj};g=TX*^Bp-*n z|K33K3=R3F75`Q;W99Jo#pd^_ACd)-{nQ-r^eLsZ$c}*g`y;jdXEcqvP&}dM1A;)X7iHR*w|cP5Rvxody4{V3-p@bT#C*PsRa|+(j}hceA?T`=f}rq`z?Pj zeyprqj|IK(@@gDhKS|ntj_hb?Z3jT9RuupZ(U{Tm_gBxM0YRjtxyU!Bgyu z+qGn599hv#Dm=4V;=byPa$DNev<;p69Wag!H`LqagvxXg?;r?tL5gR19?{(K$fZJA3k?Z+e} zEIRG+?N-Fz?p4Rht^>`o%H}`b-{%{;X1Tsz0M@^+kzk0x~HS&Ve znKQLvuU2HaU5e2eH^v27?S9soDl+@^B7*iS5_QmMXJ;pcCws8U_odk(3dNdaYg*Bx zM~`M3jH6Yzt6==7r|^^Y%(-*tNM>4{OiZYbu=`Y6)hTZ57wQCZI(fu zMB-UTkd!H+VBot2-(P>NAF2zZBIr4#bhs{DGcoZ2OFjeGkc0%=&B?Cy1RC*e92^`ZO$SCNRkz?n+%s#n5_dN- zaq*e~eu@1uzMI$^gL^@R8eKUvWSWrhHCiE)VeO}T)_Cj%6*RMB78=D?4u&sBt$F#RC{L1EeuhIWbflIKi@a)afi6Ov5{~qL zhhI~Z#(?bBi~+DIUwL#}MXD+*BO@XR>7{UA6BCoMoCbRdDj;?Fa60(@U<8 zSPoKp?#&&6-h_+j0SGovJ786KYx)iflgCvS4TQ+ zq^<=6+M?H#l-dg#zz85QJv-aS+j~9h;cK1<*38NHu~+9NXI?9HD@#vHro5Tl;rVkj zO-x!*Q4uKEuYif(hrfRPI<_2dC7mSV9~sp(;Iao@w(9o3RnY)t&$;mKn3%neEw9l9 z$pR;XNrNtS_wnORg@uJg;p+a&0^5__qSUf&V+uc9x3I8?ZFi&3GJ7Aadqvb!?b6b< zGo2Ss6D?eqV7lg+#Rl@REpN-hA!6^{VEqv`Q8y&W?qRp^-vfM!Sy_y{=vZ? z$RNME|EyI}8^@mwy_1lTKsTNob#*_d|D#9m$*vVmh>Wr_BVeR#j&HA|~d(l3PQ4rd9fW-14(?nxbDa(3!rgc%7u$ zHD%rw$KO619krN|-xc-ZyZwteer*#gDaPA54x#z=vc?q&eBa`gIa}J>sV?kl8^=x^ zbJ~e!SAI!j-?_(J;^M5N{)n7T*8g{sw>@WE)@#m5uJc6qV>tXJlGy7|iZ0}VQ1LIy z?Ck984i1$hPYGPGan>#bKH2hPNhIeXrnI4>45Z82M_=BcrYHF&K(_`g!V`HP; ztaHesx~{J3rEVeONjgBm+Q3%w<79;!`!Y@{_QzlmMD1`>kl#dA@%y7sN0~K=Cdu z7L@QkZ#*!|rPh2X@^}UV7o@I`Hfo1U#=Z)TU%!^2t2$^ucy^bRD+{O@bgK$L{adzd znHo(iF$C4|T+(I1Wu%f)aZu|?Sa>od)#3IWA!^VyVBn(BTB0M7K700Tp6q9{Q8JBM zIM`bT9zlv1xKg|eBI7R*^DVuNS?sBk-&3^j`T4CUkm{wQE%u6uvEcWYyV0^76{wzhCF%0prIW<0|MvR&aiTwXy`K@0PRJa|t*o>U0G!7pQbOX<4)ju>8vMd*GX>(`h zwI`T-mHf^sY4N!&yF;rK!?1<;NLb?(9y1xcPgy!SpE@h=abrqIm}VKo+|MtMHV-Q_ zr5nO*KsL7!ElEZmyO{0yQ5z1BJ7vQ3KF1~f!HBV_r>Jo&sp%Uo`?m+q|GKCani@PPB zE-jfG8aaa7kXY=7HkWQ(yZy%KcLvB`Uz(dk_%r7yLKh>U(vu=A>Z{MISGxeLjD9vI zLLOtH^Cmf}RGqvihvJN8QQwNMKidQ)Zt7_?^s@%X@wzZfw!PcvxpxXR`Z4&&NQ55Q z;B*|kYHX}wct0o^d0Vw%vK%#r)a&6ip!YG*H#V;PenMPad>Eo2@p1mZ_J?4w_;TDF zIH<@B-uaJn!gWu~pLM=>@9vhA3=6}`eU~J%tlGDujN!6!?FQ)B0?a`zL6O)t6F7cb zwd*?c6oau*^b8kQq|JAmfp2uYC#!)Fh0b{7*s)_8r==#J&9P#S9k@9`j2zK7G%T+Z z+WV4*03nzBS+>)8L~fqoYrK#Ew7;n>$HpJ6cMtl7v6%{AM$Mn@d_R0S-SvGf+tLkW z^XzS}^?{@r>mLW&dnQr!B4ELXPf(D0_sEghbJ4ldyY=gB0nu%d6zx1%kC?9`w$Uk4 zcmmiiJA*A$@=baph<6$dO+SF40DMkfyW)@%)#32^`22U)oiB>c^DXP8@{#h0OGMD3 z6FS3MsT&tAh5N$1V5yveXub`U^-GGwzpBnpQ$=Ijr>9|@3gRBmMBv0o2zKfuPG4xO zs-i)Gts->-!9QRn-_<3fkXX9uK1&;~u2yvHk}@VS95rq2f_!jsyl(yt6$#hau|j{1 z*o@*Uia}utO1~vrtEv7 zt3FPQyABD6jh@_95axK!-Go)gaZ?(!ea%Z!lWHiM6pH(F&^vW&kj%(?Y)1-3R`aXA zEHl&725H<5MFul7GgHvr6Lh*^xDgc_n#9IX{#H;>kU1NrZZZ`gg{%pzPftAUZN)~t z!n+g)uD2$TcdQ#NRZ!3+cX3g}!jho#+;CZOs^V&W-!#dB6Hh(Qo?}n~ zufaOUdunT+T7OmV%*K)XCv>s*}9-G+0-{l`TiTQX{|2&gWlz; z*D}eTim3T>FgR`&3V^kBYI%0XzfKp=7z1O#7Nu_oP>)F3a<8(qX5}k^CabPOCt|N3 zD_U%oTN$~)tWYhecAK#TQ!^#;g)bU|Ruw*YkYe{pVzgyK8PZf<{A_Jqfq|8kA4pt+ zNWD_>7!kqWj~raQG|2Xj!4veuiIpiTRGCf4$R^=R7a^{#Teq${=oml$M$mKW?fio% zx|RVka0$?jE>3nc{^NxMj7a(vFY@{MK`*zT?htg$+1?zso&Iqu3rZ&Y#5l@p6^Mtd zx+;jLt*x!V_wp(#n}KPw+Vt9_$)?#N<~YisT8o~x$J)|<7u`RQJ;*8nCcn{oR>~)w z(sQzS`LUa%#dm?crT(k;D-B8VRZ&@)h9HeA*172~CaO3+*6j{8x(1_-;@Pw6FV_IP zazSb>^Sn@MSj62t+IdLcEbS|l41r4->pc6{0>+;#rlOgyS}@@_;eOoZ9IC=v*lh5o z<&6=lRChCEt|X-EQU@kV;j$js~=8MzOVg~Tx^qon-M_ReA&wmagfkBTux9g=z< z{9U(Je;*-@6?mCJfAtfRlYvzu-Kt%bRK!qJEtUC=fi8TphP^=5GGM}4k6J%M8xf@z zf1U(|1_oSu#jf{BwFs;af8cV1;cZ-T7sTo-B0>wrn?Zf;b4HPLOD@$rs z#8aXLxgojdjd}g2PxP=GmLj~KKowzN6YYUuq#C-0#5FdhiChhF7@=v(CKXF@l^{JE zq)RcyMWG0`L%p}Nnt)?w9Rba`wnkakzEU1Mc<{e$Zz&6%AWmM1*;2y9cFNIFl=SW4 z>Utp0$N~H5TUcy_)DCFY2OcmN@)0fzLr@*a z#e%D1GdsH<(0w1EHL06cDTQBoljJbj#STfQ6#5z2rjQ<+zAAqOv|KQCPywu>G3>~9 z*hoP^aSP@!Flc@d3olxP!jqKl@y8vKFdOK#zdILzCYjw#JurL#BU8CdBHxBF?A4?O z*@{Ip9H6M7!npJnMCOqC<$8QC=U8{xb&5!M9@R^DMyWa6}KeFO<$McKmgf~J#B_9eY znj*wW!{OY>H*!rreL733zRQb~LW?YL-L(E$j|%z~r9TV!$&K-VfI$Bs-(IL#N$X@` zl@9Q~1FtWYr^@2A6qZMnffz)5)0x46Yy|Qw&J2oU-f}UPgQS@93TxMLb905U*Y=E_pdTSF{`sq z9+@wNCj?KOxAPB73N1Iz1PiFRST{ZSY>M{}#MMLl`t<1&3-JhCTpQ$dAWE-X>m$u8yau_+pygXF>2=$rqgf$6s|Wz}K8`ArC$k(hz}= z(eAT1fx1S@^k?1Oi)Hk-C03|!rt|*v=29k$V2HMLD8efgzOtKb;el}jX`gPCtjuhT zPD{HoVQVM0YY=`Zy7n_^$_@@UjpO)vc~ySBIk5a67s+f`ZPZy;*TN2mgEn2qQ)Vw1 zm)%*<=YKl;@zaz7^`>tb0DsjiJ9j32DnLd4L>dlfqsE+{6qVE@--O-%*|TRIGd(@} zeiQbPAO-=ym1e&W7D+&W(KZcPoJC#cx11awPni?z@*b1SGp*+!{VxNiL)gaZgNK}E zuD+`xHLXEX1xVZ4I>|vQ7zrIoUCZF%4q@T=YWqN9%AGpJTlBj2!NZ5o+82`3idA1& zgXy*#AuSIZmUVS;TybCe3*H2%9C3@m^~-bM6n+3V z|C%V6{E}s+29FO^p^A?mH-N%@4|@-3x^PV8F(zdt(8T(v?f~Bu;q|`ozJVS~$HJ|n zrsg|-KZV}EssaV?5j1}T2=Lc$-XvdhB>bGlDpz12%y60^dnlXD2)2yb^Yh)}_nlbgg+YBw|MyxRw=`nkpzDAV9LF z2x2O14$|C9)=Ux>HHmJ>zu~@FFytX=hk7@@6xnq}LeI`);R3P!S} z92*}u(AOuV!bv}W`RC7MRxF3Q>=hWuETElD51YicuK0bB5ilFziR{sbzDNNTrp2NI z!~&Xo(p9*biOCZhT!wwJ+dWd)0!$Ln&1Z&~c{eC%8+rfNuH6Is5NXq0td(QUPVSyf zT-ssq_1X56QXtv&p;hof!U2UsvY&VkXTzn~$=)oZE9m_B^LF9A+jg3(1(7Nq=@~jP z#Gl-J6%UrY8Pb@7XtWFit>^l4xoJnP zEx;u!yrC2>E-s|=65dHNw4lzm8uSOY!fx`n=Lf7joYG&Kn!MrShZl)lD|j}kbS9Q9 z9@~NPL%^nA{kVlJyx`ygMtot5CRdwZGqn_HJGv01dB0|S(o0i~2Nk`Avnz?P@RAN2 zyoi8qgouTB$^tcUq@;rH`5ipCEB7Sgzaej@qnA|J{2h}g2oSB~|OUIjEFP!RO* z$?XAu$qoIdtE-E4!-m9(J*2PsE>n3F$3Jr~cwM!^Zeig72)uFgKgbe+-~Z#!`uoqE zVm<~4=lfGr*O6psz{7p{_APnB@IRlUDtV8zbp7*<=1@=t7EF=lK@ZXp@SFJl#~mK? zA@aYzkmec&PB3s0+L3{5vfU#9#zXE90AZ8v zIvKvb7$ze0O-wF+{)?->;^ZO&S6lS557YQpH`#j&!^K>3E;qWFq9pTes;eIS*8}+H z>jL;r{r@%#^?%(*64xQJB_}lA*8I&CN}PrD397*QC2^=Kj|Ea)c+QkIV%(aB*(R zx1s&Y(Rp+W*oS@l_L20f*sk*qJAD7W76U9iH>?a#oD^O~h_4fYnk0lmrGU4cZ=hW+2oXt`t# zE09lzyaOMD-AuFrAtAnS12b>i#&zh>Rv_4viKu_#A4@~diR?D1Y0RQjx3l6CWuMH3<`5}E%n;i)=RRc0#=Zwagrbu z4B7bi93GQjc8S}vD7=Ro01sjBB~uUz&wJD}=7xQH_v(YmXOVCeARXX{G!mnCms~Mk zgG&OY%c=k_*&cY}k?WtoctI-Dpdr03@HGC?I$?r!YT=Y!YaddiUlO+)HD^t-KW+Tb z-=D^};@&a zfB{Cr3b5D0rggjW=8%Xhkc8j0RgJY~PJ`RJQ`HxV@Ax&A`S%DIe72}`R4^Tvp{Yk1 z*NQaSeT1qhFUSV-2rP$8=z|3=MQH^5R=WWP@)4Wn+odLQMh^ISH$nv_$!zvYS)WAefI2YiuO_HKm-O&-s6YKY1i+nM{PfFS)asU z_V!+QHgHgSTQsKz6?DRGChW3ciLQqgAw|m5E%3(D!oj4{5{JH@9ApD%$IPYAHvWq} zXfn+gjer;MKcvysvXqhlSp>(R^X|iknc-S5$IN1un7Qh=3t?xH)L=6>Xk5db&2bz! z+Z_nNE{F8!gJmKo3}AVUL`!Ek3gb1{i0(tS2o7=$F!)0@sfR6Fn1}#mEL;fb_#Eyk zk|6yXvCCK*$yZ@v{EiX-?x$)RPfFGPXQmjvK!2?NYpR!a(ytUb=R+rAmGwC|#{g69&=@ zxB`PRbHG!sEe~6pFRibv781Puwpi~-u%e6l(9^|_*eb7IzaHyGXneg^00M{W#0gH2 z1axIQ1cq#4%QLVGiN7cCsH2He0B`Uva`16Zq47wj|J&aE_pNYBypVmsTEaD?WV-EfT0oE0tkJ9{!SpEHo!-kK+k#E;M|=O_U05T_jexUlee zo-2|M=4%OJxFPmo$1b)8x7xaQbC?+g$+!jAK~AV;nQbLbGy36!B9WNJRulooz-Mrx zrO@DrSKM0;^Fc4hUp8*s2=quYNxyzP6uJ&R)O!whUk(=X3nUTpcX0Dya)YMswPdm# zCn-3R{0P6pZA5m~Hf;w6*p`RmXSVKY$>!0~5OIRZCz@7FE?nQa92c+3{J$s}D_ z&qnCv-`IfO(Z|n^fgI$quy_cL!U(ZPX9mM!x^<^`Zb3l>us&J=Y^DmpvE5tl2M+$h z=p{Kr1b4Ksh{zpZUz$yvQY+T`&u@fDk(QqRF0#B!aShU;7aD%b09UD?0W1r;64)#o zhnK$nTh#)RIu2uBWbu975Wu**wp zYagD2)ah^A_kqOSAVYj`0SyfeOxw5n0Q&AZetZ|)bWhrSVK;|6?RszddgS*g7*YsN zk^+NE`ODX(TQ*Iwv66Ou(Cc7oFZY|U5?e-ES>O5X|!g0u&NxvK>8**_Rd zv3rp<;P5piHfh|*qdRvf9R;IcTQek+4d^I*M9bnFo*y~a3NNic#Nm$vnwVLD$lZrl z8d{=nNXQOyfDfY+KVWr`gO))!Ks~&>UO@ernQB6K7JV6b; zO=cC=t&8(&>brR=cWBq|-Xo>pM=f@92a0{fZ0=!>9I~>BEZn};%o36Qtd=>PB^EguKvEG#3MY9 zB)Zhq=Uyu<^um5}1-7H)<2fK4&mvQ7tNDXwe<7DNVq;Mqb?xj#`o_jS5&BSY4|naryc9`- zxq_z=TI^ChrJ!&K1|YgxxW_92Ko0_Bl6+N1J|qN~k3o4K!mhO&g_-GqW*vNaAHIJN z#;n&T_$@zRRLijwg5FL{togtQj@)1rvVZShVc;9m#*KlpmH{Z~p^`45E@w&J&AH>n zaLe21Hv1}J`aTdHgg#j8TJ)Kj%Ry^e+S-R8J`YTcqUSdugwf6?U%2Gv>dHcV!lU5J zTJ=rM$_nH?aKIQu+jjQ6y2e!#<-8LQAM2zIs@6t}=loP0J*F{#+j8~i;NXXt_;~)9 z#A+v!*P`-#n~|ZKKm!|CD9p#2)|UWMb`qbUABK_}bqPFkC^9l`83P7Ex){9+g#?bL z0Q#i1DGbYHnyVv7qRM8lTjFtq!<-{0E$!K{bq)8ho_M33vk67cI0UnqMnE;F zaaWAl?flSeq=OM#NOY67>wvIQ33i)27y~x#l-YM#S?{62L?$P<7{$O5r<7xR^Jdl2 zZdWgoxPePg1`mWnG4&Zp4;h>gfCcx3&%VPE0cmI4f$2d!9q&Z7_?r3cgeO2-F9~4* z0@KOS0G2YmjXZh*m^p}T56BQwc_{~T^CWPuhljd1xCNpVE;AtddpH4tWauFayhqZt z#kmA?qs@%S>UPcM#%u~SGM2ubVWN2xB~Jm1vblXe%qvyHpxL-v`lKoGw`* zZvOaI`b~dL(BVakmj3v?$CH7+eCk8R9_SO)hP|oP#|l$)^0pA9$0-WeW!FA3JRAX@ zYpffNmQzShXgux%(bUe1V}`H-+p-5PKoX;a$|O9$e7PW}piuGs`}Py7bI)|@6Ja+s z08NNocx`?C-NBgo9(WGGZNCN0NFGE1wS*J}P{2r>iP1Msuv0)RV5%@RhK)tyBUF=q z7(J{mI*PTS-7o@%ClNha-67{-#+sEJ8z!L+@Di?CV_Jn&4*$ku8vc~3;R(4}yHavC zKiD?(j4ALzR$Xnm)>8j`+Q#+U_(HM+KaayW_5hcj*IR}nedo@di1_&U4BHG4mM!@X z{NTX04TfUQM;0O+$j%Gu>g#+ATI3U;Bd3qx2TYutLPsqF-(WSOfP$2UE-kUZrW)mB zUr89F&QYklZi*yNIl)lCz;D!OB+gBkN|n2OnH9hdCLX^r7iL6V1P5o&(nHhM*Vixo?j`F( ze;dfPTogRBs}KdRnIsD716+a;$jY&NcuKio?IkH~93s-<$xH}nr4z-)#Uug+`Aa&V z;RpEvby_Rq>IPJr-OO}l2euv-)DFqd&Fv|eESANF>p>rb+-(Hb`WoocZ5&7br@x$s zcS!13L!K{YS4dqH>LfW7Ff}_%9w^c36Gc`QJC~6T*`%vi3$51Geg0gDc4#v*bBsvQ z*tf3@dO~1b?UcQ0!BjNl+RkAkqIvL>H>*`EaI|`JrKDJOMz!+mSCtVv3?dQwRHxeT zO4U3;%w*w4?mc{{7An6W=?rf)5rNKwb$l7^8?pD~L5HhpY$Fa4Ngc2Y7v6 zW8kHt3e(oseP>H^y4@c_)mG|Wgpc9ittow4Z<(>KqL<^5@=})SL*}U&qdE@r5!plM zF2g_kMVc!j>QNSl4h}nM&r}LRoKp@|5O zKWn6n35TsOl0?n?6zR-Hhj9Lt*lo`;?HN$$E3&f@Y> z8H!H~OH+=sq~R37E~TBJj)Q+9W&^&LKk#Kn{1B( zoR)7wfy5^EA?E_23O+w#vE4IAS&@x}rRSkg@r9SBsG6ksT_s=i3U~kE%_=Nhx6ScZpq(P@Td0Ck1{`8Oc+Zf9CV~0YvT#`L7IENOTAr z^mAbYO0r5o1}6_>!CIHBF0et4jG2>c2%Nh`!@#-@T2jzRAVajqrRI~TFkQh7n-f5v zTE1TX@xT6xRb6y}icG4holy*;oqp#1|JBDW68fX4LbTN%DZw zgM8O)?}(5l$Lt?-NIrnz040oUtx3XLVkI&|uPjNiUiXyC#{c9sDf-1{{k|*pwIRoM zUwbO_RJ*G&)$Wh!{ujJVfrbt}R;aOeC{OYhblgjOxcPOe)|YN}dDlua*?Tz4#4U=? z!g`;}#y+Lm+US$Iw+<(~IA+#pi)IKJg*>z64JrV#mRD%#PQZEw3}*?`d!e54u-hqy zLSXQ!>f;g?W->9Ep~yxv#21s3nMpOZFk)}4S9mLOdKv;@p&Ga0DMjCld=f%F>M}Cr z%L0!tlP9^LH6yohxBK!rgQAQ@#kPlM9DngSz|=u9DmaLPja|ZAbca901yB-qVFrt^ z(ynyD$q#zbu3Z|aQV<+U>mpSzn#5g|;&!OuG9N{(l6_}JM#fa1{|Uo|vDTpAn-0z) zPS5Yka1jw&Jo3q>d4M%4XyTlR-XQIZT3YuYqyu*Hru1n%S!g^YBGPb+ z?VoZ}?HfP(lkjVZwPe$*{EN$Y8?DlG<#!CBDr@FjD_6V8eTeT1H!I@hiM^dP@<-BJjM@1g;K%wZFfgoNA>Tt$%e) zt#3%=i4^-9X~JA}yAo3p9&yCYVeGT**EiS}{o_q+w% z14$h#vGe*sWFIIb%p~Exuni36qc-^SdsN78#z_;qgY>lEOeIXOGk-rbdUD5(9q)?t zjmKLvX<$4c-8^&iM@B1om}sYP`~W2ZR_lm>fU=p<`B9k+Uj>674F$f{VZ*tOnlENisNo5Kc?FP-BS>jZCUrjs`}yYBp`SK zT$3}j-s2E}NxL)7CCaD;MSB-3!hd*}bm_y@72mMxG5h%_;d#t{aQA@&e!8!5G7SD) z*j`pdgWQf!?Jm}}=E=eA2##>k8xL@6drDoxfpc{t$8~jdz+u|LWtT@>#%Ua+4+u|$ z*U|`%Hgv*KIAVuHF)XQqARWvIDZog(LLsDVZyeh>_y zwE<%#=okrA2*aQ}?J{coJ?hj(a%7vNX+Vc^Z7CXOb)V@!4c7$|av_?0_aV2~_;?&+ z!Ep;k3yJYAB)@^M*%g)p!;_ninqu6q<5fy}#2?>p`sW-b3Y?mqna~}z9JX^2Vvsk z_UA*>w=5asQJ-t)?>WOC?Eb5lhA0KwJ&?^0*Cb@(Z8H-1fWi83Mhs{`e^s~c1d&S4 zNPE}chs}#j^7rrGPtVCA`7^%0FaH62M;c?-d$oj{h*ZoR$;=GUUgYIt3fxm5?U~gf zv-j#i?LD@r(a?q`CQq;I&>#4y{cE~~ z;nSaGag$Z2_1$akJG(h@ABEDS=}$x^YyGNsSh+q&9lVRN>cQ5d)87)?fAuB(p!!v_ zt9W|dQsrMkZofF$Su^|tjv}f4t>W(6M-7JMJqT=|4w@B`6v84pFHZQO#fS#2LRz%2 z$@Xw_w;J4nk`0c0jHC%L;YkP1!56bQ6Wt~Lu+Ea?4Em##@?spcai2W34AuZxH%W6A zi%%IWj3g&_%h?%9Ia@KNiCA$4iY8`^Yf(6g>%oqWO!rl)3PJ?3425R z*RPww-jb&$kjDc37bm2nvx|&TI4#T^Qn3Ww&H8*+JYk^LmeoB#eYAeiNJN>6Kw=}u zAaUM`7v@HHb8%5X0;eQ^;PfJW)$Qyubi5J|r{`eWas^B*vA04WXV`h2J7oJq?zrblqp->!r{2Oj)Ftc7*N^)f_SM5Rdk;gN^ zk@fKVP%<*en1z|JJ-TAcFW=y?qM(?mFd}m|hH}AZUQN3Xlh<8u?@07uu;jf5$v2RV z)3p?kw7zG3Mizg2%c^lH+ z-4pnrYi0Wbl|9TScQZ%I@yOp-oGDP}DIYoz_NDFg^z9DsKX>tC$_ino2Ro7T@4cs& zGo&s*n&usqQ!8h9kR$f0tZ(fpgZoUU{*qx1rhO80<RWJs&RR2e&N1$yzCR9AE6O zQixoj%H+_Bbq^(eVfC#zDJ@NG7t(u#_NfQi8ixW_f<7XhdC1{84GT@M7m}7YHc`t` zoZzFPrA13XJ&eJLq0$Y{lA$`bWSTO9wl}b}e1xKIurxDZ`6X5WXAhEfpKP+pF)OGR z5m{NxXy9u-QZo*T&td+sFqduVCYZ?))=b8|6mg9rvHeAbSmTc*=9Gb>lox&raeG9M8Uqzoh^6%Y^*X+^qGQbJl35K!qZ>69*M5Rj4( zX^@nb?rv$277#dd?fv+?=bS&z_{QhhV-K-d_gd?^t~q}-!I^mir>rQfYZL?SQ#y(p zvLLqvEu@+QIgk;j>F7#svMD+Cq_ z4Vc?_Oech3FG1W$K(07L{Db8rFlpoL8mT{fhL{eJv>kBfK!FNW85;=V zkUW_iHv%FEAjFIk!J(iLyh2Lq54|0*z#v9MKo~^SHG+BX@47033SSj-N@5_eERFNg z<4$p?bMH4i-St%ze{mnf?1Eqb8qF2x@Z!cbncfh`fAHtqUU_qT)woBBLBwB5KrmH` zolMDz`1NlalZH!K>1ZjUJe2doO3}_UlC-YZ6dZ4&!tq$Oof2?w*W_yvW@m}{z5jCy zOZ@!=dgWq8e5ktXN2Xtu3JNSIr=jK_2rTF~jvtTK@q?q* zjA@nn`hE3CxPQI;KHS8^uPNgpaDgU3hOX93P+t6Zea}VR|PJ4Xq1iqhA`XbS+tybKVklV3SlDF>P z)M6U1T5iqDjIvS^Kpj^;B3WIj6Oxx~Gt(wn-i~DDSXlEwe|{!73|nDL6&0K|_B^F| z$LPm(ivRd6vgT;OQd4BXidK*p8hYBiWM5B=2bj#OS({sLElrl$;^?V?P3J% ztdRjd6ZKb5#|NIau7$iN*jA7w_*C>A*YjBQ(+sMWi9tfO2Bm(~g#7G{5X~ z!b9pHPUk1v&UB4A93vuNlu~NPosI*YpZS-K+DLa^xpG3~u2yc*AQy-7Gy1E6G}ryZ z)U-Upd3G73m!2u@S1QvsOS2qm_y4l7-$6?*s;3F^ls52`3b%25sO5cF`$--8;jrx$ zq%VtY?i|Ire0*2uo!FKCD^;0+4v#Xb8XI5Q6W5?0WAd?(MfS4n!`X2iNiz=#@eDKF z%U-wqns8l@=Qsw~=QS^OAsX23nPrg0lbzx)F;o;B2*B_3~TnXvO+id)7E8(pD zha-;%mU(k(;T;CT$fC9P`!@HhJ@4Mwvx|uz56t>EGCKWIRq=BL8)EnWh*?@CnHGd^ zg%@4ccf5@auh7?B8tJne!nCe8gGeW{u^*N{>sSQrn~>y)TtxB6YNPdPBo10DX8P~S zM?qyVmpa&Y1Fjt1N~E|d|M1b&shYZ<><_OQ**a-5`<%^@tG?(1?0vo&SEz8^GP56$ znN>OH-@!G{C8)tKXg*{;enzrw^98o;pNCeBi@pW0NuaDm1*&h3_p!NX{CyF6Clvm^ zu}qq-7TCzCJYSYf!(WDF^8_!YqyIC{6Muzt>xvpyf}^oZ>Fow?QP+y4C#PqeTY4PutoK`Ks5t+PX#IOxt(f`nmPbSmm&bhYck+*w>SlQ{ zOoZe0%lL)wz0xFMG3UyLQjz8Rmt5}N z{kv5C=U6Ia@57_QpZcADoSRi1_u43lRG}6!Rg_k_u6x&+LMQflxN`wrDRF@n4tp-sJAW6BjB9%^@}Hxc&t=KQm#C@9|&>-mN; z@%V?Bh@no!3f`V|Pf9SUYC?5};+gzYt6!X4OWN?6@MF@fu^v69bN{4a=EzIBx&m>g z%ElEr5m$7H@A^8va24c9sMx*!z?-fx`z`)fb1CN&sw1O+U2oa-I1!X4kzGCc7DO?B zzq^!dO2_hNhx29e<4x)`-ymJ5i=hk|)eL;JBKR#v)%Zbrz~jHn#)1ok?J+*^ z_LFWE)8-H@w*1UersLZpeogbT*f*&d&7q!Va+Y)$H|>T{~pM zxBp%9T7E_A5>X4q!*x&pjaH){^j9%}XB z*FH{wxB-b<0^4;^nFDxOkXSyDT785zea5l^Bmtl;?ln3=qyXUF0*W4$+${;!W6zhD zlAkp*NeHE-r0RgGx({U{Gy+#2J`9(Z1e0L_`#C@tK-s?pwHTz`djXFF=5fR{28@j` zgYtRC@k>La;SZ;?J9wjuS~aN?m2n=cS~~(@sDVDE4(zz|Zz1{}DO@3YqYh;C5OJ$9 zm;t~Z0;?lpLMZvcF`NeYIntkltE(VXhxx`!AleZj1BeG8lmNjXN^Da9Q78ttCxhGB z$hrET7dZXrl|WLcChPtEct1aMxP1A%B3Q6MP~?e-Pv7(F)Uo%N=rTi3`R>+|_?kWP znKmQtajw_>P%Vhz9@edLDGiFv=Q_2e<<&-P%?><+SqY7NH#8E6k;2r))l~q14>T0QGJ<^;?#W^w7$a`oy&L$}*^wBe`YoUqL6qfr z7jVSjGf^Rv1HccGM@Atkw3#fq}AOe;v zAryQ*ApSEc34u;z9>j+jAi4%%8DW-bz{3ojBltTnAz49)Hm9ej@Xrsal8+A$Pmf1J z3fQ#)bpwnmx2F_mq75n$l{%U9s`%Co-`wOf;wjwo`F=m6+^J2J2lb`pVWwDJupF7rQn z1nea`;N&V@?u0w|kksc6ZUe-W0U7g1E;sx|)lFMOiwXTMk|agW?|2;&IiC;vi`@b+ zK~gUsAQC-L5FySJ2zgWe{LCF5YW8FZq{Sd#0LTba3gWcSK)i)mA^_Mx!VIA4%xF@K zSQWBi&CW*b$FNWe_h-O9f{67XSRH7o-U3}g;2R-u91A$g7)>e)3Oo?1RTv=96Tp(Q z;AMe7mIgc?3JDECR8@eyNq|}eiLqjsT!M!O0~fTup#NJb3_?gCZ7nNCNQbaC$B{2n z(Ek3+8-su);|G=EC3;2R6>3?RPP(@!&#R_?b(7zv0TO-F&~J^oJ_L z``g8&%*B-GFo8I}XCL7!GL7L2x-q_Y7z-?ZjfwstCJI^hdQx&SuxQa+@rC8MG~ZW; zga{4G^*KaGq_dIP)!BBHU3a@2%|Ur0(WV`&#p_mK8{`# zR*nqm`@_6IH-Y(-NvP5td+#&(^Q{y9@=E85Y7BMhR{t6-{18lY`iU<3PR_(11Sp6ajE}ewF{(5pD`{AUq-0%>Xb6 z{D|iv0MP-M2@vWMwYZCz1D^w@g>r+_r>*m{qHR_NXbtrs0v-v21ud^)t||+%R)O$v z8SYoi`19w_K$k3SA{E$`lCc`xFcf=$)(2z2hi_@YqA)RYP<#>ujE zt;YatHiBG04}87g$OSYGEp+d_7WlC$tydblLLi648=QnRkeR(`5g>MJeX@!e%)^C+ zg-Dn_GEo2tDguU4SxPtTa2GZl#9#%|Vg%0om&_0`IBvfI2B&}7h9@*6`vTHVBVc6B z8EaRV6!Q?FCD(RuAilrOF1%_bc5g?dp_hmd8)xv%>6v2UiK~&Nb6i3dDEW}{9t6IK zeH}ur5px4FZDZ=2=7#w#5H&QweMde8sPu4NG6wK0B3pv(65g-BkaR7(mgxn?`5-ov0xmNz*1 z{Tt52IVee_q@=Qb%ptjJUZIB!9JM5=1QCU&BAhjX)_CKdAFmvI@@PlpV1|(%`c+fJS6s(yBm_)-~RF$1xYG9i9t%HCxEG6UKSI)7iT{BRbqL3EY?Gj=sO$k_lygXKMhlOStu{w+oCyCB+;41^C)DK zr1{2n5B(ngx!?C|ufv>eG5c80gMa**Z>@2nd~jGmeL6;yt~24;;H##vw@z=lsV#QA zC>Uz*T+(oU_OXF{K`=yYs(-xI>1yzV?{Juv;Pry7i}N;n7xg6*MODAQYpta4`|%5< z??oYQJMbW*CG3!vyg<%OSZPLrHc89(yIXRY`5Ph~O7-BFclYW8Yj4~>n{j>wb!yU9 zviA~ZjpWaud6`|A+5?_&xl<3dbJK=g4$UxU(>qEYuA-~r@6s=ANscXgO4T}Ikq(DY z?>1ZLPd&mG_SrcP+DY=dy|u!!&?5ek`ivFl+6%=GdxYX`@(KyZp*&1IL&N@Hr{)w(~mwX4~7RBXR{J05ZG=QQ#v(qIfX?wY1+^ zz}C{HqxwxpM~B$&An5>yG6WO2Aw*CG0N%ROO=4nSc$-RRpQu8_1?Xe9VFHA`lT#Wj zwnAWw)q|mYA_r3xkVB#Xd-uR0ZkxEi2Ifgbo80>2hZtN(4}^;8Ll_AxIb>iBvR6Ym zrSjNOHX{uA=>fGV5F-ua<2T`4XapGd3y_>h>Bqp}=W+o#*O9O%WF`pg@NIaz^q|@O zf9jQy8XT(S`Tu{S<$~4|&_B8&CQRH$gi3Ydf94g492mqY{&JSrgBZkora8HoZAC@@mUs8`B zi&|TABIAmHmMb&{`9aUuSMta$gjrZHPN)Z>%RoWS_I8XDaiW4W!cu;85<3Im=#bV{&&%z;4^i7w2|y$hQ^4kjjM#akSl zsWVTd|Az><4+|MVg{a_+pLWux{EJAch|=DecW`9gq|?7efB#xR(dB_p-FKPl0j#Zf zkbtUF)+LaCs`Jv$oLO$##XTn0fyvT7VoHdBKb|i|{`IThoL_uXDY=$ZeJ3RK2!9$> zZ;UD@5GX2Aes4M=y!<%Iw6S0wmqPSNZ&4-4vcb=*)yWe5qsMT--KJaVx<}W%vAWI#8Ro~P7rQD{qvs2$ zQeHBj*M6Wry%^qYiaE8n6TU|>UvnCN^3yCf>G>n{YtI~l>fr2^0n-%Gcgb0mnM=I5 zr^2~Kt^rw>JZW3^7TT^sY@dd4np-5KQTy5D}Zd(g=ssWM=nKuM8J$%8%%yr>6l&MW;VAd~3n4XZT(?j0MvvX7h& zaDo40Eh9&iRuCqE6yuN=!IbDG0GfGNd?Ko|1DDa-cY<(8BeTYS!0?PkXEPKsNe3BB z17(<7e*PccR`AA%Sz8x3h;1VyhL9S0(ry|tL|MWE<#;s*l2xPW8h!|*!3X`Vtv1*J zVd;$2-raEpiOU>JTSD_~tcP-bG;@d42Qhg;cKM|Zxi=S^nwu}6pf5qZ>>$Nx&(j6% zGDx(cfAwWzX#uVYO~{m_PQK0echf&>D`GxE$!-&5>*kO(ZoM_7ZR*zv=QrX&frAwZ zX@Op5#0Cz_X4sJQphy6=rEL%$cCn?dHbEOCF%KbXfvy=Tx43O8g}4G zc__f6hQdy1X_eJL?R^?jDB!d@j-?;GN3P~8@`)BE;zJ$KHkH{5Lb;OxI z89o2LIDIFa`WpJmg3*gqR6~PSn+4koA!d(f3|@uhLaY=dpMI3$)}iX+7jZqSGt24+ z`(qa9Zg}d6F|^4@Sd3k+ur6|7Guv_d@h~`FK&B`n?8b4XMCR@J=KCWD@#?oTRkNf8 zj@$-7uu^E)NroJ0@ZKu%(JI4W4^R)F39XP{=v#!BQ3x1EFy}#+qeQpps#b9JR6sZI zp-ojVTDQ=dy-bk~&F%Fp)uUgyPZY{>Ul4@+d3B}L#a`Al_U&}L(}giOf&e3F1X^(9 z$T}VGai0XY@$D7l%jVvlpX{ThZ{sEK(&B&={;#f^UmG6lp7xE06^sM8mS$?|@F9px<)83}GaB(v(zUcWpt1mpN{ zi$I9|`!saaFiEI@@cji8=&UQ)RToYts_gC|)84F?6)I>xLR0Q~ymALgr<9f+bHUE> z`~M^thio>}s7O7KZvlek4t+Gt7J5c^lErin|J3c{LJQGYw9B#Yl@7fAOgX4%T0?4c zkB#r(k&bYB_4^Sm-eJZZtl$dCv3g}}>p76yjYHA@?>Jpd-heCot za|k1cdV7TtVMZ-XCk%w{5P`w~GmE**85bL?2ft%(Yl|32N9*JupB$14^+3`M+Is2y zr2!E2uz?bmoIhvsFeD&!$P*k;5L3|t-)LrSEf9)hq@st`^gB$wP=M4Hq^m%|EJv8D zL;$jeda$yBaXF&nX=$1I=&TW9ZvulA5*X|WV)BJeP7|A-}HJ?F&N*3nJDK|+% zteqF(&dLjtOCtBtwUeAhtFqlGardkvFQr@C*c<7;3`WpgL$fygh3Wu9=UAM8c%u6q`K&)4%L_n4_oSn+tUvFHAm@E>?h zekrdF>4@#!pZ&O$L5s#jLw&WVo=Mp8(cJlmyVCb=puJxxd!~S8A>BLf7R^azoMh%9 z_Wl_zTlnFqgHn!Et@ekaeg5H;a*xT!t;PX$$1LClGgVhS#juac3D;lTBT zJrxNmiqf2sQA#+_T6-I*GBKy*KIE`MGOSdo_3XpbR7+!-v2sQ}U1@mSpr$wxD1lr@ z0LE~PQ4Dxq0{|)YIr&N@Gr<75!oK~!d0_Sn22O`CK$&ecRSj;D<{-KVH&yHZ$myT$ z6Z`((a{9KI@1V7=852$dw_W&@3i7X>d#w5gnO(@#^ zQ2+vG=jJY;jD|jw0(I{L0P!3WZB*mJ2Q&3_R!#_Ta1aR{#1DS~88v}vYYi+F2EVG^ zGuu`_MUr=iryB+2%1cAc(C@)n3M(B^9f6AIIf)P6ht?raKxZL)5{7?ONbE8tf*u0} z?{IF<8XgK%eTe%2fKYJT*TbN!Z}>jAq#yTAMxnQzek>%TK>!0Tk>M(c1hUE$geous zwHeNwFCbvzroJfUD0g&xj7<80*0wH7F)bxGHxN`LNM1hFg6;|ok!xX>gi={;EG+8n zrd@nN$xsI<9C898mh1Jye;9>Og0K{2FG;=ePLDs zG~b+MWnz~xA9EISzj9(jw5?_fRtQ#-GJ>%{e3z!@WaA0Anmu zOiKwvSi)nX(7vxOVy`5Nf#8K6c2!W4gZ2nPTmdb|m-OMF8_3%q=T;`VVX==}*UuCE zhvY+Rbu>ndevH+T1a#CiKV&71O)tspveWL8x1A;ihrh>Q%V;8t3b_(JqcU?s`oPyn zK3#u`?bcQVz7!I>4Q5BAt&S2P)lvmF**{tH08_~HkBp2I7Jy%^m?;E2^X)c$Q>qUh z5rm2~!h7f++=bLKh)k2LQ}?iQ{qb4;YYUTq_Q2 zdQrid6aAR~9V>Qw_Nw=tsIV}_%_9epT7kPPmpkYalA5ll)7Q+m!o_f^siC^!#C#ox zs4LUc)3fhoIR1vRvNCO1&3a9Hc+zW?qYL3lNcgOTmR3Uh%l5W5y)y9hV^SN_4lEph zZarb#IE5Z8)0(O;Ra`||G%mD;V09H|h0PmU3fP2IrI5M@fm^@`XwsoLb&~$$f2;_I zOb(Rjb{RvMTcYBtqj7F}k;A=XfQe5UX)HzCAutEH>=4Ziba)=^r;uDzH2!$41Nu|Q z-~M`7&IDi$I`xVQ-G+LG3?TAIFkdJ&shO6_>01b^5IS>YIdOPOY3yiYlfAbq39>T? zPgaluO%Csy$w~y1XJBA>UQ2ZVN_-GWH$f)bC>5iwt}XyW2(}JW?8u_Z!2Y*9T`LG> zW-#cih{0;92+sU)=gg~+R?*S%lB$?G{d3W`JuI5Ve}YH`zn{?O3b~yW6j=x!aV2Fe z3NtvKBy!J<-nF(+Ovo)Yrnvn}FX&V0&#c!bbq90uovk9j()1H*W4!LlD^U~XKwMm5 zKBFUNiD>svTO;M+_nmj{$ye1$%YsqxgA&?z4`;uN7sD%}F)k7k6H7%C5!Vg4pvoP0 zM3rLFTW+uQndTkPwRb9w8EVkZr4%c>D1e+;glfDds*Bbsj@!R2`Y47l26TJMy|9FR<_Y*ilH+Tz&@sGZ#?$${4wYNEaljhovC#9Wq)DX`Ib zguckHVt>gHtWlr9LJhh8?w}1tEXD|e2fouqAL@+wVyod@(=bD-+}v)w`!r35PGOc`S%m0@%B1L^JdkEkawa0Sl)Se0U?aPNjo|BqwCxgAuB+oTTV$$!WW zb-zhyn1;0%w-sCl1&9!f=+V|Ex#(u|qR3)ngP+9nbDnCvRPkOnEsPn6vd?QK8uD*A za(@*KF|^H6t1^_^O2uBOmTVuxvNEOgE$3CcYVLT=naS+Kqj^JZq6e(!Ozb z`Mkabt5YoVm7-$Kxo2pPP7k;weUX$?30kkL87{G-SO$A%ZThT8^`AC3zKc^y{R(Zo zSgvKxTg_8jfr`O2=QWG5aA4~HvJ%BIFBM}k_$lHOnlC24S{pmQq-$*W+l)oWc;7eb zbS+Ou9i+Mg%az??;(gp06}4X#X2zRgh{haj%?6zox)h;hgJSXag9oHgt$C%Ty|f|5 zM2%qQM}-|EDG_TkL$3?Ee^i^-PPY@Z6xc0IltCgW` zLZ)9^oF3aCBa}mNTWFo~0hB@p5W{Z!MZOCVYH`Hh0dfe&D$v^ie8pKKuYf7J_6oHi6CZwv0Nh zW(+BwSY1_G>9L1CRHnaG;0VJ1@?A=}++p^j17ZvA%i+q%k+8j>?aKL4x|^hGZ&&?SbtInD0ws}aqCg!6v=`?%;| zQiR%i?2in5emH&2PbVqi_iZTtGIuUlz>zGfMZ1F6a+Wd&!j{rT&qcRt*hVH=bLtyFbt^i_*cG+;t6?-Wy@Hwkzz7 zvZZ_f(J_5^x0@zQCG*b;zkp4$zR)UciwM92=}^cU_qsaK`ghL9TZ3168ydD-05JjLWL@Y(+ik)Sh1P?ldqAx0(e*L@T5tOqZO;^4Du5FUA{6B2g~EV>1TFGq6cEM3b6Ee#Nd(?r za(oEdC~V>E9UoVI3QDr^A(6{Yk2$}ViLu;1q8`}Y)A~aUoUC|_lv>q23RX-Vytln& zB^&}~D;RhC+o@(#i@rT9LG$=bn^$^;h>^-8EYs|>rM5cobB|nJX7Fdh+fukGaQ60> zzZpjU=J=uaKjh!^kNU{qzTs?!AK!dGkDq_!yhqV6@xE-$k^^n~Kb^H~ zUdn5Yo0XqbWzO>PwEM@xr_|ZcZ_<%#r;eVVC5h@Ol(|-{NG*QZ5WIPDyO+T)L@Bnp zwSw!-Nw@1e?_)QYaI|ay)L#BR5 z<|l*o2*K=Of=V46gVWBtK@$-@xQ#PF2*N(VRkU3UFBOE_ks`VWAZ&{owdn5Lc?m4w zVi(s(yiqIlsXzDHLBfm>P@tqoBoc^BMd<2YEQ}c7+wx}ywK)Q~Bk&qRHNqARIVf@3 z$v{uSxnC;$?bg;3ECOVPDIyX8WYhz4nBr}>=bEC(2A*buHy@e30i%=z5f347_rjEr?d z=47)9g4Fx$&#rvNIBcR=x-R=f^zuo#fMHflB_N1lcd}%%fC~&6Al(Om5vbH=(2f+k zo;t#4=GMhS&?SH!{v|j*zy{m~xb8gsNQ;4+AEjvrhhlnqI%3b>Bm?nG z6S!v}cnOI}^2Q|kcR?6{jEa)})x6FNda{B0qL{?5EA}5gVvQS9r)EEU%a&d0j|SqM zkKUeiT6vwRcl!fDLH)=%D(HN(d46w_820lF+j9^q10pY)tlG6a+HmEk57#R!6A5M! zX(2#m4XAt{68!m*D07fKb00!#DG?_S3r=qT5k)OsWFn{`s*))Hgg|KgUR_ z@z)CZRP-&r-0>0)L5~09fT=x=cDK-tUfStE>9Kx)Mwbw)bj$Ld`=tt44AP!Yir($w zb}~)eT@gK-K7ZLVf3v}646Pbx!tH(GOUtwoI3N|O8vkX#rxj#b#a1J5rI3+qD7H5v zY$Yq{c#Kd1G=s1U8RdmC6G)&G5ChKYdTAIpmIW_)sYUz%Ika@Jg-C*H8Z_u5cM3ZI zO+=<@!o*#%=!4)~B%28kG*+a;fK5IPs#Z7$!Aao_uOr-c3(&|QxeQ34HY^WhPWV8+ z_Dy*4`~m`yDK4;(p54AFiC|Gkke^#=v$0c=Q!KB=Ew~XF;N<{7@AuIWBY!Y_h}WDk-7R7Vf${)B zqIS8nH2Z6RD}xAO_eA`20khdLyv9Ue_1*u`KsYk_tE0a-Jw)FHn90%YYdf9MUsP&JYZ05vQu$Y?x6e5!D?U8HvSzyLU?X*_Jtw;)mtrrIHW z6O20sX$3Oq8DJhgXp$h9t#X1J-V6kEf^?hSpd~~I49J#$Y#I;7 zI%GQqXO#z_2#9nJlpOF5QA0)<9FbU9SXto}1OKja+7$=!@|aLW5^Nv2Lv|}{;a&RY zKQFu?aRC&fK^KIxX|W8<{3{K8EQQg1^a%Ia(a*)3=BGWc(XUd6_V;{NQ}fnHmRj{D{& z!XChqPZExG3$MAqr?S_TJ#O&6f&ELN*wrV{rc%$--F(i}*oYy?_mMnIUq@3yDZ)A0Zk>O*dxK_FvY`SV~h(ZQHc3z1keoZ ze2bTqb1(m&el5_B{Kv2LPYnekvw~t51lfcUbqyfWK;qI$edhttx^WmmfD+RCz_jwqA4Ed6=lPX^$8cf`R=8dN1*_5Z zkM{NE>5GsKGb9@Efa`S^g8>+^G~&p09T1rqq)6yNx<|R)7A2Utk&rFq2!^^rnJn~z z8)Eqc=mrUlh8qrPaA+`36jHdzAkQP_@oNy>KzE4bPr^+TK^Sg`heVumP;odA>er=R7z#d%k@*$x|`v6iR3~LhV7_ zeqG;RBd>Jm;p?}FDpkIDr~Bh$W_hP$vspJwh7?Pf0?-YxTc@17W4|pWCQnnj*X`4X z*3r?-?mIuSf8O;V*{aLewVP)?(MqnaAn{#o_K`L(TJ40mLlN)Cfs_0t1DhtUs__=A z-|-@}(-u`9ss}S#1oBNMRu&3a2Aq-gQUY*9;%mq_&#sTO))xnJ&Mp8 zcV=|%cTd{$&di-*6~>2*K|S-#FDxV&T}K9jlO!??*~j26f7Q~_NrEvVG}3W`?`mL* z2%+*jFte@!f&>DxZ^iJ~QeGl+;3ic4g`I25_&zx~(eECoLHo3F z76b>1xjR=E&f9PLltLYAhU2<5sbl=FX{Ga%8V9%BhsRw^>qY~*d0KvP(P-pC#FkQv z7CV8p>gSs&8*&Dd<=b{RkMYKU_J^H+}5$Ss2l<_9#PCTVc>D=@Quuw+;#*KW!|^89q`9nrOG{gE0l z%hIAo5Jml_RNztW=+3d|xW{VOc}v$p+*RabC6rz&kk=%Lxhv3@ny~O8+DRr&U^BB1v7VXVuQ2x(QdKVkFRS3@S-2CTKFzVx(RCA|}Iv_4{S zDaGZARH6r@-)q69HvokN=Ko111)&2!{xbk(G#*2KP-@< z%Ad|eZz2k(jn6xp7W(I|p-^?kyOc>ll^i1>(Q!mC7FA^O;*G>}9O3uW?l#%DHnKT(HpLo(q5}lUq z`8A4U@zJ8)1Jc_wF`4TM^WWsf9ZVRyA__ifg-5z{dKO90VU*a>+7Tq0Y30CDCfj-^ z(-y|UO-Jj-{zqVtV&vaTG`aXFm9I1IWuUBz^|WHZX8-S{o`l*Q=lwSMSt&A?66UU~ zh*EU7(p#Yykmn6ge2lEx_1nQBe#N)5(8|aWDZI_-EMay3)x@iJbDJjKQ2FQ?_17!8 zl)EPX{M^ysc#m{hN9HQap?*?{N?ekko>H{|ut78Iz8ZVOZd9^|C*$|r@R}3e63tf-UjK~b}>y?9PdP5`IWm_cSB`MD}OanH^=ah=^~kl zCYR)EYo(XJ2kat0p1hB~O?QW3X7w!ZJS=n1&183|WW@LGs}Jmkr-dr>XhND-q-pEu}e5@tzPS5VUCkXq~%v7&Kg~x zj|r*ny}h^3$m~Qm`p^1SuLWPbrZ=47S;T4eEV%h_YiNDjbylg=B-rFIFQzOfeXzSc zaOe7_z0?NU(O$_t*sGVxzg+Y z6%Bl1~-AfH;hu3B%7 zgT}3wM3W~T@Jn5L0_-K*#G%Z=QU)%k!UkHlwK=6MVYT^6(XG##_Nx3|9-|U0FqE{R zP!H=^RC@b1kBN!t`pugi1dx)C>Y{P2)g>Z=B@9r|um=i7{rQnVZOuqnfjrNKnn9vu zZRLw}_Bm~ujZ$J|qm7!m&>f93wQkmtk9MjXiqwP(4v9-%^9J85IA1=qLpB}7Z%PK( zjhkk04(2gORv9Ie1st?%C)|-2Pfci>jItPYq1i}ee2_ExjVbE~8PUC0G9k?GBQkTSX8hkO`#n40$ zPZnK+N-4ogulxYJbaO1K5ysOGE)#b3CrX!hK@2Lm}4rUM4 zxNb^N6cm3-ZK_O2H|ivhPK#}HGpzo+LJzBXoL>D-R9 z6%seZGrGTg!L7?rC?^m?e8O7h6{V&9;X*;+#OEl(+d}cJ#Y5q`5&sZgkL~C9=a8@v z;P1b(Yz!}`Tz>sp|Hrq!TxR`fDm-4BhGX_AVkM>pTEWePnV;&@Vq)Hy?ed;-%yGRm z3TC|-o#slqoq5WYIIW$z!$p+^+r-q)AJxecQ;a{BG31dPC6Wpi{4}DZ9PfH|tbAzX zSKR-33u8Jodn=P$VmGb?tU9>P<7d^RU618rqlH-3d@s1{`>{^|ObDMeRWmTiFnRr2 z{*2K45||gHC-l&YFRooO;v15^9n+Bk53T9@cfu|-l-pC)|CCr|D{dx}Q=yJkgCDU^ zIk+62t3HJ~>2euG^!S?DKCfEHdIsjF#7v#+%+jxXGjAtIO-mnmw0Cm*n%L$k*AYKF zoR9rsIC_meWZij{MpH{Bx7j9!LaDPCWzu5l&`wac`XHh+HH)$_WW#M$;F)3WHe~H!;@0J9gI+vYQDgeU@GJ9; zjt5k&aN+$=*A7h1qCJnDJA?6W~iH%2#hw5tGR=CKFK@g zHZs7%C}2?&?X77}I}F%eVC4JkbtQ(;cE&5El>V!DaQd4;;&ptcGG6U^J3%_z%h+Gz z=Cc`iVS{Ul*@>ilGngQ&8n5+WiEG2n(uQg*XB(B0dnPIW#>-}FkrUSuFAL9-q3Pc z#!yDi5=>_7t8C7>bVj!%^Jrr?zA83hg`Q$?U8@%=gj93++@y%nTWSY0%@hhQ5^aN4 z<;=mYL;ROfOVk$_;hw;c8FHniZLXVavgAFo$!{Mp-r>=_SkCTN39%g zY=Wt|-Q^1NE5DQBWMMP=)G$m6$TNBS5S*ke0|_wS5hFXl3; z?dKE8@}-dN*cF>RiTwdJP5QC`G3Tb+s!a`?`N$Oq1Mz6OHhrGg9 zEAyIRYg#N^*=!RXV4;!0*IXtdJb9eEpGh#ZoEKH4?@VXS9-ID}bvdmn6L;vefO%;3 z+Ik$>b(Ml-t-d?y6JZJcZ=Ak%(m)vwwdsP?3M>3fojgCH_Scn~N~XbejZwHeXTNN{ ztA(td&?7tP7#h)}Q#$jNrJsozC_;P;rGIaZ5V5L8e~LW!=5To8(as5aczVU!?%t%o zr^wiWOwikdqgvN$xqGZrOd`kGqFwL5_Bwt=FBLvj{;JaT{6Ohj>5H<}>K2b= zsjs>m5$*e@??u#^2xPXUg-@T|eyJZQ7y9Jq?W$y$Io1}&&nioAwYDX7IINayc$fn_ zj16X)l*Dp%Jm3mW>-bqyx1qx>anwT9`CpNOts42EtsrglO<<~Ax5M^l$9a_1-q7aV z-q{@ugYRY4{aHgBgB-g&YugStl_zKLvIPb2sMl6nrn9amvP!e#isJd&Wf`6*xeR}m z9>$BTVYbU@f2C3_N=Ewhc1*^75)+28J8E5pp#mHQzx{u3OU6(2Z{@3WZgRn+UB!PB zec?=%f9=Vd%NtDPgKI#--S*owpxWwc0=oi zvDTXm+BU!Xo&)@KPe2HaaLYiZ{%^ z3M`;};*41AQ%JF47T#~jzgX`+I8<&g+zaD%yl?v3in8U|*_~-#6d%SF?K-tRkY}fNmna*iP zhV(AIlI5XobnEU&8p}cA9FwMZ`V^e^^<%TFu%}&4kPi@&g=l8~O>X(yBmDy9+4deJ<>)YSd*qyu^MzrVo^Ta;=y-KIc8y(0eabGz1h zH@^>aZP$9Vka9KYI)yT^aoOr>*tbQ?XIz|o*-fFJ(kpdBT&m^15}QQ~Q(Z&>Iq)>aptmF>pvEuiM-_ z36jHo=+Pb4=ZV5WJ!JfQ0W-{{RfS4a`Q81|q6Xo#fo*hZN3uBdS*PRqbd*4bmgAaaCUj6Kzfb5W#lG>QJ>VI<;8=6?27pKA8=Ij``o2J6d9E_7too?3NEtwUPmePLXrF|$zk)zpyr{Fc2qckq1Vye@lY*zGaaaBP1 zeg$!R*w}7*%i=|k_a5U@GS>*Z} zAsv26pWf)WXLB`?%&x(eo1CF{Yg#+odmP_)-~n^r+iRF>NsoP*So?aFDI=UxnRxDn zvd~NSy4+K?-U=%cOZd$o1TE6y&x=fP=N>tB&JH`oZ+rkAcv2fEx1x>`S`zzvZ>>|% zGq^f^6g`WsA{DSBZA-$hqv|=P)2)O-$5;DFp%1^}c0OSqwqp;lKKrg-L_sV8m@T(> zY|NxxLse?f+za0~i+VI->&;0(!x^JTH5jtiMjEpe32!2)Izcmq_ zo&%Fnd3ORrZG#s!?~u2^Q=Ah&SK1~%kFL%CILaNSjvmFna-#{Cq+n9sRjtHfO=G<~ zisK3CS6BOn`)DX_g8#n0J{K?eKqYFtKjUXuLPr{$XEKojmMq++|eC?tjn5F2Z-z zDJo8Zd$Y$bOgW$XQ$CALaydvzRb_k1yXq&kEDd)&E?t8al~eqQCcVb@tFWE^jn$7M zCKMa(WTa`!0%=5(c9EYIvF{|B&7Ugls&Jk+-PrPFAHN;Rdu)>Nq+GfyG9lD4@V1h= z?mGq2n8^YUu4L$$GnUgi|I-c6kW2i19lT-%343-Xy^oC=1&Gur##2HuupDBg`dU}oM#?3pGi{tbfRp8mNc8PVG5tN2Cz~YVh6LelZyp>u zYg!0KvT&;~&YLOpiBE?P64~_$umIwMmq05W5ER63>;&(||9o%%ZmOP`DxAIEW*4SB z{K+PL?)c)&EOy#ZleUID-#z3UZ1&86bMQv3>63eXgKY+uAMik8BYs#R1dALF_=%je zn_Iq)i>;>$QJtj^90isCIP@Eoj%g0wL!orV|9;8@`U1fe1HRq;UY}E@0zABr%!UyF zkoFJIOh%DYSoEC@ZQbkLno`fdXM4kIKzxL`@@<2V%U!+N_u86^Pd2hYOx(9z#0_hE z@p{l%(?)P$<}ulp?<&_$dJFailsWs~ldFz#=l36tqT9O3|BJ1!4vRYf!nLswcLfOr zR7yZVB?T!_x`zP)VHTxRT3SV6B^4Nv?wTQ_WhfPql4d{}MQVnS7-Fb28b~ zrw{RS4f#jNaumx{L_ucsA^)$9EJ-7n3YKbC%+M-~cl+ROyP^Jcs z0u;sue0{WFs~h}v!rZbs`C7ev;_bhA_{5U^s>&4w>@f(RkJq@xLYU4Yv6aOt!j9w( zC$Jo)0a>Xs^}#!?U7IP|T*H|4AKivNL#zi5WIf-HLd0HqhHJR8pc)Ik&cxkA95W;O zyL!ZSNFySt%(=xs`+lwg48g3jk67=)OZs-En8#aJX);-)`U;baM!rIKLx`r!!XGyW zHGCI(v|On~u#{yhBOm4-J7xC~*6ZmRnf*ghk_;Yli;s%5@z$U4=Fc>*%+u6Xf&e3b zT~R_rEq{@XVh2-U9KZd1r?9D@F0o_P%B^6Y@qCVhdQ5g>2}|CY;1C%ZblfENu`lm@ zVC<7ffOGZ^+K)T?&+xvNH;8Q)VeQ$4c&?|{q`1+fIR3qK z-Sv{ch?lo_s?h^39ap$N_~Z&FO9&|T+C^(b@bYACsK<7aH&g6&;0}%XUOgZa_@C1V z%g$e$E^ehN`D^ep1t%qY#L-Nboj9y95DxgPu$L~cTgTlz3C+yW+DdKIY$J`;REc7F zSaTOm{>4#@L{=Dti`h?p@bVZh4V;({WTF_c$I`S1#&Y+U>8Z1^S%TrR2x|v`C=-h> zk7@A-s=>BcJ*fS5e)IaFgHYed#Mr4I{T>Ie_6-VonUqQm8K3X!eHf8nnVjV-L_CyH zUtu8G-aAA?C;FsFn{zl7V4V-FCl5UC-Ov0!-omD~EKwQY zEPj;;@1_Z%jL?8)XyR8hmhe&7FE;p7f!!;{O3$}WQm64t2q;Kb&U%y7Lv8RE%C%pjLmEN3ql9o|5(PcXEl0-CRb1|Gfy9N-weY^>t5BuJty& zcgi@b@(HTgj`C6+`e*-5$j0J(*lx-H;fBQ!CDqIv`2#c0wDsQe5i~7+5uohro@ecr zuZa&6n8>>p!R~R0vlCM>JNWkWf=r%}6*^ukm}URTV421D&_?dNT!5$us-z5|{hAS)7u&K`CQc z(9BYC8wvvz7e$OZ1d+YxpJ%6m<0)#$z@+?2uE`DRiVV~)#Y$aCH+tTZvYAQUtwz7* zu==5Pobll9o6Qtsv6*enoOAXO{*;F#(Ej(3_^luV3Xre6z!zb{5Roo<4^4>q)4M&Q z&b$-srL5U}#P(939`8+?zceeFOuG`ln~>|Y<9d5P-1!>v-jKAd$gXt{r_&QoLQ;E0>yp=lQW5WdLca?!;yyZgN2(ZiF&R1;7V@{ zvY~}0iOP(AY-@4{58#CMy^f1Ac9oWZ&KR7q=e!O{P#z-`fIk>MM4W|#t&I;e!}M2Z z`?}i|`*^ClwJuV36c4IlvfTrFXb5kUQN~i*=g1aiAA*|}@+F<5duDZqXh_uhY>x~! zn-3gtG1&K{(=N&wBI)PPyvq&K6YLGsMNofz9dnkJ0cpFXYKEOG`JTkViCk9mUs)@~ zi|ZKEYbPZz7SjsnNssaZHJ689~2s)amD^@k87ql1m~0c$51>W zefqeOa8dclgBpbPc&E2s=xm1X%wp^O1E);V8E(NCK7g-s@x43b?o|JQwDMBMt*zYnOE*PCMrcaS=fRrBrtVwQ`2EP^sk+s}UDY}M+}ABueZurBZ}IGS z)}bp3+GT*~%i1w3^*gbfHXFGqmQ5eIHx<_DW}_jw)oJci;q^#y^o<*~#;@tG;Tu-3 zQz26)fq7~UO~O9jIYl{eGQ9g{VJ!FjaR#T&t)TZ*yH^21nSqSmR7eNw*?k-7u&Z4& z;KUIxjc}CX3@iy{b&O%3Nxp@wjVU%@&ELIkdJSN|3;gq06h!&gyT4zPVcjNcuevww zm`}A(0*)8^6}pf5Sjt$W%tA_> z#kaBxl=a^3Z2e%`_UhL@8ZuUy#Xi%#>_y0mPAFh$O1T`7bsKf>6iT-#qR+5~6|;FZ zL%r%H#Yk8l+uxlwWk~&2W2sDA9i4+(gul}<)hoaI!Zfsb@gx*?tPtoD51p#x*I`$sQX4pL_hi6s(wDPd{ju-XJ?q>|q0g_Ph& zSe^Ba!q+o7bvcl1@;L&=Jsu%T-bt4YavF$hwpp{&`7zaw^`3jceb%!)O4YocI_BHn zar^H#i-!yn`>-u#HudwLlb5ECSJnvnK6UCI=TORzPoWq!@YM^^oMpJ(3;fM^4Soq! zQ|BKM5G`ZAug3&Q`TbyDlk+hGzYxX7y-7QI`oMvU|6yOusSpdMIrs6@-d<148nJT3 z*=-%Dmluz1j*v(3`PsefRS4Ct!|~;@Ji4=MuMGXgI9AN#Pr*x37cI;9C>HVY(PveV$xyD>g7=& ziLctKEy^-)Ef5vv12Oi#ENW|VYUrlV;Vr;2IJbWK)Iag}8IzkalwEMmPQT+ASkrUg zEAi)+F4AFL=LDO%@SxOZ^Ed3bes!uDWOEN>D|*BZ^fcvb2;WzU!ZI3U886r&QvNtT zqrzGIGe8|YKBmEgVmdxD?&n}n!w_Yt(G}lJ8Piu6d%v5E{*lsD9R|-yV)DO}SSEr=(JQ57cZA^+ydR~1W z7mT`@vgmtNG7@$;{FvdvobC92tXDD0|I&^M;!^iEW?Fi%ZR?#Os|_|Epr5rc}# zoY~tGM~4|*Z(2qBqE1YyNmSDVET0v+zXT@b#mRTt`G;~b;^C$Y%6Y?iBN;7G=S?E; zPP2`-gS#`nXz4ZK$Ez}pv?5&I=gOB$32Iz86LpLvA` zR*A^Xf5H*EfxA*kecalDApq9sUaf~$#g@Y7Yf4VHY(33W5wBGIQgE1`y6p`Bd1Q{> z%UYC&0{VmfQ0VahaF2*FN8@x1n5(c>z(KKR8ASat`9pkV#2%_Ih|e3*O}5TQN^O3V zSGxP&5=avg{>_^lw^sUcsh50K!o2O`5na<*IX4v48N>EbrKbpUh5$;3fLqjIMh5)ecF^edN@p1FDg9Z8Fqglc>qsU8tT^0YFX##@}mzdwuj{bB8XO$sP zHWTf@AGz>hGu0{2j6C|T-E{Cm)h1BKvgZ4?rR$SKt8=aqi8MUpMVrV{yH{JT=%Tw{ zd0=NQc2gbrsQ>r=`OHd}^2<_wTXPwKeC5f058F15FuxOZO*d|fB+Mr@wMr%hi$)Vr zD(*t)7}M$AMrvHMqz2FOvC?qH>&CbGc4f=HN4|MI~=GE+MynEudDt0+2t~mmkGkVV#lZ!Kg z%e;Erb#tIS%4N}2ZA}yQz3tWhujWE(8x>5Qw|}$Nao%WrfIqHWBzwd*T=QpR3mx7< zC-_GvM)D<&)Zeb7gY--P(qU@XBl&its22}vwiBljZK2_fF%izkMu6^EBKn_Y>G*n> zgWR`;C%mn_-z4l++HHc|4LQQxn((0CAN}M9zDWLS+4qLpxAuMQ#weZw=}SsMS_$)l zKDk<7-J1L!$Q2_Q1s9M5zZRNJe<C@ zL#cGx47OV1Xpe#mEg=l?lY%z2EU@)7O zjREPbd1t;+BQR&L4M*`7u^8$I#|LteO5Eohv?g<@Ov`Sy%?86tpZV{GUHV_}C4^=U zVazruyF;z$ban2h!__N-YcG6qn|OQO6BDwwg2a`&WSBb+$EFD(kChfTSX`Y7kmklM zWdE)OcA}WgzMGVZFl7jkGOUqz7G>5>;K1b!J-k62I7KFW{$>I;OC1+HQ!3y-omIex z9-N9sN)0F1c9bi>XrU0jeXLO&S8)BlKuP)^x{U9^6SsG(u5{<$BM?WQIope@$4@t*SQzDqxc(`CQ;b)+lBpb!b zw*6-gr#=Lv`owN_q0I7atBBnBH8PC&!KmxM9}eFxt9@{0P-*jOetzFvyHCDZ)?N&WMs>iaf!D*I_)-cl)Rj&;i&l8cCC-y33rDyNmV zH<~|DNYQ**k7nRhsjuHFdmr<}Gw41ZB%P;>vydW{|G>MsH_c?S031W%n=z^uKV5<1 zryug~eHgo_=}4&~={8EzejRlyqB7}?J9sV&>3^@oQ|G^d3e!cmfHCh2dX1Cvm$z*E z%^USP!ne!Ud*`X|+9(})YO6H}k>{IQ!IS~_RrP$_P5#Yi+5#17oW&fr3vlwqS^xXC zyLKa0YoWers>R#?8$n$lf4K6S1 z4X_)pyP6r1f^`>4;22Jw-(NTWFUQv>3a}B}H+BCFdW7+FidBj)7v)^~f4#yb>se#T zyalerF#1WG-7;ZMRp+R*pUIv!I?r)_#I>LcH*e)Gz-!|E_l=(sZujd|j1h7^2_xaL zn7Nn2vZTdVO5~CB6UyPvrI^~hz-E`0?j%doncKrz^+nugbyy`#i*Tql>7sjF0dw%O zGJX4Xq>d&=-vXZAc0~Qxb{%A2^6)+M(p`!4^*_|?*b)_2O%_e@fpzu{LGU zE^k&7KP#vhAgSGF++q00Symyc&bnn9 zpMZ^#-QSo5pqlIU6K-Kf%I*f3C`Q0BJ}rlF%Ct1>60866b*A#%F&?g4nMfa9u&Cn} zqxj#w1amLvlvrrs!rO4h|dm)OIWIirTca&=AT#eoc0gGtW?~pmYtO z`zMdLNB~steD2@4MTSe${c&&*{jT9o0MpUaa%RHz9QP!^8AOQ@Y@A3J`6HXspC|#o zUW#dl@{V&sRjxhn4rUQ^5+nFfuxFvS_9~`t!(z-1Y)wVLzqTg%-1R{=y2pJOpx5Pp zR>pg6olK|cYHYEmnB~@Oq%Lx%RlzU!M@CX$&%Jm*v{QkX=6MSpu}p~K_p#*;Mlpnw zPOQ*f58kPqwu>puR`kSR@

=75zW#h&G!zfl}g2R>EyREg)6$$ybQ(XmL-eof!WH zOSQjHZQ=o=ypi?2D}`X%4SC&Xfj4QEXCag>`79i%h4fyIps2Kf4rf1;(EYMjct zXpduf+Zpm4-K9CbR3CmHfn6NCy1$FXFM#^%)O`qPhf4j{;&(N;E+c~A2{~{ufcEUg zYF>ZJD@PQaOCt@xI*g_r#?5KD9`!2OVs+=ud}#5w;$o4yuT1m<$6!@X#P?A+bTa!# zS&a9C%9(69%1+VD5$x!^^@5X5nS}a!L*&%z(Ng9?0Yu+}4nvXPjr47%fE52_RBy`e zPQk-NiOD#lRb$*(el13!VCZ)>xKJ6v{dX7F{5Yjmj|+!MLeD~E#&38Ar04O{PuHn& z;%{}w4GiR+HixqfaSHkzfUb-w6QIr+{1Mu{^+E|#UN3|WosPi}8&=TW zZcD5a>ja+CT~eSx7!Le z13}47#lty37qqc0UtXz?$72eX8~5S}Pa9i%d90ftAiAm50=qgj%>Ww61@#Ro_06PI z)}%q_$N5Bv`^230S99Q5ZH+-j%cxfCNa|4?q6woD71!N$HNA6 zmw;v@zCO>!W4!L9I7}t35g0g>+u#s;1#bAaZup@VEdRaw@h8KW;acwOZ|^Ik+&cT{ z`h6@rfEsG$NN(-pe_e08V=a2wdd;q5B-;0y`;WxrrJEHidaWHjEww3BS*L&0Tm=p5 zl>hxcDDf{&sUi9?YaB{1JAFuQMh?Twc$ML}6T#}|<-q_iNqcHP2};M z$l=Yl?kL9l$9}%51kh54MI22T#sR1Zou8viC&Kk!ywD={7`e|#1Tox{W8GPI`V6o5 zSl1-*A99Hd1A}i8kBCox5Gkqo_ohzK3U!8Bb=u5)Cfe&HsX1b9Y5jS4UmTQA1p{5B zt#%WayeF!c1d$FXCY|}WNCC^|lhfYYPbN2T4utD5twZz>A-D1G+x@I2qbd#Xc+pw6 zYjbiT-+>Cu>*7AeZHsz-z5%zZ(=o>@k>w%8f=~KNTywRy_|1V`vK-VMj-nTIVp;HJ zn8%qmt7_+5X2xcJS3zbd^KQSe5TE+LfQ~WX51lF1`TT*uX(eyC=Pd>K`=`%-vGj5O zDZ_D3VfwCjDIMi|c~Hx;niVL{>iip?YL9x@gk@QM-HXWS^)Dw9U|HLa>>qc8=6^w* zzGg+gwH~mA2Iz-TkfZA*4ASdJm!UHAE0W7?-=0Iw8C96QS*>1j_P_=`Z>Hgq4JX{3 z(+tQw-P|=Us-~&>Lh?2xYj_+GPxq_`gzHK`6>ef`3Y(sOmztVd85Cc%wIe{aN!WA6 zl0xxa5du>SpI=ZOww>?-a@6f~{wrw(fGA>*Zd#L8E#mwMv%10g5ZIjZ%t$QRmD}A$ z(Fynpr_cLP_nnHpT5jw47ukc{xw8C00D4kfPBQ44Nqa4ld@@0>^<0@~ zTq_gE_~lox8VoXm`Bv-8p`PRAKvC(FaA_8%7Gdp1Pncb z{XE6}=`zWS47DX<(e_DVnlq01=T=d1%fz_Gma6 zYhBK)?FUocNrBQ^vM>o5k_09}^N0pJkbKsE1bR*- zxUc*OWVMDVskp7B7E&Wycs7hynIkdnt$*?TjyAn|hWCJxLo9=ru4Y?w8j8u4THl z@*&gu%`wFYPU`7QE{DYA?}`mGH4ysARNBDrH!s!)20w0X#LKmXuRMOK&HUFFxfQ)oczg z@Laj_^H-?G$A^WMFd~D}r3We=X`mx|y%j76d|Z%p7o-t#pZGnRPyNE?S>go~gPteL zC|g&lUo!jIdNYP^KuV1b(ZpjE*v$+5Q&9qQh$XpF>r{luj$wu3s)w&n-U zNkM~wW!cdABz`Qf@#?xzKLEW)H4C|K>KnMPAxzAwjUV6;6Qr8tB`NUbZR-bvLTzu| z#tc=-cdA=+GpE5lwF7(=S0z<|N9OSc*m=t&);VY&ErQF?P$&F?W{LQTkmXbz)m8ptudKuahWJE2+eJ0w(p#$?Bvt7 zFRYfQUA~z|+P1!s_NYf<*2uAUSZbv&Ib6E5`{ryTk*1OLcq38t*V-msHecVgQ1OHL zqOaqtJWEije%m*N=UR-;=}qilxN`LL#@0c`5pn8Ky<8c@ak|?(M}2OFT`Y{|F?@iV zJ;69&V23YGa)^w*WnaK?tLaATF>Sh1dnohUlfS$FssUW>y`gK%haIMPbkDy+v zD+^xU9ke~s6&spBNi-Nu5YtrZ{G+v(%9~}vZ{3~Fn_OagWUcx#!FyYy^*+A2U&qaH z)Mb0QS4la<$+wgjjtPI7+~J((&kyNeJ8uTVRnUIFKT(03B+ohcVH+D2mAY7W*A=SC z1gyb_$0pyuf?^XczhaUplvzR-=7mRf&*)q_3o{YoRv zafuR7eka$E@*JAx+=xNE`5g!i$&8W`Axa;<=+I;Tw|s_~*nFHa2wUsjy}OKx-@Do! z22kuFpg9HyR=z#`_QL#d4$yAq8Vf8DsFCxBE~uQUe0_}GH}ze>PjliSdCQn4Q|+iM z)q5u;G4^(3Gbh6bC*R5Wz4#%RUU1j2b^e$5AJ8HD(@0I}#vg=)^IUQL*MjTWYx~kz zlsaQYsx*Sfdss(Oss*+cNmS*>^u!*mq-v&Qw2IgP8AP8Hb?XO*WDj9ehk{?B5}enz^t0oYe*Qa zt#w-%$pg7$s-xO8(p&H0aN=S3MD^-`jg$BVKmQz%Y9KH8?P6gD1Oc>!JPA2jF1y=n zfp#@}_c@=q%tvKb{?66WUS2T)-$fpu3bW?ED~=wkTOf(B6zUmY(%3NZ;APsK*Z)iF z8S(S$)zUIpJMKs&anJe0{`uQZL&mskg*iobIeG$mxw(^HpEg>HFrNtSGA({*cFV~& zx!(3|Yu9>K1+(gKWU6a>hMbvX*W|P_YHrOkVdjX!CiJAr(XUCa)+#_amp&;cKuKk` z=v}6_70lMFKJBtFvbi|?V4z!Pi))S-31+F7zhdMpn{>Vba-?ZxFo;26VY=|Aw^_nx zSHF&0iQ^6v)~^$;V(~oAr=nj3#I>4tLr7NQwPsHVJkeB#4pwiNkH5sY4>>9z7+iIf zm4otYcRVE*znt?V;q^}Z!^7k%h0b;gOC>^QYVqZeMNhAhw06rgorh5f{a2l8vaN-$ z8=xJ4J*!r}E#T&@3(95%yl?aJ?-tr%Tpl}5ho23o=bElslw&%~Qmn75^62~c-~?hD z_a|6}7(U(paqS7!#@B2qH=Y8utL10VsgwTm7rsyG*}jDc8jk00mtN)hBDrR-87@qJ z>6Ppu)9{Xgcn3nXgf=^~SKqfM(TThBrqr$W8#~Zj_B1~eF5D-7DjN`B^Cae#y9~Sb zDMe7kDd2f(u|nM0cCbWX*Fmgf)GfwPPR<|nlX#T~o#Wr#$RI3?{qSM2yb83`hGvUF z*T46i#_bf)Rl?$M1~6~l!uRvuaZ$ILt9e*mWa`!3u#h{D2fqTEl5Kg2Z!Tg7xv0g< zY^EFe!3nvrAHNwmd;Dhk`{gMp4PJg0%?|yA+CQo+=zin;d1tj%q^15|Lh^92sZ*Tn zP@GrdkdjhSLsrZA$cx&ej2joMb&5BfH;5QoMP&pK>P_7~=-{^OnAh7EXf3m;trr?u zFb&^|g)9OEvb|jvgd~>}l$7m%{Ie7oSc(e{Ra(L4`MO}3Up`Egy0fvd;R(kcdaP3% ze3#>7?dbUo*p=Y^>@p@GJUl8u1}uAbZFeUL^u$cA#TVGTGW5LlX2>IYUZWmSNz*`$s+N=|dQ04)fBeRQ$Gbh> zSnbV$+EUXua+ICRedCy7qH_vpNTCu5KfBG=Z{O_qucI?mjj+98^k^*{iU4&B(wGP{ z2ktTZ(+NQwF>`NfN&)Gk2 zHIaK?R&Z7IWpl`X6mUSks-oMF{SjG!W54=4f*s<|4jCSCEpx2V!KAF5Z?>hP^G@P_ zN(r-+M{}xtsc`&Lv310CV0QtPv-5lPHSTGadSV1F`KE<|!yKby()_jE@<-H2>ES2dc zpjeGx{`=J{tHSx|PCo?!&7b z^GduOaXazpuUn!uqQj(V;XL~bf|G`^d5)fL>c*NL)~YXd7V}OCv-3V%m~mA9xxANe zxtJFg7V`DFT3&MP`QSKCQDZt!v=fPA@%Z&dP-F+h7~|Y-dfaBogCRH4yBqD4yl#49 zDya{`nk>+I6cmx7?B&>y)`Q_>KmH*WT6vOUP;cuaaw8d?HUx~34IW66u0=Jmr2IX5 z88xTA6RYjfLR0SP4VAl_qJE|iR>nHI<=T)6;ZrtraR{18ny%;79WZEZi23IEXgge>GOWINkpUVODl7EdKA>6!}uGK8$S>G0(4Y@KXB-I>IR~$iC`B z(#{ky&8*zuv$Rt3x&0jxo?}v^2p?rc8+UdRVO}VSV!-mot@5@C@G}H+iD*7b^{~a)@ZJ zsWPT2Rw996JmI(U!CQC&S~>XyUYjsnUQ@aCXC){$xDGLEgdT)~06A(tRlINAx*1t) zQHpWT(*aH?M&iT0lj|BgV|sUU7X9?%>T?4Nsx?PzvQZzh04VCnfsI1y>SltJlNF3& zjg@fEhry}opbbr&lHRQV&`8L6QnmHn4LQmsi*F0t4b2M)Ijt1BCIjjX+#DSDD9X3& zfHlFaTr1E#yWDO!j6-UVGwZ`(h0CNpuMB}tSjZ_E7q?3L{YCznR z7|}h`12UYkZWL?5Z+$vOk0D!%7!7VboBHv3fBWt4X}}s?4VnC2b|x3 z?*Xi6B%3F29`p6o{PjB3E9a|Svj0qt2ZBFlY@F*CR>Cc$vi=Wl1*f%4BMnN1{Rm-N z^j-WnpExTt;Pdwv#2WGH6x(EA+zTSIe#sD%qI+T`&0ULH8d*{_?2~dz#QM0~fhm;C zE65mN;`7}Ig z01Usg6A+sz0$f@^kr8FnU(Uo^O+kCQb>nFNldX|?G-jk!y9Ve=(hG|7`A{&V8@5SL z^$hQ^@$=F2Z;p;%skry%k;~LVQgq!{AFYpiiyIW725KCpnDe!G;^;yj$lb-cY1qNA zQtfevC{S*&Y>wilyt;X7Y{r)_h|NPjpPnBj4=?y;L7=-Cd)?%DL}Vm~5=*#(%G+z5 z1->L+J|mn?#7djQ!gMolfvx8%C?FB~3X&jLCY2WWsA))Tq-!G(l&o&}$#|$ik?AX( zD=`!FGp_=|w6b#}?9%Hy7WG`3{{t8w4ZFmrt;u*`m0i~T8-6)t+hL09*|~iP2}Sbt z?qjDVXzCV_!_Qjb?Ju{`)~VOV?ZS3cFo5V?%$w* zaB`|P8yfUDl*+!-W@uE)>FPa@Xf0Yk0g_bE5RI?Y%Qo-h!Ot6EpJvs#(!}LCnbGLz zEW8Q54Ej&dR^N2}^n_lI74gSSyB2}Ivbu?ZL{vVsgHDjBe!W%@wMxmfOuI5+MaWlo z^TB+I&0BP8$)B>XybBsm@iFvN;y3o!1kZgaXIxlFXc{CQ`vO^gvq39;rfkq&BEuJh z#gbs!HW9`PQEq{bre<<;voef;q2xrcLaFw(+ioBdoK-JHiT>b^@1Htm<= z2!{1-0~D&`=C+zs^RW1MdWZhv30lg@q=284gJ2N`cFhZKKX)y=V;B(-|F{oCEGR8x zfczl_dL+z3L#!HN61TTVcxaiQY~0r1g>p~Y>DqE+d#vr53a0>W94M}P<;R;~e#e&1 zJ+3I5h_=~$Q(Ha}4XkhHuAM9Y4}I0`#ctXt?tWi*ljMZ^Q5?^Ii!gI2$C>UKtvIjL zvqNxH4*t7spkG`mrLmCL&#MHgS2H8f%{nq+Sy=sJjc3di`bMn_xD!i3HW1T4{AA0R z1eR91MD6CGJkQ?Q5N+6~J(Y(3x{}e@oVk0Y18%0}-LO?W`SuggeD?7Sce00f4dq$> z`OO2J=PcFL)Ivc~ISVrQM)Tr5>jQ(ChdR6W((kbs=WQjw)5=oP=tPZMyzPz!svPcU?01R)>{vIK&oNJ#TqU+4?lk|pqu$J$s40mg#O~J-U5_B zbPGNaA;tUHlvSL!jN$PAbr3kKD9x6@(B(cWw#v5(x>zo)N2~FVp9G$i@>1U`?qi^j zf(MvCXX5Rkyi zAJv#yY;Lt2ItbB``noOj;@Rv<=>fyg(AL>ky@(>s@gU6EIr_rmoBO8)V+3uQFl^Bi zhNf~TK?uZZW-{ZV(C7>pGfvr>o(11EiMioQ zJ&MitT}fjtohsQYpILU6Z8=F6|K+U7AdGX5r4;K3G|3}&_J$6{(a5ny=P)`X#UyhURyI@IMQjYM>X*;dAJk}4ii7XBg=P&fr zijh>p^>#y^P?|X>-h-IfOsH2T5fqwcMqAu4y>oV$V5fnV*3LZ~=y z-D!-Fvibj1UE+R!H+%pxp9Wmv?V~y(xn4!~ao1(2smil0KiM++q7X)RJsmQPo#H@0 zL_t+F7fLjsyMSoTM~1q~%o%aj4R&BGd~VtVc9rVu52RnSLuOh}*pj|GzHuY9C>|HL z77&#*Ud$Il_lU)X|^P^3;z1y4EuGPAZvh-Fq(@FhW)y(`IC3ch2X)r+M+c7%&|1nZ4%sd61sL6a} z!sh+49%(jNcU6{a?Vr?m41Pz1d@M}nfm$duvJP0)T}2szI#+@H=_I#2?YSK_?T_wD zR2;&MnRd+`nQRAp&X_gkMK16FFpQ34>2v8B>HN(tEK(%r_U?a4gRCAtUttv;6?OII z&6`(4BAEPnKoSav6DNTZeZK{W|3VO`aA_~WpMG4AB#{TXi2k3mS-1-+tPH0G!-Ajn z1&w0!zP}pnx1(+@o{z~(uuups$J_@=3+7wC4jCW~7j3X1ktghAS&XxaETSxRhRz0l z$wy7MSg9H@9(yQjhcOhUL)^p+Aq0N*5k7 zy#=xKp1n@y{orXTP*Ls62ii!#-W`X|#J>+w^vEJvxR+1EcE+D!b59)}bc`|eO`MdHv8BfS6%uQrA@bVWqkD#Gw@>DQw_;3!m z{$T*dTHmTzm(61-Y#ex5OE|2cji4AoxRSHDAzva(On(918H+9G&UbCm`Ct~awb{p$=^c3SS-@8h;uq=D zzgyBJmF>iR9fEN=%5Q>7C?~Q&C4AOR+;i4gS@ne~=e!3j^T$^)r_<1)5dqKoZS5C> zd(yhaI_V3cjt7>0VJ+#+MndbuhfIE(xfl$o_S3O=>ab%?00S`JK9{`w*B%~0@xR@k z(5K-Qgz4epn}0C}T0y#C_O4>4Jd|hvsajQM+?yN2TGu2EaY4Fpz{}qK@kaIXWYY@ra>fK|TBzGS_a=QAxB^J~mhylUQ zt|kjB=LhA@sdc`7^D@);X5GD3LtQjejx$IrMQ9Ki?=^Sp=y6p_;s26M$C94NRzNR< zjiGPF>b&%&g&Nl&XcOD{>Pymn)`iE7isnVD8TP2fp)uDeJl>vCZQx>ws~ ztt|N~QguQY!-$%M^I3`#dMaQ)l6`)oxhM}j(7L`rkTj^;zOQI{Glq7&B*%MB0k8=@ zLb@TOhw3&VvB$!HmRAqkf+EcbPtRfUd*DUtKSaLi(CD3Aj=K3*NZSQ5)pKcA)NK;s z>psr>%Xk~(l^S`kkz_l>sgxwb->IXrH;QL?+Er1G>0LQ8a#j zG#zwuf(WiV+~E_$mA9usQtW;DimvxOMc0$U5-|-|k(^irKm2sHYyKHXO0ZX?H6N)V-ZD|54EuQ^*@aWcSq!G3toeDVDVrpwfc)f}b~! z2dbg5Q@^{J1O~Dl5Sth+cgvsDH1WOf;P=WXSoX>AoIO50IJ#5jHb0ajI`V`q-4`lt z;Eik-H(0mEs0ND6T4Lg_!2b=3Yo&`augnETc>S?N1 zN*Z#@@A8yU&ux-gB()>W<6s20e?!vhhyC{=0IvB4Ur6H$G02I~}&>XPTGDA6nHqdVymJ_|=*YVEN3`H?Drrm@DEHU9kj_T1hn3+u|4NQeB6?!w@b zYaV3zZ`!9o;lsQ*2Ztl8s_v2I9-o1#L>g-=L;lq`o$Be9Oedh=rU%!gfSQZCR|vGf zlWTW3BWtESM6ZSfeXG9TueYNJT8JPx0J2peiN>dLx8%XCy5p(vlRG#{Y%bAVxIn>6 z7y~Xxa<4?q8drOkuIGZLXKwColVor-a5Hm&wW$>GAksna6W4B3rI<$3qgsl>rh4Uq z+=Q^_uXev4J8B22ms=IU)69Ca4T`il*g5a0O3#KZttZBXfE7ncN+ z{PMPZYXOdMJ7&J87%oOH^u&kU9%N+wyUbfW5ZFNjQG;mljk5PIv?$?9HyaJ{Ew22z zyF7Kw`9I|;Cvz`cd(w{S&r`Ad+Km5Sd0Z?o4^Q}=zvXe>% zojv1Hdxv1VR<|tZL=&64!Alq-kF)u8kdO|?UTTHOMq@_Zcfzj&hIn8q#Em+gQa=L} zkWx&CAsjX&XA1HH1@GY=z=Ppq_k-rh%Ix#j0k;pJXsq`jgG<^t?YT!gCgra+M#f^$ z2$kNMG4V;;A|+pM7I_?W%y+E-q?=$xvS8sUOgWe*p<-Wd2*U-sA))839UXn>RXbW> zk?0{0vt3^I5%DlORmi$$2F1SD#wq5`5Rd}GqkYg~r)G905{0;MK4<=;Io&redFp!} zCd|b^s!rP3QZPa;S!*(w%}DF}JM3OCp}4KKTOb_}5-EBr3AUSZ|6#I66qJLZPzV<$ z7s_nBQc}hdkfr3qGzSWv4dS*SssLji@L+jp>FN6tyCrSDeEFgnfx%$JpK!_y>geh! zM?8G=NCN(4>eIg|`fx>&J+w|jOP%$PUO6>0k-9X)cpF+Gzco+s-MhR*0;<{8Iezt> zv7d~e=dbWsr{DF-*Oe=xE!PuvFNF}kw4q%Ozxdbv;xKNzJYxL(>L}lh4Ew4@c?wEI z3DB_xH3PPTY5M(U(MFk2DNZWhd&vW1Jyo?L0%9R2pSbw@WIP41bRH$MI%urXJ-xVRKVN4<+{ z)_CqG7J6&D8->5>Dvd&3~H=~+tj$WD%DptW7m1?n&*NP7n}#i;nZ;L?9hh} zZhC7nOw0(&vMu;N|L0(poY}U10#jBfpC3*=7%m zR=qDGacbP$+{}`DMni_HoU=f2*8o>N^Qf{wU5|X8w|mB(M-YUb+-BB(sm`;!y@B$W zXJsmzXiC10;ql7l99q-ZTOJt;txNwHffFK{p$OiBKdd6S7b`0wIG- zdgB%3RzfhOgOc+Oz{sU;t&@D*e0+F{!`z&&)ELIOc5_5ONCk|Z!rUVRdfBf|sbj)V zIDO5$*dF&Gsm6zdR%Y>no~_8-H#d}%e4}RaH zmuApu4ygGJMlKcj4^XoP?rV19lb$(i!@5=h&3s1{20&4%i{|^|>jET66F(2~_(r zuaVjCkwXFB%)3tv_RJb{fXW2Gx+aXy{qfOe8`JKxCOwJk%cN3N`&&Hpj4+5vxWb(+ z;ra(ggm9Tjk$GZyV`aW_!FqMJpRzC3*4FCZM@(<#z+h|KWUP$~pd?tYwfQ-r?qaI{ zH-9$LyDy3)v24#pl5pNnXsDZ%t)p+~wfi8S(Xd2pe7J6HG5gqv6c=q;eoo%i)yQP{ zz+C7*kggFj@S?R{sevm{9*8gwKo=M{Fd2tJSf!(-%?7qIm|GZ31(M3zFohC|T50R+ z>tXMBD7!Rv4M-6)3rV?sDMd`%%gbswE6Y^R3U_XkUvxKP+P}@LT#)jSWDglQFRZF3 zxqfBuzCHev*|!DLZ`=Fcq)y28`}?p&ivKu}eD1N=w&zwiwfOzsCj;`PKChN0w^U9r z@|X84AXp3y?EfP*8@zdbHfX}Z&d;#y8Usr z$4{p9iXXF@EHKvbcG3C%mEv-7xHHm4dm+`+NSLZnaIHVT)T2Hk{awHvYTm8;+uBzq zoW8Rwt**OJ&~qq!y$|vDYsL?`GSF_Om7HT+YL{%jwdIlVrT)d<*&ciFZ=0wX7A)p& z?oft+JDp<0!wXBoCBM&+)WURBUl>Keh!35Co`sg5r`z*0>QFVRbTKT+^GaVVy zKI+2}KkIH%f{4fb|91Ap<0(?iNe)s#r^8DBAFjSUkjiy!-)^!~*d;@R6lI9Y92rUo zndf;cGAlF7*g$1Sgi7XlnO5e^JIa{ZvO-8@Tp_W@GX3sn+2{Mt_xtOd(^;PPeV_Mv z@8PklbhDOZ-MoaZ{b-$jcUT_G>YG~g1d7N}Z z?QjPREcHL#N_Pj57Imj_jbY3&5W#zJNKQZM;h#K8mv~gVzAvcOtoVUkUZb!@Z~G;w zpM6o*vO*E$zR#M>+h0;xpT!-GzldId`;mK;yzxtco~72$YY0wLT{ z*S5ch1aFggEi)om9tnSCjhYO1cQ8|S^QXcV)M9%9SciEsTsXqWO;ZSDVYpi5w;$;FsOHn&=24-mz{-xO$wCNiL@Hn;<6g(T3#gF8b6^-aoLGiUZQ05j%1n$I z*#`2RLcp(rj0!;H$40%&ptFOm;@ON5%j6w?)4X~Ui5ZTq)K^NuK zc^L7Y`O!(7ST&v8UsySo2beG=3Jf?+i-DJdX38=Cdtqf&r~x1r&48s3iv zU+N!MA~4ut9fQefhI*&&kJN*zEUvyjO}WeqhYn`Qw$|$7#xE)eJrX?kjq=A;UWYPa zCY5U=83B5(E$nCKTL3GYUJh|IK|cug`kN(m%(z_H(n&hgVp1ShzZf!|0RbL^;v-(5 zcT@0tLZp#XY5)DE+)5pnfuMzC?5FF>MiiUfBQ?(ieKW2GB+7o<$<+s^)$!wkWOwW7;dpr^@*$3|JG|E?IT_l}QA2lGbx!Ui2H5kgT$(4yQ@ zX+-DEK^fgp@JvkeZZGU�B_=WX+@v=o=`lc737u`HNcPV33*>Am$Be5{q@fuDsbO&IG>= zu9)qScj!%5EE$<+mBD!A595rAuR`w;ux>sdPd<*;`*P!r$w$`Y=coAmL4z?4xAAu9 z!rbCh$CYN7sB>OMwgA=5sO(w5>;XUoS-OmBDc{56nPyFS{>Daxz7V=O#sNACP#p27Nv>_E3YKyK6(_)lYVV9Lg zD7!ptXyF={oKR^IKr=N0dRmv&v}#hbB^vM@yLWqr){DwcIrTJXm$bYksl z=;$+t(;>+7s;SG!LH9{z_YUTv$ZWTnkYrb$9Cci?w=K6eSueIO=C}GC@#ZOBhlSKo zt*SXDWz0bBafs-H{zfCID&uUA35>whGB$oCEFvO}*)=e?j~U{Li=2f9QF}T0uwE(L zZIf@{aK4Zc=P9@qhug`7t+EPG530Ai4|A#x{=1rH&BC(;8NUkP_^?B=XOImOZOea0 z06vhL+XIYbfc``S$SWc;>geL~Z2ALLq{=7*tl!up)m&-tzh$PtAiQD^ntD)rZ4@^E zA5k`4#kns;yCVuT1r`&+J?C>}t=Ld{EAInoG0=u{U`Ul{fRpg6?@K=S+cm5Kc_jz! zEbb(|YNPCnh6>c<8oJH1KAJBI)NeoIZnH1tl@QTB{@f)-z}DzfvjblQ`9XbliiNBy zj8pro0);_T4>h5@V`$-<1v6K<#TNXlfohDvj}{KUf)BPZh608Oir)PqPg)qQ2{=RX zkalq(DZ2C9E>5SSA>%FCUnOSL!N5f+UHEK}GPHSGu@SFX4J5swBgnj^g>dXw3n<@I~Vw zAyQ(DrJ^1!N98QRD{^H=qgMXn!YRey4g=C6#sTLWOuNb~tk}N6CVkkdCbqpk+wO4j z(e4e{YSir4fz;al(^IWoG0)(qV(GSdiK`bevkaw0VmPgxS&5EU!>66L44qyQZm{6h zf_mbl8Vs|OHx_u@KAp2-Ly_n|ukbGS5OvVxa3*kn^YilfRdl?(Dj-+mR(V9vIS5vp zM+K@2tl~~aH6Iz#&E9o;P|Y*qupcuH&{qww5_W6j`0RP zFwagDdm*_Dj|@4?mC{MX!YqSv;-kLPN#r)o!1NzKex#E3Qq#>uD|7UIi;^pXS3g(W z!Xdr^K;y(JRZuNhQy|`8M zOpD(v;ZTObiGvx!&%ctINBd)80@vwV$RtUYO0BFC5R}Cft|+%TL9aQydXJjp*@v6G z&oyO-=tL-X>v~@^drcXsupzf3Q)^`P2R^MTs>4v1W7U;k{9}JI?(^~Tt92`W0qQEn z7ct)!W`Tea5@3e*bRP9a0AG%epI;RwSe|ueJANDsL$%?b3fXrtzEAajQG<2ckAI9g zsOTsbDpRZRAN%RsDAm<_0jk-r+nS zuNW=0jRIWCy3*Z15?wdZ9+F6Wp5cz6m!`U|v7Xs|C9^dKAfh*kcY)}1>*MQnfdoC0 zdN>3+==V}ez$u2o-*HwfZtbHJ#9aXR7}ihg-_;nLe6T>K8v z$~fv!#;KbOq7#FbZO2oXDqu7G2XC_2luDD@*Usa=6lA`MM2KMOHzR4cpN}{&*Acm` zG2{>)XDJ7ib#qo}uL3xb?SbEi%)61-ADQi$X zS#{a{2Y%%%laFlN2;y-=84RMc()8_&DMu?@S(|eG?lY(ZXvQ_p zp?hY}hqr+E9t}HNxcaU6J47Ibkw2Ud>~)fEx^Rq>6S1i*S*1Meq`N624ZCQ09<4gG zsnK`XNir$D%7Ddlz5dMDR%fN^YK4V=Q$G{5Ci7k!+E^kyn&3zA@BiW zS-_5)E+9_;)>G#9mj}c30X5L(GT7UXzJXvN?B-7k!N6Io^|`1J?NEU$40i&wil59g zD6L!LEgNJspji3vU#md<+=@j>S@Ww8@Z6wnHYO7RWAm$Y$-P5dz>1F1)7Qtp6ZvBp zT7Pw7VDi=o{0}Mm|InA7T1tp9@{l*Nmbvw-5Qv7~==p9Mlr=E$ll`?}@sh~AZ^h(= zxu|iLjMpuz^dmmTA`0|U?Vd6&>N)9EN&ixH|{nr>IsJY>3a zmL{XNWUd~o;JFsd^{?F;${F>#UboVx7QJ~4#&-<<}+-}Ux{~o75&VpCo@!m3e z)#~)S;@nKU8!qR3go&z4-&K$%Fa6pElK) ztKSl$o#B@=TQwpSJ?e09fP>%cLrF<;u^3YZ$)Z-SCp8oP+@MnDV|>m!x??wPLY~r`r}yHj>j*dXYtV3Ehk7X&k>2`FxWji*kS$GB+P<)!3{P+MF%J{ z=u$9YHRp8BFc(Bu+$xa!p(?*Km~)C#{_m1MEA(#)Zu}gVD*LLyzLR88Cvs&=sKGRY zdRja#7)1ivc2dAPB^^P<5U8?=y=xc7N`4?b<_V z6Uu~`RVAj@0%uTUhD9uJNap&WnhWu4HemX=2ey=8&o+nNU{SAaI4kV3P=v$d?%wxz zMsW-FDpu@7Ybyjq;l^bmDd##kvy}Il)YwGRqjH!KEok;u3+*`c(Im{BV^X8;qGS+V z7Sn4B*)77hcCFi$Hp>#%N4p7l!ZBV9 zX09%bvK^C0Tri|^<|O_a(VqW3A=SUXhD-ZYpxwXg)&AIf_Vsh6IZ~Ba!b^^xSmSZ< z<#a@rxYx{y7a~ebI}^6*_*i>I#0tKl<2>^jddjMSo>uO&bQ7|k>pHinP~Xr&hBKo3 zwU{k@(h|FVZi`b4naBP=Qj~A!%j7!wXoTrBT+)5Dq;18Cb6egsA2`*;)WjP;lxgdd zlv!ML+=`P87MGtM1pEPf&K-F3s4V(adZCBW&@Ao9Y3N;+5fFDlN9`YDEL5i+ad!2l z&gsuxBw7sJ)V)6^eJTPbaW#xzu+V+vLBH_{en=(NK&KgaxJgT2KW=+-T8;>T)gZJ; zin)B2M9q{cDid`cKq!YF)-<-8)NPQ%!(`s*NuMydBG$fQ4m=nTikIgDxnXOac<*NJ0{mOc7YwH~= zJHtcyn1gK_q{|zZ?=DnL`{TNkAF0T?cl)jm+488!U{-Pu<|4?stgK8vo>I{Jquk0e zxqkhpB2TRI3vPWWnc<{5!(KLKUaMB$Na5+|#@t7qR_ZLCARAM zmMXLQ>6S^A8%*80aM-tQjLL9u<D#{3((dSOk3qYA=NFF+6HzaLfxW8|w8BP~eJZx+ZCr^)GP|qbYo!OwId~|6;(t zWB#1_yXXO%%~*9~uBY6gjBMJ)>;|*z6u186)4t2k+Pk}PmlPbJ`w+Mn;WzpRG%Y{R zjn&mCMY3k~W+*$Z%=G?IfWQf=gyy-r{RvxmC(kzXli|L<)i9x45W| z_-Ja#Qa(15+3->>RQve>k3NQerPiBV5$Myr>$HdUy655;RK@3GG83rUZx-iTHUuJ& zL;cSD+O2K5+K- z&V}|EEVci0$Hv@#zjD00NOP-0v8XDg7HE8Mz_YGAzrfFb?z(XubzajeLK`dV5byGE#kt-E<{+N_Z+Z43ze_JT_`J}o z^PiukIanh8@8U(UIMB^|pcZnBiHWII7_FK*H1jGtm1{4xgCU;FCcXefX5V7y*z;qf zU$nkCAklhaY>r0X$>JogNyt(skK2CH(VHBi{krf~b@#?C7O_-?lCj#^!y%9M`A>VJ z_0FaRZ*yPt`teC#gHns=>E^Am$!_>ev z66*V_Dvw(w#B*iz=zww1s+kr8bc%%&q$M=<^o&{$;wg7Z`npS^`|BhJPBZ41HAyb2 z&7pWE{@@ljNqXYs9JMKDlE;lLuYI<29BWZldO6=&>Vk-`!lDj4Ka)Z$lxe6lT}Lr6 zt@vQT+9=^~3gRVyGWJp;dF{UXT>9lm6%St5Pl|yPx(fTXN##xKFn`YGz4F|aX}8|% zmyd%7(2+=;v@4OHgZxZ#f;0C$E(r+2%d`{McznpNOv; znohCnwRpyohYtx({cCE?_N5m6Mv6);uxyu3&BGZbEP^TJiG9}h2I34YjBQWaBxIx3 zh#@C&MI|MNp42Oqx83*@6iNToo)#}wJ#pd6(%vqIGIp+rFK;lU?jJox+H>$nR6Q8u zJ(t!Uadp{WzWiWDB5PULr{E_x(~~1+rHAo< zSe?bg>$xJzbNF&iZzFGIq5S%?h3x9~;Drw>EWNzt=r8T+_K-sd#_IeSIXH^GeEE{G zN=rv)U}W?X%B%hRDB<-#PcEwKf+T9s+59!$G`AECn|kLubdvzt$#IKJQl75~+`>SU z4LEg)D>mp(?rtW!;4Cc4j71FR4XT}=lfMTEhIePY`70lD`@O&=9`)Jj&1~%^-jR5o z;q5iGtwvs4;JRQzp(>$cEZU@YUMmBS-b|_|4fK3AAGd84RY$FORaMH{%@$C&zWwnf zWRo87|A<eJX4VzN z>wdh(i?wHzy}hpJ^&|g#bid6(I_Fql^%O&D+im^*hb&2s=OboPrP-hBlM<3*cVFWsmcu=WA2qCAq#I;cer1)Q=aL~?M_y6w>F;PI={6e^h;vux=GUB zgFvLm{jv9esB1{40@90uS492xXlfwDh)6C<{GT|gvycgE1r@M~8rCd3=T$KBI*^EA z3&MI!56-eLO(!!h3=HY~w5DoNd(`9{+^=v9}v{2jibf7iX~T|CsfG)EVM} z$^*7S1GHG-3!a|+`}#mdVr4*AbM~`)w?6pbV@HpszkJC;%PM-E{_@XYpb7*oZNnLv zE9nr@#bBx%8yg4G6sS>C-Ec!t5Q7YQcfs<;Rc9@ux=k|DfARCYaOovSCF9>!oo{nK z?@+?(^qSdzV?1X)MBByj<$}ohKHD+Qrrd|ET6Jqgk@A&pU60XE$zg|SSJ}?;pEDvv z&STf^$m9=-p_{b(^A9-%el5tnp(6pCA{z*25rAOeS8%J?+uI{_p1ihbN!;$1-MFEf zqY>9X3;qJ+BZ@AK^8s@TVNqY+(XMq~?RwVO7AAA&-hYB7-Ctz?ger%qsXMYKUu6rc z8Ri8H9oSE8JRr!UQS96o7ua>jnl{pDKngr))?Q9o?pO5*6(y-YTW#)Xt9(`2g+Y(F zQ2^lZ`Y-){1MT^~7mi+SQ+N3U`7~}}GE3Dt?;ifWMu{)cp)AwuyxCC?q71~0DA_(# z`86;&Q#lgwzCP5-4&-^1YXb{F?Sh-qVx@0ro`n)_7$A+ue0(avuQdc&n9bp)#zsUX%F51m04|u+-7mpoK|2BFo_Kw+ z0FHnaZP*)r5fR-TP;6jiVmhNT_qnPRsUVQZ{jny5_m!7bhH2SOlA%-SmspZv3)rZNzHp?SU)R@w0@cR`o-h!wvf0 zG858<8th{(=g)o0YZ1Dn|ALM<$m@v?o_xT5&}9O*N$L*T){PkeFcUO4RG>pf><`Ub zXfC{&+2F=SZ3gpGGq+ghTu)A@2ceYO``tT2MT~bo%wSo-hNzM&}8-uw@`S0}ObIi|w&dqyZ zx;ZR2zz33tHgi8;tPP)4BFYg`ipHE7*a|5zcrFg2Ph9P*$OWgl7*wg- zQNbC*{Xw4{cGQ=q(r182#v(fHUCF+X+|Ew|jJ-vM8Tbjww~93uZ?7zrY$k}~&S5kd zM3uK3hcy=Q2Hl-3`EMWhDCMJfMDHwarD})}l|L%ZHjA-Z!CYQnHyUDPm6sE4gEE6~l?T1*;$O@x>mIeP2(-M*qd5pksG6RHxZ8C$;sh((8u5 zd)cIZ_U@=;%UV8W zutV7dP;LcPsu$}wViOb1+zTntF9%29`;F75PC0CwX=}TFTyB2<{>0KSz3C3nC~tAl z+tI-b-D|$*ZXb}tP=a_%CGw5kO@q9_g2y_K-v9YQ`CC^PfE7?+}O$K#}S-y z!keci3w3K2$YDhVZQeTB@w3!@TEiSFWpMDMmOqM+ppYZp!-HPrpQ*j0I%40_+Z-fT zPN?Wyd!c(w;+LE8eYWYuGb#OPpDP1RA>++ns5b3~6fT_FyRSwfLVx6Us}h!iJ+kV^ zk3U1L-nY9FvG)Dh_FzJ)r<45#D(_f=MLCd}{D7L?q4D8CM3e=Qpp|#joUIo&*DRwV zE^jtj0EB*|#Z@x!^GKcS$dTds)zcOm6Z2S0RpxtB5j%@}{*%dn)7P`eiZYxZoIH); zJ(al{rQ+%u^UA9}Z`fPcN6DZtg@H9JA3}|7_l!I`7n0+SeA7yGb8Bo`&&hm_Bq_6g z&H1sD?j`?s!{X1R-iyUM6s|7=r`TiA*p{8d0V)DZKCHFzAgz_;+I+mdtIGgqQMUrv z#o+NXTW`w$hVJD=4cnyY>su?IbEnD}xzN!YuJ|516ZhfutohAYi~3a$l4Ht%-L9p(MX}qr zpMQ22wXv~5ZPduceEfK$GfCR;$&)9ZtDWubdDvSc70$5lmW8sWl^omR8CEQS23XaX zBbsTd)vIjmCAKwUshFQ*U>EOaa*~0wNyfqhWdZy5S&^$}2D;;%6LEF8F}@UH%P157 zu{>olW8wfzZ87Ptz$~<1RDDe27?S`?QKfQ$4MKp_KzIq-jjz?P4Vj!s)& zp5FOts$Pd7>A1O-&>T7>c4w?sYA+VlE@1XAJx9UJ%#2b#NUTE^gLU&=X)bX#sl@%f z`@lZ$){u#fev#o-)x>u1K-`y?)*ndh<1^*Q&SA`{Wii);1H_zt!|;2nji_U}o1QIP zb`M^-oZnCoAI!0r685$Trry(gM9048QeC1(j?mjLdju`(%^)o7-CSSB7vE~JtZf0+ z%DDk72GFk+s5Pk}^`7yH0MHY~Y|K~kr~voZ@#8P_5uA8bTcTR`=I`gfoF9!V3E#kB zjtT0=`^qN?AGKZ|H`T`b*4V6aoXv>7da|F`doPMZdm=_+Yq*l%oZd04{TTMs&%b26 z@kWu0_#$lwv}wWFeGUy4(ecl<8Yo3K&}j78~)=f5fX@D@B&mvLFj z$w=_U6#Vy~Mlp1+ zlCBW`Z%^AyJz|_)nnx~zHXG@c&2AK1ttQv;F18xnYdfV&Ny_<+bl&GOY8P8te?G z94cl?LlyaKHa%(7Tx#82I-t}&Mp%gLCVhJ`0iWqkzqPZLGQwyki5i4py0%DzCQazQ z{ck@I&kuR?{79uHykSJVfkIB!(zh1kE8ApkJ-t{Y$^iUhJD@KO7vFkxlz{=k5d4yo zlFxXMNtbMkn2k{2gBh~cKsprelobNQut2V;fP zfmRQ{*m-5o^`~bhhm1z>y=<36q9^ z7@@7z6RlPS`oK1zl>}k@+-y$DAVOOn>QfGjv7Xdu*q;aUJhyd>?mlyoc7Ime#UZuw zLBwt35aX!L;~0%3{-$9`IVRuy_VQ``h|PG(dt5+q$%=PU#?POB*11lysLwR5m!eCX z)CULdUxcbR4d5f-MI46DF@*!J@DG>@5i9+yz+x^ra1 zNa1uGrkSM5a#=;?bdHz_%OwA$%=w@c1Az!KP6PiZJ|9qJ-wl|btb47jt3G{}8&oK5#WFoUwQX*V6x{y#IWL|we#-PAxg5V8JgK#VACa9JZJk>c*nk~Fj zK}@<+*+CTHVW-PwI`=Yvi}P~sTw{Q`iRKWwYIu9aqv1l{)wF8E7k0(v(Dl@Y#jJ+) zQo10hOp+i(1T5%z_jErHHywx1Xe9FXi+X}RN9}JhnP0!&G!NP$LD2r-04O+u)}quuaF@uT?ty#OJ*%}{`(je)yvvHojefoDO~bN{Ny`q8owxga zi@ti~6zRE|X$D|gE+t&#&mUZ;Wa&Mu`$V5ffq{X{3bLfF1fXivsDoD?eq;4O%Iin2 zxbrYCs5T*WBP*6_XvNrTMB!ae{q5(1<=F5qFF7%?g0$&QJh7E|-I z5|TSxITLu7?VYil2O^x$Zjyp!7RqY-@+icZKW-;vPu;nc`zqh7V|wG_ z=D|uIPvcJqK1kF3CPlTM$jqG3Wi8~mqhGbUIW0a^DMWI#%uL3Z*b)sH4$RU|)5YSC z_AKr}XGs1Zhze*99_+m|ov52%rSnoKkt^k*dJfFg1^lXjidb9zV`r zqVOX72wYNSP87Y#haWp~G?T4`q<#GvILRy3#kwKY?1}TtO8b@1@^2-MunI;%Ht)M^ zF}U#J4(U(^9iK+fn~yy!;pcnha3>Tf7JsIkJOb0E!x#KnFsFMDR{Kqb?` zA#ZzQe8*uRS6ki8YWXkJg0s(Ea`uyxxQdR`InzO^mN3t~1DwQc(kK5nMh8O`@XN{3 zv&(~JkIH9rV_gZYtQ@1f%Lx{htF;@!?FO}rgnAWclFo_^hD=srO?=hI>n_2xi~jvj zX-#LpM@K#6kh32}V6={YoQ{qTXedjdbm4!%{&42=gQ##jIW;v`P}2qPJ!cX0>=9J`Ye10-0>?R2 zFgCVL$^!L6BK(Mel<2ASk>^jnSlA}iLsYBe<8Dot(_k%lpZi8?L(%&4>Jeu!{(@LR)!OE~R@Q$8mLr7`fp% zj9`b#>BD+IkL=;>7iANC9K*c&Gnj%`(Ro&uZ6X&vAUk+ zC((ZSjgoV-8DG=Qtszn~oV{~9W|tN>uW)78WOo)_sEfXvwXjYi&BYug*!{cN$3UzS z@FNI+V>>2wuD70FTw8Ff(la*py)i)n`kyz|r@Pg?z5861KW}UJk+*t()j`FQfaN0C zQbjIK{@DR+Bn*T$2ob$4oShn-S*5aol4(Nw7}e~kXv z%*!u|>6k=J=BT77dq+ z(W}1|!taH18o=DZnI1~6SFHcgyg2pTw5ALQSP(8O0o?ilLeu z7<+7~48W>KjvOI>V{u&zzf}+S@%ZMUFIJ01&cozxV(e7g znxkdn-E5&6;>E1@O5*LcXrZ12mTdb3^^4ITZbnaq;5C819-; z4k+!Di-+iTO8#$Q;Pvwdgz0|E+kv>NesPu?p9(6ZaHg@Kdd97(Xs=#l#iDvbT=Ale zlBkJ#ICG%jqH=>_0l(w$=Mslo-E1EC$V!1vtUcykY>c~S`9EKX9X8&Ip3GN#ku+?+b?Qg^&M(eiRe^`|&b>=>5Ls1KESn$VE&uJG zJ%*Xz?`gbTkp3NOUnyW#rxTLo?e%utT@6hh*H2MgME ziDl0++a8E`8Z^)eyNhGLyxt+3A5wS9kL}0lYgwCRe5~V=);zT1TBCHyaHE_vY%Z_2 z!B4&Ti8HGWg`#ISZ?X_NgWkxVGnEB8kXG=e;8u^97=$YVNEl`}u4d&dHCU!xM#Pqt zy4aUpn5+BXSVnFW5dCFqrKi17EVc3O)ozT{^Vo;dlO_LP*if%1$iL14@~6wPDcSUk zz_M~6mF^4SwfQV2B?@99W%JmFY0Rx}L^r*s@%dLKf5DjgAWuh$wyn5y*1{*0iq`+q?6+T z)UGk1bu)aJ(|FBbKqRA*AY@v&Iw%%;bm={=6`Sw+!b_@MSN!0~-naW&sT@vIG4 zf#6cj)KmNXd)~F#Z%N2>ocBA*NT}DaM}%lONY#je}$6m8^+6DUYX8c zW{he*dJH@>B`FI$PGQ^!V?@+-0k+P##J2Syh3gexwZEMoV|S2>zVG%G{Tpk#agsl- z9%piVKv>@PmL`3#v7B+~_H)RAw`)H=mR9zp0ebRjWhw9D%ctEJoXZR@DyU%un>FIL zIjp?a+7oMW5PfWJBQ%>uJcpArKaJ&x`S5=C@besR5m?#k^Tp_K@)_uh5pFm>7RLR~ zKYAc#B-I^#ZB%o(wmn#=y@<6+?iOZfg%P|SOm_FJYZofpC|Cb6a)n{UI98f?+&5pq zVVKtO&2U6=5yS&*wNDfKXw27s?NMAbEl?I!6v|ka_t%RF{X#k2$u-{hvpDnU!m;^{<&q5etl-%GSwP`l~?83x}RMXAFqAXx;U=`Jjuq{E9sNnHKYPd6DGUIq2j33zt7~ z!Ny`ubHCzy@fnK24rb^F+C{{15r=Z<+_nQ@YQWn2K>Sn`t}eA`UhWVlRQvNPDn*bX z_+hKjLl>8G%}k>CdRoC#by`oKy+kckQJDVDE9UqFEF~Od9ZL25ly3$q=i%i}OTQN4jnuWKQA~qYCJi!J+E6h^2S1Q zuG7RykDOUEFj1mR>G7jVeI@+yH2dd)eJCr7i8%_)?F{euW9;mO@84guv9*V6Vkweln`){Nw4aM^XRATV^;rdx@y0qg^=j=C4uDRVwg%CyImcY3wv= z#NjW~^?_w>F9J8ZggreL`|HTpi}UiTZbLoh8)J}eW2!r=N7rC5C6NBFx+8=Q{Bm-Z z7Pz&S_2-CGGr*l=0OMoVgPOf^=Rf0blArx zCy)IOLAI&z)7aITWU^afd%l^N^HQ``b*74{M4j;DmQK2`JIADy;r z+isiZ-GMHP?7z1pZH-j3F`KJ<6>aG31}GFGD%#Vj_XY;Ax{wZ2`dxw4qWohGL@7}w z3K0P)Q#&viP~NAIDSh0`EN&3nK8dw=WMC+IT~qblcq!Z8CFcZnCH8ks6yB&N==_N5 zb%yQLCxi1cnbM+ahIZI-(u_+JeyfQ7Si(TP_S9|yT$>8#2qacfOFt@GApqEOepo+4 zIjYF_5UR~0?i~zcfg3|Ayl8^ZF1#f;I2;gyu~}mBt%C}3#Zq*2p8C^7#}~gzo=oh| zW?kF)4$* zw`N*ej0OsvsZ#9$jflyFm>r>L2u~9OHR=DK55a79_Z~Fg?KR_rKj{$`{N$#^=|8>m z;msJ|M8fwigQC3~o5Q#Ko*FbhTedM`by-_mWnd`^-cKxqOsmfRCUuEkFOHm-Yc+HsX!PODDa_cqKIw3`WK zfaF4;z*{jrV|n6^ng$G;84)>sC1#~9Z?yVbl1-kQuc*Om&zjo;Mq>$qS9+h!`8PEj z!`uH;EK6w(Nj>f$zU#5Fnc1g;nEF2*!)fp(oyGNQ_uoE9qy6ipz_FqFHSHF*wV98% z64w3^F(kfwHBN-mI?t^mM+@$006YV!p`5_Oq9nO-fTXV3$ruJ7u6nG|b}_Uljn zvV$N-szyBo!z>ivz4nbwb?~QM*3$-!G?jTak1@XPkArJ|zZkirT>4+I=5z)UGHO!t zK05BCr9OUnAc;Z({)?sa)H5g=4T2_29;1K(FPy}h?6^*arLWioYXdEzar0xWurq2$!5#buPse8;Z4M3dL7gz6ZCJDi;QPe z%dK0CFSZ$$<*nETWR6MANGSDTb@AIV)PQ7b{-k&cs!qj!_F`XHU{sLO=XbZNz>5#b z6A4x<>c*(?yXU6ls71&=(pn}cs0~0P zT?P~|RI=^oV1^Q5$Tp?0?fF>&F0LzR%nS_;4Qg6L=QZr{=ZjKsh3pbe-$`!1#5}u4F^xb2RcA0+LD_%w^N~7P?jyj&3z4&nVxf_ zk|2tt(Sv&8KL$$F8xmXI}F`5wLD^~KmWdeN_tg#Vb@%A zngSSpr(w$=#$&{}a=m=VpIGlQT0I-WY|(=@3M7)>o@qs)&(Dr9fjrX3L{I)QLws4G zS*d|cWnErH#(Di~H5RnuhKl8Ci-9%jX>RtROvJ;^dJoOS2%{yx_iM)-iyzcC4?X!``vGo8;|3kd2OaW3ofHZX03EQD7@ccxYqM{p0kawWydN?rne|B%;D|U5S4(-@ zmDC$%l72qjCmW^qVZ5`TT;bgkzq_QELOAHs0`bpKWmtO7!ni4e&Dv%6iI{*o4+I7P z&43bzvnP($RB|JQ{lEaj0wbg(|etZ<@?{^=>rVj_aeJ6@$5q@`op*xsCl@ zXwO^bU9_S;Mg}%>$t}yQWMB6OwQ1G~*@(Xw-jHDI_`3e|lV@ssBkImxmOu~r<@;X% zQ+*9TA8((4*&-1fhmi;vEyraes0&xm<>e3LXmTPU9jMOYK=_K(qd>p^%}w~r)OC~c zvzF&Sj`K#X{WMm8XYr_cc*@>L@5$8yxxkeuVnD)hM4f7U>NAGq(!NK{N|2?wIJ@TqCK#6rMM@<|{Et~|P=W+yx!Q%x#*95LDkK!Y2k44to zJOx(sDm`GETMbk1- z;DHs1pu zzN}|4Z}9jtxxBt_k@V`+ciAb|m{9d-v2$||iIj*S@x-3(&;d=@GelYi)yZ0VWLKs@$gsfB5d~ zZMqpJH~E7H1W62J@zW=y%lp?anOU|h*w-{*FPQX zt~yhrMQ9MVyp$CeLwBdwJG@@!p;_S8ZTsM{9$s~Vgp>l?R+03v^s2cGf|wHXBc;T< z>aaAtl1EnCm;Z3M38R401EH_8V|p3?P#Y!`#$>{fi-bUfeEn{@omx1-WGcPmKK~aS zqJ-Xbg4C4IyJ4`*<+DYKg~?mvjj(z0yz$<+6S~r|LB>wn9+SJZ64aBE&( zdgu@Mi)Km@aviAfJ?;ym2ehRiNh|kXcT;~Yt_~CSK(O1IN&po-_hi6VE#f{%c&Odm zwa0UaRIln)p!e>pgf;o8|Ly*zPXO#JCzI39M6ebLVw7-rnv57_b;GC{*^`tm5UR~| zK!))Z&Zf}!$_Ki007*A|l7PGg+V)Uc2?Y9H=Qd6~kU$MIZ0x9cCYbCvOUy3o!`e1j z;A%^nrX_gvZ|8gWp7K>=hHzl zY27Nk{^6Br(bhyej9|bhRph<@enDW0ER2kD|ASP=#H2&i3ld{u6ehU*vX+rW7T__u zuD<2g2@a0qnG0Ft`Hl{W=}q@}R8UECL41?LU)s(~*fc>*-lgty9P*lOZjn(cdpxD+j zvp-}M=3;d z586zZs!1lG2BC5jl6@}&0W_0=MHCPpysq9k0Nat*p1 zWiF0K2fhcU9bV%iU5_#}`Z;ldzi9lpDn~%oVaJ~bJ4s_p>~887nU`Cu+TA&NGK!FM-S7!{s#Ci*%Vf0g8>3Ts>l>pu z(aoER-qMo~r(CP+s@F0YF}H@qRh-ZucgRc3;PEq?OqWPRf+p&;gyzP4n4@3^s(pb0 zQr-&%2s{T$&>^ij__=)l^)Nbu@s^a`c9@n}<+&Bep}6`KV}5k$tLDPC7^od+0EqP|*- zhY_!8_+&d*bHI3iU_he?2D+hwc)@}&Bv3GxU{2%^s2^&=lEDy@WFMaA_ud)D{v=NG z=mKA;5cru3t#UmN2Yd^+=G6Zkvi0wa&qR$~)eK;WHd4^p)b5@S<;Ar;w|0FVLy5*4dfhsRt|QSlC+)_1=!cG9vcqw4X<}h);Chk`Oj0F zwl9%ycPJM}$Y1NXCDB$RwC+itTG_g4>+nHqX8zC4h7*R1jtT|XK;6KM#}zVcPRUT& zCU$wp1IA$W@14kt83PUab}1`527m6$8Odj6cIk^xJPJ&pe9dhF&8ifSBKGeIqN?5P(!ww3VxtK2DT52soyFL`VU99r2*_RqrVc{`plU7^GXdx`+oq+^#7 z#Upf0zEsy@vai$vo&!@z8-wJO;o@rFo*J-Ts@%cp}N?v-`R6xO_GX z#e$`zu);#&rEZ#=blp>hAsDQ&07z&_c?Z zP>eMpOO}#-H#7Dn*~t`k;2Azt0)HmdLaWEU@-uy z2N+yOpj4nB6LhLK(M-Q%1m)xE&0VjygT`7^qz>V;W?sfZib1E^NgllXUodR*r>UdD z^+>On0oVxpK?4&~>8muM!bqr~saI-}UcQjsK@&r^9=U z1y)Q*@|bl*Y(ETYxz1y=y8)z7rlb_KR`%?}p4sS#G>ssLkL3@c9`$&=8P?ps<*GPY zn6g&;R|?<%Jo|%}IONq0=-C${$i;xiQT0;*5;}y>g=jaSJUHOX1z>L3k4SOWW4x(F ze@@Xu^}jj+H=1kF-^_9E<269U7qzh8?|a5`jBgEzupO*4&Qz82<-=cKLenP67QW(` z6}w+!e?lu5bzJoJ;Tt)nl9vKEu;>NcZVNze8jT2O|I<_KDhRwpp+Vkp0Uv!FCTw+&%mwf) zGRtInOWtG}ksDwDDyj$$p6(O7gcm2ytPeZk96%5X8d(6}7+_1G;(QcnKIQo3^?5Nr z020Q5`V0l|tk4UZb(;tnzP}on0CZZW82I%SziRk|>-VC&-X8SRf`c<1+thD-6s}C+ z5-cV6R|0Gphfy8vYn8VO{NZ=<`un^e`m`XoH zJPX2a<1NV6W;|Pc1Hc{?j{TT?L8w5#I2FA6GPhSb?98B;;^hqPnkRvpv1^FsT{HKQ zRmj$vS_L#bZgTOqD!qnT-FPL7-q1%pRid*oUpwonNKv3#Ou(8$Yy_^2+e*EYu1VJc z6FvmR1f45E(xE74oc-d(9Dv1|1L(U5Gbp2@(+t38M8$UrO*TPfM!^_MGMB3J#cTb7 zGMc;v7G9>7@nBjP%Lhp-7R7xk&V8=ydDn644c!Zouch1U!KVevaV3G zK@bQ4{8=HWveXCNt@U(t_SB%JM*t)}AaMe6^W@-5n{z?l%*X~*R+*ZCdhA&RQuO8G zaN1-_te5?BLB>T@vC_rgIVRD0q~($y_&-#Emu`TRRWqGj5!7KBINU8V|2kKX4Fobk z->$4b@41j+frEb^Fd`uu;k}c>7`S`*@RNmt_&(d#3m42+9uKYv^;QaXB^68urduDW z#4`YeLDCK&!><-VhA=W20088gcFo_6$ibiWbEw9CjRv*><9u*ajmbX-)Y*?uHLfi^ z8Qa@;r}_QJ_GpwahHeT&zQh&!1cP~t3?cZ)SO(8$qw*KtC9Op~!b& zD_w!Am0v-Hq?wf_ab|_xqbPuOj_%jlTIWKs$SLS1A1BF-KaX`c+vz>V!1CQ_>yqF6 zSc3Ob`G1_fo5$Exz#Gp(FtbZHz=KB(4~lAD38oOd23|)gtvrDhOK5)$3r(28>qgPe zlRO6^S{N=1jJoUGZmLp|VO`jmt*x4q-ndX%_xi{o5`gR=@Y^CliIJqFq?(r2z^}>X z4FDenuxL|IL9BcC?sG`xf+}Y2>(@!3fRDx+!XMh_0jHW^Ikt|zso>Y1h#o^T@j@Wd28$khIW_ya^>bPHVNY| zT*!*Nm97&qPj?7qr!uYwvFS$h=Ce!fQ8k9(?FE=HK&Vor@1*Gq-t)k2RRElsd^}L; zftS94ie9r)C2xzSLLTiPE(eP&vq_7FY*_i}h0feP(he>8OZXex!5Ckw)=NOASFXGT%mr;ckdT|-)c}wcHB%=lZ|Y_t59cg&|%hj^h0U>QbnIuoI?tPAtky=aH#^aXwadejtdP8$#Wj){YN0W z%g%A8fF1jA9$QBC9~GD4t%*>wK;g27H;M+Iwm)I%9SYdfd09TuBZG48@giO%3GW?{ zSluaV%&iQ{Y=V-ycCbgOlI8hJpqC%$=KkuEplKnfT!fg*L4BRi)|i*gWhG5v$k9ms zr@93jHQ}wg+|S-F+>x>w|0GS6oU%io^7ipJD_|{WXPmo>yBe`_`t?O?6~ik&hQ_yT z_V%r~X=R!|ktmVa59%M$yQot#$3)}y82;>aZhpI&Y_8rPh4LX7(Z6DxEEr!|~gX9LtMM(idcm{TM_COfe zn(kOc{K$GgHXsfs3_xwdxtqte`gmt81ezbp%#;FF3Q};=jQyieH)^--TotcyOp@H@ z73vu%j#%Ka%PPfy3lt=iV#ZtyK+nfuT2>5T!yPcCqoo}HKEiP%^ax{|xe80NZCJg1 zsiCvBAIypT9B0zSiFXVR)%3|LWI%$xH@2)p0 zuhZp1bwNFVdQk7iUEncg?oupd0+p zX@eHN44syCwnL5)svmYKpqO(e*)IOwJ1lh0fZZ6NxszCEL!c#1>@9l%9Q$yMM*$!x z?$76k!rEe$kykusPd))h=(jGao#gjz%dbh=qL1Ah$++ZyZ}woZzOcKZgtYx%pZMc> z(fLHba=WfX&_#0hcMHU=Fj(OQidLaa3j-*uKGbO)5%uvX*L0lu*vB@8d z^6~etnzUhwwG8r;P2b^z{wDBzALS2cg@F)gHKS(uU{axWWcRyVI5jQ2g^%6t(6CZP z-i>SI03`{4Vx?wCSZj~&FXgc)GL{NwNz@1wx z_?m?yy;H}w#rKwjwos2JnMwI~nw&oInh`e00X%mq)fJK`NiO*?Lhkc;hmK3}VkUGHc28K$yFV7Gt}1pz?@NNA|r_SCY~`vSz_| z$?#YD>Jn1SxigbzJ6K~d{7nwMwG6ojtw==H?tG2%93E%vH>>d6KcR-5A<}Tvvg2UC6wj$PA~2ULmBRxj?f7%bkRf>%6-JKF{s%cfsE5 zZ$H+2l9j$QDCcQIY^ki83zM-46%ne*IJy}Ue`x~4`SxZGFGRg2e@4H?^(C#qj=V4J zbYVqyZGMA9S%coc=2>U7+oVWI$smPgqAu>R6%s|V^~jqL%Me4P<;xW2S~7~O?NQcW zaUBc@p@K9ep*lHZ9xp|-xZkELD^FS5!Y!v^x_>wIx_&2-zWJv`GASWz?)d!K(C&BI zyOpwDEWQ;luEkiT@gU@rX4tm7m7 zk(}`+Tx&_e{_k)>et+D+CBK-LH?Sk?P45q#dn%gO6c6u4TJD_N2CHu)tzV;hRkt`y zgS~{ez}n%#U`~kcn)qT*=)q_KpUlK|r7wR-^_EFF-YO6JSY*?p2S=IK}(r2>pEFb@{U}3&-|&0 z!b(L8w}Um&ANY`?9B-h~{i{Gh2iksJC_=R1yAsMR(Pd$a?zMjIk4*E#Wd>nV9+?&v zSYpydXlyRcP)!PGcT3tOx$X^*C7!Ennif)Hf2{j*fO40P(1P;i4^r}BTP5)QRl0<< zxPt6efXL?(>@KYqZ`}&CuhF+7Ndoav{avp5z^j)&aGZvqF@r+rsa(3UpH zin-}CInjkasA6<-?wMWit_rr;DJ;r5AF+LywfS+w79QYo*hm)@(*Vra@y#-zi6Te1S8F#PKEYH}96LG)9i!4dmq*-H!Kj*&7 z>F4dgXQuR8FMqV)TWON$vI1q*fyQ8FP~p=&z}ZWb_#SNF0_GDd38Iv^*^hZ1FOU!m(8QvN5Cy2)Ex($|Y5uNqX}c;Ls;8ttB4j!3bdAkMPT3&Vp zw6G9<2r@%t&?eMJW2@M>QVl|L1OM9P!{kZOu(I8@+yPLUkp_z2(E_4}Fu~fnx8r3% zf7(X?LHBKZs>Lksf_S3qt$CUF&zruK3RN{$N#vHPDBvhli?x;AJ5`D049BP|4xVP( z>wa{YK%hESh-~73Xmsa59T9eW2Y|^V=NrLe9sr0La4LmE+zMBlg}Ep#TtQK0Kd7pP z<^~1TP-iioN}z>OzdZ{^mZzEZSNSw?wTbfMBbhTvo6VkMt?8r`Ca*7${E!gur{B&z z=2bavd*_=srMiXcZ$qrV0!XnA?4iK~e=Gfd3*QSQ+;pDp@x)*!=^S)}Y^h&+Q zPeriK3VDDf&iOa3xc8T5LQsJts*V@WaP5^b_+oFjzI;Zvp#D{7Xfxt33nR!dNpng?9qNi4FJ)cFkvfY1S3?sDFXAE=A}ez_OO{pvFIHI zu;jV;MkA?!2l<~BBKJ%CboC1rad!%H8$9AU`c_g*E-k6bDHK@j*yZK2M56C0G<&v< zb+wu6WAZ#?6S4IgJ#U;v*DE~4V1kX7?2r=;uJ5^HXdgUY3Lq!|2CBrQs$7pj*IWY>DK3vDRk(INVTBdPZ#V6>I@9!*e3{hqVILC_QBQ+M?gz(q zEfH?7$nZ_o4JJI4ax$A>8ZrRXr3QUF>$tW3w;!HH+*!M(`pDt%7I`Rw&F55(1g+bn z0pAVu$_?->1kx?n@jI%$;Q4!iK!onl>1N@IkAr5G+)j?etJ!squD!x(vYBrq?`rpp36ZskC zA8b25uu}26<5c{EiM))aK18lP!{)oJGq^tq*FN84Dxh#{Q+4M8I|Cl`5qhgr)sLOU zO*%3_uvE;y#N1LLO?do z1reD;yqW;U{QW^CpcYZl>tOlV_A?|i>6T2O6`&LGnDjuRt@~$TL*xk4ySgOFJz1j{ z3VUP;Q#Q#|7XXtv_-V)%V&*jNYdS-sqvyg+h~dr_hq7QkCGDvyfbNJT#zz0;*n;ll z5qBnJny&e8U@f4G0~A3)SObs*K`oCOLCyl;Rbb=h+TE3I0!zDbs!x>oz1m!n&lZ#U zUj9^QyUk^8wLkkFc~Ku)ajy4OqOaYH+>Pm7udQ5LxQ+PLFGs&zT!iv_Sy`UlRS&{+FS^>ebM{wqv_TdCP{beM<1mQL*v8ZEvAlgJ&r@ zFuPglKcbaKv(@(AFj6YgAq8f5TYKbS=lwN4y7}1^?In2Z>W)QDyU@F=Ik(O?6j{v7 zbLywSNojBS$qrTf<;3=v#~bKjQe)M;{X&{POjQLtDfha0=ah>MOu?VF3Z2qR0Q4PW z%yc~bUTN0?o)ja_8>jg<3f9huqqH<1rG`)@PSrsRjVjMK3yQTryH3E3I@xR(hmg8U z=$b_4J~Q77T0O7LlcfT5ZUmiR-=50B@>{$c_eC^DlrvmEsO~l^bA0@ThbC>`WG^dV zOUd_8KXl(YuL6dz&;!^w(4`CGIdzPf7{8!nl;{7wZrLA(3D(O zV{pOKer1snQU8nK#I`AUTUDpM$d6unQhy9JbNPfxRO1*WjZbCoO`$(BMrXT@8;mD1 z;04RDcwz!qa&VU4<2{scW7GA`m7+I((kwRtABQ@|;uEI!{5pCqj;;up(hVi;O=BLeX*oVM2YK3!hR!S3IM>_gQ; zg9Bb{2%X=8LA#5?9AD>02yL~G{-vocGt=XgJ={(SBZ`Aup*L3dq!0gUUrlH%*X*F%?shGxv#mixy9iNchdmp%i1?BUw7B@@A`V?f-p+!`V%nU~s-d)xK>Yj~zRJ5)ZO}i@F-qG^XuBK$ zxpv7KbMMgWuM7dX3u2a8Q??qu9X8(me5k99R6^*|jx#PP!Owf-xui+WGgkjObU2Ec zDI#=cntvDv&LyegLDMEf0x&?u-P|j@Yemd^Yd;{s6@z!4F&zHlu*B&R}-kB{JC%h2r zyIk_B)L{Nixwfw7Qxz*&=;aOQdrOMoXt|GL+!+FH!->8FBp~7L?JyfQe_l14Nr&Nd zQMJd;z3U_AWe6O+wVgX2v%BivXUyX!J$B62R8z2xQr%W5{D`cxZ4_x}uN8qCRd zo#@-|%7|gftE`$e!A7RMdz$W#nzxSVa2&a>P09388DE16_X6V(L%}sLV=)uZLIUvs zhjHkNA{CjvX(k8?Ah*KjN&p#S(+;3b^=p4<6Y!_-PLQu-M4iPOvTN(#X*V4(<4j}l zPjY9<4tJIm1_j?B^O4f6zThztH)tfFilL3{jr1q6S#&dF(2S2W1zReUK``5U~^&SLZcr-J9)0wcE< zve+a05}&$7OIopskQ6*>b3tbz0(zDMI~Xhvh34Ji$mNm3pA776NC$(uGs_Tb*TjaU zSues|#*ni%k5@$WL;l$MHll15&iZFj)!TWy^4~Gc*@p9;!JV>eEFCO^CX*b`_zz`1 zB4pwY7+X{!9WMgO6T^;p;l~*rEr&eQwQ;=x#ilsRQo=guTnb?LTXofd&6l#7QXgst zJ^ofFuX8T*=5G;5A#Se?T0KJ@UQfA+q6j|xqvh67=ivT&+Dyp+SS{ftb( z0*Ny~`d7K*Vetkt(GF~liOge~br*wSH6IINe(!I2!`kC_h;qpH)6Q+PpxiJaYltaR zGG+=jGJP{esshV@>BzQGX;Nh9>PIO5vM`H&H1%wVwI9;$!$7wqPqf(k_NpW(SN2N+ zaFbyY`$Xibw<*kYwTjknTYDR`So!asmf3nfDz)> z;|Cw*dxzuNzH=Eix5b-3jN|eaxH}vtd|$H1h}72@0TFEj7FX3+>~yrdY=9m&pBz?9 ztMbK0Ot7Q*cb>hOQA@!8q5eru4)9EIzerwqekNpo(Vx|^GwD#D z+Ls(o^f|C4L$#7+&PGLe$0WdMQr6+84&n@LfWK~PoTXXhQpHGBD4epBy}mP$SNU%_ zuc@n3ZsOl`0|Iumgor60KK{2+?vxYyG5ygMa*mnaa@xHMO$xYL&|nA{Jre9ZT1E33 zPSIlE%PwClSLO#NoP~P9qOo%$bX_vdu2OI&2wylgdBr@wbHv=frP>sDa zaBfLULa|17jKD2mZslX$3TZ2n2l3f{?r2_3<-a5Oo()BhVUhC~-%jT2u#hJ)Uq4G8 z=3;(KQ|ijyj*V_!WANK)#JGE52$KX*CQ&03Qa#4^8c*o_Vp$g4Vo(0j$uuOO$xvtD zFw(@HV0G}Xu@#X|3FlCxZ|FP!?_)pDkuUzC)Mp$>%11}$M%jy&HX~2@5ZeO( zvU&Byp7|BB+TKQ$5==}$!BzW8evgdEv31eI&5a15*uJM%E)~DNoQ?3PgWbDWA>$B_ zSBOJ7g7;c^KR6BtW1>`t zf7E?zUJXu4_kLY^S+Me8g$aMKAe)?87ef+8G&kac)iqTAuM#yOg# zJAU~!iFx8JCRloJn|%~BR;NF}guNf*1J40eVxErGXFD=PG>opC*4*GZoakSabN_X# zr0mX+xRCzW<$jR{7tn@)m^6$OofSi2cyx6TFCO}YM~m^R#fmoi6#^xtR@ZcpJ7CyP z-K#;%i>>kI0yi;%byjacG)MLKT8*8x|^@zg(|)|i>@B;wnqkW&2_%nLp6Uer#>?? zMJaNlviJKF0jk*S`~4-@@l`gZrg=51$@u^thIoLD?IY?bZHota^Er*pdgx>9=j%|B zHFuZRL$wA4^GH9l%sjiiUnVdy4tV$pdAg@u;1E>nC?6u8;`dvaX7PJ|DW3AEOeNs}#R`G??Z}iHCPwJh zN3;o&sUmcS~0c6ud_$=+c9N-d27En_2N^C$cM21v7FJ6Zj(hs_G=IqND~S+M*HGs@)w^r%0fZAG9UN?QuNxjU=v6 zR&l9)y%t&ORlEp11kya{R=~F8)ew*&JN&+|9QNX2{KYu#YO#B7dMu@G^cPGgV2MR7 zrUB{S&HTEqjJ`j7U7+f|M_J1yzzfB(v*~IxQL6#mV(|8}T4a<-%h|cLz{tv${T5!} z)YoZ>a=utx%HhjUv*Cf{H#WzCY%{efpjbW>FDe00p8!tkrm4+#nX6Iusu`b+&sZ6< zCf?Pf70@F$t~ymp6Zwr^>{j+LElq|b1HP_hoyvCK4^fsq;;x}5N7@sjqX97&veY+^I;p<6 z7iFO?tDR#K%eoIlTIZgWRMn_u_Zg!Q3HQVbt;&|uML#&Ou~4@22Ny-BqzHhF4B%bo63iQ@eDTE}Q)tf9 zCmhe-uXyUAFII$0*WqqL%Cb3T&e@{P#C$!@E5#p?_dr0$wPS$F1{-njWl`;e2RAw{ z4=KEqt^}prVtx}%8~iflZCFFyvw}v{6P!a7^_c-wb1ez5m@`~q8oeBKD8SthDa*E< zEtplP!LJgQfBG`epI;c9B|P6-UkBdQ*^cOTrjqB0drUf;dEqh>DzdnGR5%nB-8)_> z-+;{1u>sT@&tqgnH{weG6)Cnnv9wIpe9EbE;@~{f%3~=HiN`vf z`JzFK6Xc>XFK#67XQ(4_1V(gY_3z^K?YM_x_G>)5Fj@Q?kiKHw=A}thtGAMu9k)rB z%T;uN1r$@>LuA`SERIo}KK8@ET-4q~*nd83=0!yX-GzVt0hcGO$z-!5j41Km=;$&G zW2ZOSBHwB>>)b`y6~9@=ej*}!BZq+Zo(L-+WPLok;fMn_VFHZ>FG5qkm3;}EzV@bF zRb|Hfn(Lm2ckM9f=+4#1Fi7wnYy(+~)UT(e-MoQ9DEyFZobUZ13WUW9Ju<+Q7u>5y zGEEwm>_zP&WtAP$tn^`swSJRwG5J&Rxq#7|+<_hw+atI*CY>95rG2P~18XUVf1T(d zh4|AEfHXEo-+8A%L@uW|AwQK$of}d2j!En9ELD#?GokCb3~Liz7)^AIiWO5oUa8W6 zC_WcKN-x<_+U!pDljix3AEC6&0omgd6g{Q~DzXJ|YB6H2&YC7hPg7{WZ0ahy)sqtg z3QpHez;(P`(zfbsdPFw5`DqCHHaxP|Gz@4KFnmhuIS}G$=LQBQN0aRJT?Uc*5~&t`o$y9PfM2mc44xZo zUDYF}Y&$>t7c0Z1Z^}%RK?7&PC&ocMdlJ6zjHg=rJS&+;0`>_vfj+)NdM=Q})8LtQ zow%zUV^bF;G2@8p1v6Ab>EAaj6lCgEG>BZxO22Yix&Ck$sXES3RHCJMfdWxFvAq$M zJj!4LLl@}-8WObv<+~GI$f(Cjg!N*C!`b~-Y|8=ya$b62+*^x5rnnvrA& z_(t-Bpuli(B$4%TeVST7BU51>K9*ZTW0nq22-h_tO~|SSTZ(?=JaoD<&r?p`2gJvn zFB(A2Hh;^0guBETzFHb#YA%qm-6R{F^`cdS;rzA;3LPFl5_fa z^Oi!6=T+aRRdr|Jxj7AEU*sf!*ju+mkA~-u@$bmvsPUD!3cWN)K$VgmGw5?hl-9es zn`>c`QuS}r567a?jDWmXLYruZu`BFEkfLJ)uK1@ZHzA4p*fe1txT4am zhUiNqNd-dv7K%#s*Ebz-UO?L4vRx@Ne98>Q0H4e}O8JZ!Z3bO}z(}H^2>)HjTQk&6 z3*ZShS>cBL&b2yl!PD>q_BzXU$}Fx}5mEKy6&f<}^o%)t+Dj$#&2@Y-vSp?0hX+p8 z@ed?S(IB`K@qPKo)HHpMFsJdCUeCi7b8~ zpr_r@OSPnn(Q3zchjIGOg^w?RW+S3K7mp4_5{iQ}8P;kr5Np~apm809bS=FTR-I{U z?#Qb4maNS1IPHpp#GBR$V6FkXXKWX?R?Ana@NY11hXH$#;#UWt{c0cH_d=0tnjW`KASh68#qR5>_Y$-Ot`Bd}ZigC4WniC32h|E=LuATN&3ujW0{J{Kj`v zqx!*p@JO*AUMdXN}7(RILUIVxH!FzREyRQLF{>h zxykXs&vQHj^MO7eV}6ft0_4wWUyb|$wu-jd{ofHhIBu1@vPr{3w_&4j^I1RIAp7S& zPE{uhg}DHyi06PGr*ENaU2Z-M!VH1W4N72Qd|6xZ4bbbL+|hr@LG=wxvsLfWxK#2b z1t83Mh{ae%+q5ZbKGkwjoiqLFtbK4)Un^URNN(7}&k89&4%KkQj5dqr}wycZ!YRqU_W%;yi+#Ej2h&p=@xWS)?|uDZz*qf|Il)_ zzN`elbLbRP9lDHK5A(Zj0A&DlLJ_`R1_r}HR@q6bR3z9UGy1zVl==Id=Zdx7V5Ihv zUSKKE=^^XwZB&wZWhKoZ&4@1WMBYOfUX(k3xZcCutJXb`QYT6`#ev)^GX4?)nE04(ft9??Gp8j4v`Fg4dsSHG}SiIKZ-hr0SW9vjEPtCZmpZ{ZB>?xsA z>is(m+-0t2*LOCFWeG`q6Eix=&Ae*?E%JbRn_;OlrhpShsPvG@zjJ-JEmyXK zG;{38R2;-Apja0M6)kwPakrQw37aGZk-D;|na zy2GDh`4FYv&+BRHo~fck=WpH{c+W#@+3emIDJU|>eXeH+bx+_KHsZlPLq*2nE_V#b z`xQ5m1{=csrOQGNpM3wLLgo7)MFnz=eevZFaJ2rbA0C+`s(^wHv`*}KrmX2nM&5;R z3kNT^=;80;yq~Rb9xNHT{dfYns>{FlHCe~aEWot_EqOGzgb$Z0&q}$c4}oPiODrJO zn_=)Hd6?8C*pRSAeooO7ivoocVdFZVepVBDiguiN>OKPnFJ^L_AXr{k!ZM^VySbA4 z>)d|wp&6B5`{;@R*WP;nFMdS2lj?FDKqWP}QuOI97L!3Z$i6YyLX%_)FI3q4A#4O)mEg3!ISILSFn2*=*>#aCez)6ykYdD;`sh`w!zUP+75zI z5~w*=KHT!mE07-l!Lxh3a17dQ617RkrY|6%Le|uW(ix%G8*_+Nj3CkKh+H%(4mHCP z_C8*^g7G^^`0aK=8Bntwog5rW4rkJWjJqVQi!N$e5IMV&m{b2vC=l~xZ-wSYAL2dE zxF?-D$YmeF%>GBOB-_v5YOgonhxP;?g3ZW=t@3)vJ^Io4_=`_rheKW!hJ>L|VgC6N zK7A)2k2Qng+_A%Z3RI{+y9I{YiXE4p-OaC?ku@L+MVVx)IO|J0>zg?1njr624xV5r zN2TktR^}@X?APYFCSCMC(%e_CLZcRgr{dLkMpoG!Uf1f_VC@)gu2Gz}_x;*U`tLSx z&B2Dn|7zaWPNz0nDBT$91|4D{_rQ&)$l9ZC@cZ;Sc$@CXK@q)4na7#} z6%vR`fa~2Qy}aVwncV_#A5vW=4mtzeVRn=6dWWMBjlKz{L2~H_^!sN#R6?4woz-yO z$1%d$IpilU1rSSMrbj5bN?p(X{YK%2Qt-)zr4hmIoQ&#z%?TGhuR*QD*fg@cr1rs6 z*9IAf_W*pS{i})Md+*Bmda<$Fc&{8B$lOZi9?{r*^*-qDzY+8dM)yxB7WY3KBMs;~ zV+Yb5eT`*g9A)~AwVj)keSIqS(EY|qwH3(HueDh(^zYl8R}|+s_s`J9vTu#!O>!54 zzi@HI$xLoMyR)j|J~8MwmhqS1_2ezn!4p&;e+h%ESY5TBm7!#sma3k*db_Z}F|PcY z8lzZHoDy7ffkiOoHcoHN4m<5Uj!UfC%NTtgeJ^H!3eUEI@2NV7lld7_IkvAYU#+pJdA8!d4FAnSS#W2n3Sa_q9z#@0w`F z#Y>moUsI#@G=JQy7WUyyDSaKHD@|#xb<|CPp9El zm~|(yQ4TWF0+@XV;gwkl@khTeeM_9;!4KkRE`U zPShU2m4wJE9}fEd7OTs5eA0HmLx8uN$i1Cn^p-bl=uTnOk@=P|g?tR9SFi1wcKf9n z3Ws?EHae@QU<9Unl_zvgZ3f_7O#E@A7$={xfStXJD*NoksmShBbdXeygoT`7+u*Bj z75*y+e;=+p(+NcWf#)q}`RK5v7!8+z1^z;Nu4~F)_>{jCkIF6{RA?LhCT=~st~v0b zP1)W>^|_!x+h8iMw|n0AQ@?u-2cfc$f)>j~`zR zjkfP)WQ#ZyQ2+c3fmC~At0mC$=Ib_DSGCbr^IM`oTd&8FI;A=Zg{lv9Do*O;Q6pC3 zagHIQG?k90Uon)`R1{9%Ke4cdhu3PQ_o5t!O10uh?LK4G7W&S8fBsx}HaVIsRb~BR zCXs&5`=Wg0o@F*cBI9@I%&Gog zlOo$#FHB9N@v^+3z!%61_PbJ?&*uEMc-2jYZRHY%^6ekCo$cV` z?N0c*Emgu}z$>2KcsS3%o>vCzR8$QQzdRI4?qH=eE;8Pk^raT33u;!(rf9-7bfoyyG}jWvpva&d)0- zX~!S`HD(}7^UFxCJPHkjhlm}`%2jP~+^x?OIhzA6=>l-lPPM?<3*Bsz{SG^QeZ z7aih6aS|32d~Jh&zEz}4AAN-|2FT$@OD?8$wEMOn#$>HNJX6Sj)LL72Gj%QAdvu6; z2^U@M9+E=BOw+!a-aNsuD$eIOa{YvOL)Hwdlg(4+pM#qfzVIBq3ZsjaiWd=%$y|f+ zMk&&jI2h^U;8CIvR;=w&B)@Z12VcZH>o&84m<^UI~=CVjipfBXP zVRjwv>U(t8h4q1R_>t$*nQaCaw859Y%=dM7&wIVRC$O<9Hh>=-&3qt{z1vr!Z`l@u z@mmop+sy7#_^SMuQKe=OS>xhHjd%9q#i*WA_>KAL*k^k?@pp*1TNQzaKqIQa=JVi6 zgkKXF8Em!KsM*!rSY5>FcF*bxUqA}8ASLJ#TyqKc{Ef2ja({hMc3)w*dh25UNoE_d zYo{l0y1dTqE1Cqh9eQHox8670Qz%u{bX#pavMi$DNS{#2OeNf>x@=pjqH?~=UvbpA zJk^+FqxP70;Vn1ou;NSEV&7SYZ7-$V zJMP6~;fu-l*>^J@-fiOFv}UdJeR&6TvBr*IRp5F2``2r@YJxLoRBxmg2biPA&W!@l zF7)|034sbu2A&Ut{dCsTGfvdVk1T(Usdm+w=cD7h9LdA7&Ub0OjsAE0h0|)K@4uD4 zUn~t2iI5N3d*`VCjOAdsHzXxn1g)io$~P0hwNd)t+nNcC*Rnbj6D1js%|kL zB6E3$5{2*U8c_}ni9X(&gZMEW0avTy7t4Fp!JnmOdJm#GXABB0GmdQ#9RVfq9x;PY zuTFhF*y7Ta;|5W=duj%irkDhEF?Y1_73%zHUS6kq)0!zU*^#_AL-SH19a*>c*Wqc* zCL>l&SpM>j9IJdCU}-_NNz3E_nU#!L-1Ij=7PhJm!a6J$tiJwd9{}xfU~}ks8or+n zi0@`Nd3ah12}`_uTQjIHDv1jSO3gLCO}9{il}}VXL#tLrJEO@hlkG^cQ>g&AS@@*lOLIkI5`&|I4&1LYWdTcc#vn6m z%+GK6D;_iddQVod`K}_Tt`Tb+tWxcgSHfae;#0Zp<2vdpiGLRJnQ9QOod^j53)SSQ zgcNDj?d|Q9=(G|^qxqERg#2R$p|Sike|_+TGyWnM5yn40k?^ z_tH9^;=Ke9X28dk0}S}*3?%4vxGp4s8zyi`N9~(X!LCTaWA*}_0v8_vSR82@ff{l5 zp_$Q8>^)=M#)2c%tKvI^2wxr*Xtjc)T=XR7-)Mg%zOfg)2PQ&V4r824#HD6>RUt8L zj0mMrvoKrwGL8RemAfDWdhijP8WI2xP)_Md=hszZNnFW`ym&LWQtG5}t}q#5wBMJ{ zcNOpK5XS*wVP%5*3NFSgAOhUo&?*AYr7(wq*sy+`Qj_Hr8R`{W^j9b`>YJj84Rf-W<%EDmP9dv}<{^S~E3%L|6B+h^I-{*?(w1k}gg7La$$S9adB^B*#zj-*<-xPSka|1ax1@zmso;VLqNH-htfPb&BG?F|PEbT$nw~!c9Iyk4R7p|2f=;!Z6e+k z{jM>RlC49j${3V6BmUMpnt*UDuril*e9g%jg3X1SUrr(E6`%nO-0GE>U!cgM0a1|>uRH$0b%rRe8YO2 zCIp;y(6pY8?Uk2$9*FDxZCB`0KRcXoehzjll232)m8z8Z2>Nq4VyHOrF8b)3sk*)X z*89j~b-$T-qj?8-Ywmq*ccrRL)^C%4Bm zF+5{M51f#w?F;_nGO28az#8y2_3(HATo{ z<|+5A2Euldo&PNIRk!r9e$>m@IwC5LVhK~fhryJ zjG{u@&8Ao@e{No;6(@40Z0z`%_G?ir?(JEDl8SfLL$6hNeqD4}{Wn&$$szC25@|V7 zTU=9W$0zO9Rx3y0;iC%`m2m1mxx%m-la!5tE}>q+k94isScYCg=ut?8YC}Exs(fBs?WDzd5pL;(cv7sGm_plp zvf5)?UNKJfl5Z@;8dJPRC11ZwLYp!|i>n&6?8^_VX zKj7ks{#>|@UvXU!33qgT=oXbYwzDEdUUdC;%YgbR6VdB7exmmpXInn+t*C^iJY5d$ zoO|Q8U!QfX5)$}$zhA36pQvz`n~$g9MOo|IzOAM+y8?eEmv=mWMjJ z%BE*>Tv}P~hj=hyP62pZCx4!uEuRdSaQGved~Uum%ebLOq-~{!_?h>-?!A@6WSn0L z8t*dVjLf=r!wf4L;{0(LB2I!nmutYrlPW~T9G`u~m{+Ei22{e*hxkM8sUdadEmHjv zWX7RWjbC2fw~1Az^-kb#k~GSy(&4pLPil)1^iW&S;Py2RX6e+(t0{|5j4A$B#9||n z=6>Bonb4^Ie_Kz7>eOs)@#1cNKdJd^GJgA?EsKtFqg2B2|KyTE8N4RBALpN!z-Iu_ z(crtfD9pndprSEF)S=fc>GpNMid7=5uM%shVlZ{TL@Pt(YTI=j%c_YUeOWH-ktY00 z;w8F-SUuCSyh?%RvySi3@yVP9O8FvstYyYZT7?Am|4;Mjg+AOLsz8p1B4?-0EksZ1 z?Ej7mn2S{x)D^xQ-)5}`8nhT28*@SZi=mc>p!Vg~)h*DlEtY=ez-@CiZp&#@neMFG zgryO%&a&{bV@tMtZIh4E(D6TG0dy*|;w*Pv*|et(pUf@gWTPRxigtoPn+OjoLO)$z z1?Tb|ZEK-2tiJIO*LQQA;p- z(hZ*)o0UtSROE1K3#o>(sR~k)#JknRKeC$Xlvnlm?m3`4#r^y5!v`)Zs*qd9(6UgY zSkNpHX=+*o{-#A60@zIe>i|Sut<3WH6=<6a&0=0zj@%<`AYJuMW{HoyUTncz!`QyL zZQVETC6IKBv-I2{EE8<{+0@cfhmLb$Umub|huiC_6!_rCD5qGzL z=dIOuXE#ZOKVjKGpu)s1B2qpe2vnbo`Qr5L$-S<^E~0#{OMPc4JkI@EUGn*(8jSKv zN=juDfeBfK%I(iM?u-I4*g{5u6Q0d~P&tiU8XXo+i13WRj zYruQ5uLt+2lzfqP34j)fkYsDB^ ziAe$Ij7>4-+tbt2D=S@UN@%Xyou!NUKe1RuM2KmgPD9B3u~;m}l`D9JOD#5*2Kt*u zm9B4(oz22x6+mk9(`v7YWJ~1Faw{B0ZupQ`p4$(RK{pti0;XtXU>(<#l??>GgJQ2$ z8*?)=D;SIgDk1uLjbE_{Q6S|5Ey;LLVnxUzVJ*%e*7t z{d%4^uas!!56f2S6q*&( z9yLw5Dx$&MLZR1m#dMJ9$=(0)W14e^;-3#A<$!eg*X#f77a;!l!RhSZn;lT_^c@~zk(|Hf9*lQ$p=z6 zhJ7FT{X&bmRb3>In{u^}+uCoOVHbWI||90pZ;ad7`uOSk=>O9olI<;FU zpkBj4Ss&pX1YW%E?(TFLpjY>+^?$AOvjzuYi}q0sU)Of=^^Y^FxYw_B;ossxm4Hcs znVA{%0HUEKpk-{F7ew!6^^)_SKV=Mm3Ovh7SuH^Yy;8m4t41D_R<~JIbVsrH9ieFp#=FQXp zSuhnGG=B!wRNpVFUd#%Ojg3R4Fj)M=e{S)<7e2phYIZGtU@i*gqN>-22bwt^ytqj9 zQ~}*$!D4o}y8gf1tkZm?o=0xWkVz|0J=L(Iic!h@$~E)Df;!lX=FDm(Go$;D*Ha9I zOqgbL)Rje#Gaf*^XjcV3pDu3Opb$>Ak#VJG37a9Kj!ZI}gF`VvZ zgg7NXb-60z+Zz1%QK1FZDaF;pVA#_$GZoQ0e*R_lJ`yH7GsVTjDg(CG)>CVP5BfBN zKrsGA);}Gc|9a8Imyg#~j6KjPsY%J3IW95v=ZYa=^#eD==yd(x$XzOTms|wqJvTo+ z+tK)5FrvPg;X$f$D~;O4eEZ9rUG-!`I&fU^0)OZZFT_)Kw4P8<)e zb*oTxsK{sN4-|hX_$sT*nNIwfrotTlY?mB)E~{Ngr}CFqsYzb*K8(xF*O{BHUB$Kc zuP5p7f2z#&Cbw(}n|=f;+ln@Xpi27zK|1f*f`op%p$br`=8fU^-CDVr`^zcuEO{Qj zC|+qHm)|c@;#H}*WLU_mcZQF&KBPw+bLXn`)w-~b-IupN@^rhm2!F1LD=pAH+dwKU zn=`t8qV4?L{J__jkV)E_JnFz|r*#QT50+Mnj{3UhHUH(NX<4Q1rb}MHsSu1)-xjz< zrk`c6pPt8>?(;%`Mtv^RgMa?Li#LAPNe0fh9GzPPqSvjXg8|u(qGqXSn5B+j0v%^L zATE^t+eMlFxp=&eBipuPzJe+eU%5J8O6>XQtFh5}QaLwZ_t6dLo66zS}FyDLL!_gKlDQtpz%F?iW(VdUJ%yhT{>&qpYmg$A7-&ANN< z_`p3&^xLvlK)6mN49$P};XvJR?ySO!sGcv_8kx$O!G6sVnj84ijiAb^Iz`;!gd3PA zHtG*zPwW^wXf6e;i}d^t)G@|pX0jkm zSiL6x1gCxKP~<33UkJRNHGJBNETR7}W+MkvK_#%|Rn{^fe}`AWA{;v;H{;H4QKY~^ zcg>QwEYWtP#|*BRW^ZrV>b);Cnw85Z@}`_-N!G7UNii51j^x!uWEsTla+n{*MSRUi zH=D3yY84w(GW`hU)M({utBG)x9`@xexBR+8uqnuDK*c?Tt9q-#tGzH5D3pvZz#2xlSlc=B`M0bDolUY#3Qc*cz>6O$Ol z%6QKVEd`|=aZWm8fX*BfMMcbL-BR`1nPD$ieddxeZOQGIET>~XRglD~Dte=Va-2+F zg)!#jGo-{TrGni-LGO_rm287KEKqrdWOEB&cBP1iUbuuJ8Lh!rVaK3%KcRFt`vxDX z&pzp&ap!H7{vVJF@GFOk=Gh0qo2!jv@UyZ{IqF$uz%o-wlK~4uRA-u}E9_lv+{$w>619}Fp<}=EHThwQ`BK9X zXg71{mKn&VGw^*%I{z+7_SM*_fr8zoXXKvkE9o{Ya-(t}@&SR8iZTD?R5@F$H?VRYo{`K1D zYZ`Of;)J?oxAvvqD|QL({NlyZ*2EtfgCxq-x;9Bdd%Ds#7A&%2Nbd&7G9HiYnu=g5 zDJ~{;VV1f>KAZi0zjOcJ?7V66YeGtCpUS+d(kx0~)$trmB$=mYu}yqY#`OGPH-c{< zgOy?zZTFwK^4BaqVyO~*}nh#<%HnZJ1bSn)C!fH)?2MSrClD?@qPOWoE&d=aS#5daH>8t$+9AN+T z=JuQOq%+`AS7u}BmX6mgjzZCe%Y;Xzj4L5ctFolQ?^4H04jS-TWBLj%~H8X52D zA!6CoBN~pebvSnWqPSr4Kh@yq*6_(#EPswuIZbwdtZ%0@wT7zRnRqO|K%GiXzjpiC z`N(%5n^Wdss zg`(qUxxsQ{Q#-nnc2XZ*vvjVkj_;&s^J;SAV62zm0s2X3RJdPrimWm}FP8B1ST)XQ zHNDC_$)^9A+Iwm2UzYwspBhbuDta|vV=GKbUeaNm$I2)Y^lMAa;X1So8%#3%eN3p+ z-gUez)t>E5(s5isZ8xzvzFX&sNq-m8-E&UW=yE}CoH-GNWn}XLQsQhkD7`f>Pl!cp z7y(E^>wVYfd~cQv^QIRgqRm)B4y&!~+uMAcWYP#Ks z0Cl>4)Ig3nnS)I2t19EdhoY+HikedNA}a;KW4$Ss9C|G*D$pW~Ftqe}Qhnq>SNbf! zNKRF^9bgkKU9Z1!sQw-NbsheG4vOU|rUM01J5?vi*}mD{8mdj4L+w4lwV%#wjD5i^ z_g0JfEG=mg?Iv?!rX}m(*-s`r+`qEBG^rwPYKmC>I?VU`zm1c~cvTJOh2?dMiE740 zq^j?{-mhDVr^paDWyU;?5jFR^J#fl9_ifjoIsAH8zQ6Ukf28d%S<$1?1z{$#uYFCs zz1lWz1e_r`VOC5L=GE8s*PmWoHastt!2pGD7o4R?UQ#ksP-RGr4dD9c+WdBsR1T#* zlG^EhyX(Oh>@eM=ORf9kl@Tvha9&e}!8f*WJu|eiKXM?A`9u<-rL|g4R(l=n(~oyGFX_WsjxlYSaoF&FEvri=3$pbNW$gYl9BEGSKG@Oul65 z^fH7yQ*jP>j;Dbhx;)Wqn*G?LxA%zzAWlP)6wr0MWnqyC?5#BC&m(2L!a`nPFuwIb zYQ`&*C0qe&cinUzgDLsGbGFjUSsyN7y4bO9fiyg^|JRbjUGu@4sYtZk*}HqEg0zK#kA(0K6I&u>X@wuyLyqqgIJ@*=cWYub1$4~ zXXO8EydzG1xUYssY?7qGz><{N<2N7CpDQYA1a9)`<aT^H-m+J1fa7@l6l4AWty z6(?xl$(di3TDvpOjYyIzvxHl;CnA!5x-Q>a&PANyi#jQPyO@Sk^8R7zC-;sph5pNr z1OK!0VBRLyJamm6b$p(Pm%Z!ItzCE#mq?p51Wy~hu*feWX5?-=L(9TQg|hOOG5Fjq z*oD0k5d3;(+eq5+OuSolCno6(MQDY-dz%^ks!N`1gw-cZ)A%~puazYD7)4;bg&fZL zyGi=H455>CU zXVCgQ6PWfW3sfzcvlN@tewC*z%ha7uqpYCa=kj#yV;sZ-PwezYix(c7a8Qx2+|TWM zfxA2J_dMfkpaTDwTdRST!CU$fiTenI9V640Nq35UkxS6S8V(pV0|AkJg~z15oe7Wx zWC;67B1DwT2~oNC`N73kSlc3;HJ7xxTNn_uV*Zi)B@({9!mnm>zf>|D$kky=$HtkAd0@+5OsIzKaU7e<4AgtiH3Iuj5It#NNt6T4`I* z9YCqn^q8-bU(WBPlA5!3;-r~zzMb`FNVB{(S*)|si8_o=G>ZkM+2(4)fXtCZ_^lFq z@{uL>+oJec?PPb4JxejYoWcp!=|8UZ4Lro8M+~2!daUG#E}QM5UuWkq^wO-IJ{BzbMk)d@BRV-zx5MvfJAIJA$> z46nB*U38zV2?n4a-MUz*R@VMn&*+DX8EfvGgN9uTqUg@TehU%G=0u_KFTH)IMO)pc zNuG+!bUFAu^bIB=jdNHJQ@qJfgWp=6DY0y)SJ1e1>qV&K?tQ(>c6LJQpQVY@@bZS7 zG}KenLQ>!U<)j=gr@1OT5_?V6@EO=WxVx$8#Vxz5WJGt>YY$_+ z=gZ{)fuvvURTBL4saQO?hz8eIf!@hlAl&}_XWkNUb$;`?yZdK8oFuKblELJdmV$%9 zM#MsAgdwJ|la{WZrr_l}6+Cf-!;9@VOI`xGhqPW}9aD6?qasKl=N&kvX74RFu1tHf zD1pedSlH~sD)R1IUn%_ATq>t>=6pek8Y$g~0SEueoTI-1XeXpefMDJOER4b-uQz{v zB?~OEuq@Gb=)*)BtXb^o1*^ zB3XhhSDWqBOn2M=7FV9Ab&GA2Oc^~tL{_K%8mph!E;;ooUo+Vv`rR`W!H+cRjLv|c zs%F21I9R#Dapee+#vYdwL;(_FuyH@?C}x0uw(DE{Q@73W=$T>f1=RKH*LQYMIz|~L zJKxR%QEN)ga>9aDQv^Hl(ZdKwBBJ*hqs(Fi{Opei)CU_%MEcOeM18>_zq{$mcCn4^ zV~)70AUj7L`Mul;e$B{)whXrPe4FY}^~m`mqs(wV@&4M~q}&IZtFV6_oZz?%!t(uP z;#vwPid)^AIHS^%b=RukjwrJLNi-4B@%Tm_(ouCX2r~BF(hE8ITQ)Yh`Ko1A!j}`0 zVL%=XrUitkF_NI=OSj7?`Dq+^0z|fv>Z!RCL;dYHKRZEQR=B&Jv`~9pk)x4{ zQD&Atdg9&&KM)`0-NqD?yOt9fzUCpPI)z6?a+3Cs!C*CyQfFyZYS9yByr8Q=pRAS+X>1h>4n8;H3Jo zf@LXU%yok#jKNcLjE(Jf>1lGMYa^yPio#)N{S)sIVH$*fSy4fu0|^yD_b=#jtI%Hn zFwqChz`un`q4yqjBeWwb=5zsQvf*38Hta3g=in z*L^OjO^JHC=Y?CYH$Vtfjp=8kS!K?-U zy=_owdKL;}Bp^dji(HBmuddNbU3lCP%&PvY(e>+}AuPM6sZso}x=tOa)*qU9){fCD1Jv-`yCk7>_b2Hfub!FI){OhA3TK;m-&M zH1Y}qu|M}7ep`Hxua~#&AwG)K6Sp}*D{#>c2mLa{V_cTuYgr(@(?zyrNuIT0rM34#e zJds{K!`_CRfRND4+1icA4wZAID#6D(}C{U?Txx(#Sw z>fD!iNdNP|eWiMOuHUdDInv(ydhQS^O+IJhoU-P!k*K|FO4L{ozkp$;df)MKrC#<< zjoumW0bY_w!ICQh95IX#)GAxJVBG&DzgUaZ=Qn_u=mr#*@)sy*Rqxll)ph$QWdj-g97Y7Z$u9{Ebtyx04$0y zt@edKCQFy2qNs*EClafqzS(|`D1E^x{pPd=+E zGp>;<3g;N_uSL~V$C>^4sJar{mm2Xzjy=~*b%580m5u#Mwm-#9jsHvuE_a<+IgDH+ z^75c=Xl5e;rv2wv*P)H$v$FvaCupkzA-?O!wglCQm!xTsf@2V%5u%w=YS)+g{kuiE zAs3xX%hif+!JMwEcpXmmsWAWIgS!>LgvB*xM;dSzJ2y*ISLWVm-Bo|ydnV(i>-7h6 zE_V1PH~&1RaX~CEi^9^HQ*Pt5dVfS83>|4;TB7O`MTwrLGd^Whv6f5VdUCes5Rv#% z4ILBT62x~Uxb^5ExDJq~Ngt~8m_aUDqgf|-E&(mKP->)FkOv}(8|xNDdXReBSXq5) zisUSD9#t604x8#`OXhbr z<>p@0(yOit!8g~!9q zZEd0cbkAgi3Qiq-=cJjkj!4Y)371Q$Uv8d+i}N)qRf;x|!y3w2Ct1X-VLxX@$=KgU zQJj~GLE9-N_=I#Ss%rIjizvo9$6s!oIt9`IvG+8FrgmuKAIESvAD9N;J(Ilps* zMEZ2`{$pnw!ZJ3e3G`nSrr;?><)-ZUlF=lXX0uY#*+&5AIyxyBCOQ%CQf$}9i)arc zojfie{Wfy#kP-d%7o!)xU}T|pCcichspTLp4QvMpx?|v&Q|K--wsQL|lH?jyYEwOx zL8O#V*9=d&x_h_HxpRt(i`S&k@#w!E8glF>PPY9woJ?GuG~21J4@<`hGs`vvTe)YY z+5&FEj@rugIBSxmwJG^I&AMjk_WjID|1I3;4Lg6x*R zJ>~weT??{gC8lp5N(}u(UrJ~D)aoqES&=38MD#SB4SwI=cc$kgF1Xzv-COMv+d0q+Uiom| z(W*X}(#ZI3ac-O~Z;UIrz-a(?mLdXTpdLoyv->10(|l{z)Q7!ht{B~`rwoRZ z1K3Xtp!PD<)&V5qC%XO%B@*Ia`ESJqC^RrWg>+bKfCE290&S5z89w63@2}{VXiV5r z1;lX0&NC9Z{rV{h0w;9R6;M*FHYPrM7;79F7360>Nd#d@L}QWux<5xuTKBip5fPn) zVBw(CKaB%T_-)sM9!_c<9v`%G+m^H6SXB@pFNk{(ASL*wIV!FCvtoaO3% z_n0IyPHb&tvB090Cc*Q2%#v3%;*VCkHPssJ1SvLr$&YFT5$6WHn0&(VJoJ zp40nT-Dw`)Q|^)$i^vSP=4@4IAIl{?Wi?dc8VWsEg{4y3@hD%&{`KF= z&zz6u8=rc$r}<*KdqBs%WlbwXMMTwFbAm0`EL4M)fwQRfs{Hf&u6OpabEY|>W29Ks zXW?jmk9ot^1Tl|S)8|k~0F5XGgLVl*tAASU+EupOKeKIeXy$Eg?Nz7WU8_kBD#q5G z@~R0_;nZ&JG#BIUE{lp!ULUZyf#UaKKF6VzrfNU1H^mm2t*%mCdG* zTW*16pkq%zpPCe5s3e_pk(<%?`mN0J|nA!de-N|F>xk^9!mYQtK-p}M%&7luN zRgY*6UUksF=A#(rm1-9=x@EuxOcc@=B9j3;ihQ-8)Ru%)zp9s-I2Bb?=zhH!Imv=g zq4d+V^WOM1C?ts}V8cU2UmrDDTvSwSrhR$o5Rnw?QH|fdHogXx+-)FN(6F|~W~wC( z!=536W*O1OqFRW67Q8}$pUJAL=6Ln%QZ2EO?MlCHhxV*t-T^d(pp^mvv7&NVnIgas zklrW(tI7%Q3O8plKQ1l~x)p==2DKxSXPAGzRi#Z_i5-FKih&NUFqROTreTz~ermeT z#1$7>xgndOd`*$%saqHaxO(x6VTzxiEQ>|dV}4CWQHR^a`fhCp8hPC_!V1n~H4AK9 z0|l)MNOdm_un7N1U=^A?J5oKoGbY3r1=+M98TgoJyv3< zte_e>sToN!bV0TvMD;IFDUOBxXmN5KZSP}jc zTft;ufJ%$Xy@D@Xtt>8kdnN9jelneYs!D8bq4ecO%c$j#ni1j)H_Th~FBReaK5pMU zLSJHMkeijS7j-5&o{y?fdE~cu`*6ayx{;qq+3R>#O7eYGnxn3SGl9aA?ZR+1>=#`> z0*hY!R_`^L60|Z~)IOaV=Ros1_}~=sT#8)qn8|+9pQ zhyi109zwc&07X zg@a=ti!=s$C5E!iXOvdkdWdE21kMsqx!U)h^3A+kEcbn$OOUnEZRHN~OnGTk&GBaE zn?OD64UYm_*w$FMnwemn8|ZMXeeQ86YaVyT*3{Ir*B@+^so8HbsYpm@Y^Z}15}b>a zq>WSv4Am%a`^0jk+Z|l;+-l=#3l8t$$4#+CT0JYX7ju9>GUO<15=g*8Fkjf`wcZwK z%EOk8@5j75STIi&w0dpbwUOPZuDVwi>jM?M2eGaX1g)P)z`s;hOR$R@=&|~t`j)xW zn>Nl#qXc2+ADkji2aqiStOf8Ke8aSbbD-@tw!Fy?&9U9?1G=7oi%`N4oU{G()K+PA zovcy|KNil!%$dbkqN21CV}Y$2m{C#CE)SJ3f;W$t`KW1D_`|{+Dj%j7N~}6A5g=IX z0m;f-q>2+{=e<7U(F|We3t#)J&V!%V&gWmOSb{)qQDRH5b?KVT6TvAa`)KJG4*6;^ zKFTv_AJ+Kn^zM+0IG8dEp@JQe1a$H$j0ensYS|~GBIES+ovJ-ji=4?Vn=YS73~#mQ zq5!P&AFq50-W1et7v2emD zdF)X=()$u>II5T7#Z zdA3nxlqseq>LL#L&Mn{x(JKpky9+QZh4x!XW0(Q~>JWy#<)l5@Jugt~*@E)AWP<zvQDZI_R|4DD0 zp4_~Pl=-S(=V|5pRmdeu3Hv#qr;BeUP4U}nmtGrMyWL8QnJu5-^qxQFJXSmsi>99#`zWqKiE#FhvkagelCI6Mn@!I7M=9|yBTlZ~gL<-TyFPh!baw%3)DpqhQ z(bTT5m}9h`Hbm}hOhdwzr;PS&O+dI2^uq6$hF`6Tii!$xP+gFnkacJJ?%+=*4ioj7 zk=*E>xz49EiX|n51$E`@xQ=uE>Gl1oq?;9Uby4Ei;mm8T4dyo`*3Ae(-24Ne5CVdN z(}~{m9UxYi4EN zci83uxDl{iG$s0{bOuknVgmF!^(i zJ!S9*@|qrWde(OvAn^$QZWWELFhIAPX>6S&0Z(w)oYp%E9MeS{jJhYIZahHhb9t~# zHak;rYJ{q@UP__jYi`|~ii*VtPH;08#Su1#Wl_AL&wO?Z8I zb^2L-G4y$&fHJ(9AsXAqFzqCvNuQ9}3eJl|!?%o}i|&f_+!29yP0_&6vW7SEf6jpQ ztgxj>;R*_F!CaapgqWuIgEcyR<{_1a%J*cwTgKM*mJ!Zj4YD}Y&E*nlH~$TPwZ*U^2eL`{J86u4=*28#7uLZJBAW3{O^Pvnvj;hRayx)hO6+{GQMm&do&Ddr z;{~890BaLP8AN89mJx&jL+d?!tVwh+yU=`$!U3VIA%v-4aBF0aoGx% ztL#59(Op%ya2@}VR|be3VD%Q`E{{&5O2})68)N-&&12v>L@hZYN|Ep#WhlqiqkZ5w zY00yNRm_zRAc~NP>^GA3y1hRsN2>iZVo5UwAhnKQ)D%3nes!AShqKy)MVvXuwhY0j zh2rUBD&f1gdjVD%-uDI=X%$HMe>?@tPAvszs$FugDzE%I&CtCDy8VQ^V)7D_m~vdc zd~M84$Jo=nHu#{rK&=W`0!O<%^&;-{dm`ub`P0+^h8%70oAQ^oPpJ+!>{3~21m~G2 zzA2AW(u&m(o7&;d%y37cW^9pfKLlyjdO z+!gQh)65yT{pA7^m+4VaQMdDs$}79L%;-;YMccmQJS0o;Pt)U*7!-*9hP_gUGN_dp zxNScC1$zy?dDqLsI|K2tI+A@|DRYNBZ92hu^Qpu|I`X@Fx2G2?o41On^(}L2HT}tW zV*K7A>Bk3@P$8(_Tf-3LX}#!!4jL`h-+k`vnWe$B?9xe-0Zz9&_iBs@X3uG^Jy4ya zC7v>~?~2K6cP%a`TQxoFNG7P4<4jkr3&sIGlci~1sC#1TE1Uw-Y`77qY;7`)pR;?6 z&En0yZs%l6%Bpb+1KF{nw0w=9F3)ve>}?> znb2cR`3b`N;`HVBKM`c+;=+ISN$K*=VPqk8_3bM}o$DxN-mLg76&+vRB(Zo6?4Ww% zV>#0Ab+9h#tG~kRnlB0Vk&qP@a;9gmgtm3FWU!r)TnJYIk_`VYpH>vM2CA~w`)~DG zaUjAB_2H@|at)l@g!NcRxEo<*btIC6gA?y_J)E1Z^2U~$2SBz%dGNWWc2k03h$j4rSr!aH!2(tV zPHhtWkOpU+(Sa|U%=omch_52NoF3WBzH_nG&P13&fSJ9-H25$|%xjhtB4t399mfE( zbHLS9*aZmop}<9`c>t5qOsA$iBpLgl4VeqoaNGI8@~z3V=Yf?6)aIYa{8v@+Irndp z+3sDe7p*n^8m=NBW76r_yMuvRX8+z$C z0tYB~A<>oF;Jt>a$`Jw=F#8}c#;`KGG~CF>qgV-}j`2;|jL8MjTSaZvvb`WuSZs3}&iuQ_fY`u>fQB zt$r`#UH*io*6X49uUv$*yI(yi{U zPv&uxW1bb2UYQcaRPUby4tsSH#X6H!6=6)MY5jYA?X8u2R?HZLp0I}ff%dWyKLd>! zFF@#Hr7uuEtuHoxdl=u{Z3Vy?(fQY-Tk2~cS3BewrXZ+)6e{wa8oHo_0jKEw7~suq z_HWke%iHbP&AM84svoWnrloB5J$n6O7bO+dbhF8BA$V|+O`jU_IY3a>tXD4}fV-I8t=`(uzoe%BX9X6W zZpDv-;*88=?&?(CdAcFt>l+cKMnuS9L#^6x2G(;Ot*1Q1O& z3?*9c?reHNcWm6bd}XPb+k-frX$s*5K)Nl}P!v9*ZRh{~&Q+PT4ha+QHw%92V!j|9 zxxeG#Y7(BwO$R}g_zxXBoz>u=68}0*jhy9$ji@o#@x(2%@hdfoJu7zN%e(GFJF_RJgH&P7=n_EQ@k>9D2svi#lVmV*Cw@csE2Xk|vWIPRV@Pf{RBjasi z3*AtE>dTjijf-0~*;_S%0vjHX1s9)VMxBY?etGL*PA5F!BpeaZJ&}7bJSjZuso0C% zdKQGQ@vBp5Uq-H*B42qi&+Lp`Qqd%;$$*Z=~O=SJRv8go5wxe)Ja5~i+E z=rW-SLfQ#V?|v6A6^Ky=GKcE^Gp@x}c6K3-Y(j3+-O{BhYL0|M8-R5PM>=|Mx%P=P z0jiZGCJ^l`&iE~^mu&3~4|b=_c{ZqZ92qfW zs&xAjH4L{?Y5@|)bAcE4L!0iq1G<-4$_8?ai;E@B9{4bFZdv|R!ueh7w9sy{6sk8n zk4a5fW3;Wp_*_vty3vQn6atv7=M&m9eFAj(mCj#Dhg_x(>$QBpo=XQ$Hl)NB#=*hS z3C+?Hj|F2eqElh^Bc2mku&GqL8 zgLzCe_L^dbqv}~fSQ}V{blVWFH5wU#!}s)X(2~O_yt#$NG!%7#G-NCJ%)!RWCP~@H z*bMtAzKG-c0N9v1V;r)&8|!#6yZbN(%fcovta1Xo$(~k zgu?HaBLOrwQ!_H8oszxxBhx2Ee&_T1S$@K5Ah)ezUeU<@n>wvj|CgY~`n0XbTKm1hw$=NcIiF)22(sT$;h!~w`uYH%>I zLg$~qpI(6(dZd7QMEmmoPEmWNUe8BjF`Hp8<*%93Tm!uVAB*h(j0fxUhSQST zP|(Xzinalq64(ivusi#4&YO=X7x{5(2^g*IHci0MiVE?>0G(zu%IUkCV$6N5r1{&_ zf%NxBH3`m}732+SV$Kig32jq~B?jBw+_2_AZMc+B7-B|FW;4+0FBhP62!<2%OCR zeE_NewCf&W-4)rrXO$SX&hag1V4!& z8>24kosyIQU;(r_k>dCfodcW)wKGg9;QU4@Y_^JLgc4wCihZFtJJP#}Qc>~V>b4)W zKYI79GmJ;N7X=r`cpP7dd@-HZ3xLM7tdO;+1#^fCZ5zo%Q!qs_bB@O8#+G$fCD1}XhqPvS}0ToCdp49I{eW_?s`F`wmmt3e~T53 zWS~A|%JkzGkw8vfv&-|m{?zBq(<^{US7t*JFlyi}u-R||1MQ{twnE$p21+EZkG5^) z#milx|6#toCiohxp>bDUMbt)tS+>k`!=js(v;y~}7Le^pf$edtBNIZXVGs*d2XWM8k)YAAvoZ3{ZvaF zCG(t=?4#`s<*sPI)f++SH^yJb-iBrl*Y=PtRIlovi-g;IO>glD3` zu&u2}`Pq$UgBhA#SZ2x>l5!avEXn#2LSUX^iG!%HjQgRE0Rb>hcW7=K=1G)4$Qf4CL7|mJp=4qvTf94r2Sq z;#~>j5G=Fxie#5e@kfj9c zB>z4CGbDF=$%L6bcF2G4HKOD51~qF^s_J!`04cQMWvpMTBqXsJ3gbgNpIhE3i?0)k zA{P|Tt!yJc&k6{DsGR_d2veAvgxIrZNPrN;4GrD?@yoB}k-3q2zYW5}DY8JZb}Ji) zv)2SXQ3Kx=evoX+kCOkn+BodxTUb`pRStus2D;-~_Y##f2j1ZtMWzaw^#W7$;Wo*m zF|G-?k^8!>aXN0E5{)t0J6*LIgO8(Or4md6;=B2|6|4CR*@W0DK%KriwrYtCqB04< zA2qcCkIccz*#$8PD}o=x;c)2Q8I%X-9ze|_qoUTb<%!bd_$RwUPvoJ??CxS*3X*k7 zvJ)f9W;InR7oS``ttgXv6=tDN9LX$=QAVl281TT7=e*SYPqvM!E>?!J!AnN$G!u%b z>$1Tx65bSJ*F}-&RaU3LHdLVBB3i?y1H;60f|2tI_D@YpuK4-BDypcvSJarU&}VJb zmc!8^AKBM(Dgf4cq?L!X#LhSa2Z^>v?-tjr@^Dmyc0eQBZ{dEJ6}!^KfVz}nnE6bH~-|99~$L5H3(}b zjN>AmUX`ki+Q7^#30_j^*4y+NHjWik1(=_fB)oB9F0WflIHp`2#|%ZM1ftDnhm(55t&pYJ&ZBl&vm4r%<%);frBH`LTDm>~rn9 zdzwC|PVj2Za7Eml9;;rq7;jwd}bDw&M01GgqNjEQ!f>fA0!3yr)?u zH=jVSm|g+hXQzGr`UALXqjNA4h>dN=i~l--34v`P-Z22Z%g{ARPNm$ zb&7uRDodG|Q{lGWnJAeQcM}F-`YER>!sgnD$84QV2`4kN%#=0OIWvQu@e1l(o~wCf ze3NqqCwCoomK1}w3%ws^QK)OqOVK7Ko%RnsZt1hTO9in2ZP+e}zr#_&guO zlkK})17@@?nM>_e+SZz(=f6$sZ%UsuTK_ztzo_RKG%6{wexCrzW}1yT<-b|&!`Q{q zhU|(8eM(A74mP&1ZW{UViHU45L(&6Ca|y{t{eC=Rm;?@}xd19(;Y(rfO+V^^LzaPJ7{LT&C zP!WJc_x1F!w{}55nadFubfMZcwM|Ro?6HSfX%o{pn8QJWFNTPo_s?ULJCS&EM0EAz z?dIEm>UZls zAb4I4mSoe_iTbf>Eq6JrH4lk;nuF?5!(htQqW5;)?DK?$T{s)FDE!S_z?RU$ZN>9v}(7wah}`HiG}bh zKG)CguvE_fNX%ZEemOnpSAR;kp_NHmA-JiZD;XZ%U}t-+!Jt;Pgt|$gl&$Nwc4pn> z=piNyUcCaJUd$xjNF`0~qWoP~7ntj1g+FhJoC@{_d@s41Fh)0B&Hp%K#NL*SPUj^1vTn-GISo38sFp-$9I59ud;5(Zgsvgu`e1`O(cOGQ- zZA!)UMmZQ``HUKj@`6xpxM`AG&>o)+tyTo`I@NM~wq2Pr);A&-ig`Gz298(Qhv?cw z2&I~BsLpOj?We=rerZoD82lycB$0m_5?>#;Sb;_`pt{Ut%4sQ=(&$9H?z$t62 z&-)d9sxqmAQ&Tn3ZM%ff6g0i!06r996El}}HHfq2{kl)DjV@v@p(!;(DWOSeRBy~9 zxH$T_xA??MeU4e5Jt_B@h8aztQ{n5!m0vzT+!+2CrJ!Hl{(^buZr~7@>_r+e&^MY5 z+_4>I;O;<0qQ4OW=k*^d!;t~|yvTVn7TkBk#A(g{X>s^#n;l!0&Xf;$KQ=<_oDA@& zIUtQvwtP+>Tv~wl+HFBKoBYNm9gG+LnRfxwqlDKGnX_ht{Z~k@nHm#u7YMLNfvc+% zuxSHw8MD3^a(_k6k6ACQ%TyqR^ij(;ceB%#@GKbY%yPEfx_h{6}!TO8DDs8_*@c9pnz1+cUzFh0O z-}ySFiWw>bo_jg~Y3n)%I>s-JCC%kx65WYRP^PGwCZvy&!5S$k$TottGiFF%G)>qx zd!80x>Uxq0j(rpZ(0PO5=?_L=B*T6f)Pr@(Mm13(*AWytHFB#Z1abN?Ao;{JA2UBx zl03RwO+L%DQaul*2v6?|CFcWGn)Wp@5w3iqSWK#9ez?iX9Fmpn?Jd{*V|lZ9=>|-y z9^BUx@o)s+s2dOg9{^)R@r`XZ3CVQVxbC!joBl6OqHV@mXhC%xiTlZ7%Q0<>Bf#}) z_htY^Z)zj;{C$b#3e$f*hyfPH6qf)nKOTS*`(GVNP}X%?E{jfyx~{9RU|y$p*U=oD z$j1G$CP#=G!F4S76*M(6x!w;BmA2Cg{5Q=%H-Fhkm?iaVdB)qz-3K|diCmI4=6QXZjXweZ6Sm$cIQTe$$1v+ecQZzt)A(f9Qy(V}{xu70q@qje3Hh@= z@W7VEp*~TONSAuNc1=uwEcn)AQLf>cd(`Usw#@>no6P(DQ(%pYhX>*5bfT6<_b-)N z&puW~@_3-r?xh&*OFFH`9y`t~pB0>HK-d`CX)YGk=eP*}j1mCh0K6hDF^nQEV!952 z^8Jm!`BHPyz}ZwdG*+rzL=}#wOvW_48 z)#fc;2LNpj|6&)gpW?kmgHye|qE>G!w+d1Te0O{0#5G?gKv$S#;mx3z@q`m(dfauw0ltZqWuoOO^^$-3KE=1y^cU7+X=*G3 z-uKYwS2H|X=i}Scj;{W5cQKxPr;Dtcl9et#t5o0rq33`klvoj}o4AW-Op5xG zd_IHy4-?y98#?IUmF&&}+aB7Qof+Bf2zBYzgvLl;mH7vyjvy~Yc2$YW?+Jrp?gX1> z9Q}1JN`I&y_@~7s&%n$*XT7k#YL;}u61}fUohDo9OQCT@7Vb2MTcJ6X>7QoCDxgy0 z_4P$s;q|RhwJIG|i%enB_kspZuJy&+q$0ID(ku%)>Q~!=K<(_#SEb#vNFe_~{?`hRpe8!Jv5F+rgh$#^D+LXmu+FhzsSlU0hthL@h{=xn6|R5N zH}^FTABdaft?;mXl7n&l5`tUt_!Zi;;very&6j^WBuW{&0CSv4W|y`%WZu?cKadw6 z?%>@I79-oFw0T{M9V}xD;UQV9ZL+>Yz&hmGszB(Y}j8azG%4g-72kxOA zX<%7WMI9Rft2VR)lpzV=31YF>R%E5BVxd2I!$r-PKa%oEESnijGeE>LH;4NK>pdQG z9*1Epik8W0ikZKakRY!Ka1o`s%s0^Jwp4%LpmTt}$Mb2GAL^Pgt&1eCZ_+Ps2x(SL zr6CCc$RF2Her#c82eju_ zWQFWsZZ67wN>i*G%~p!}kuk)x+JN;JpasJIOAFKkVJ+lDeOp^UpKXO(yz1`()LfZ= z!ECI9;~7_vMc?NaYi}oTfcI!i-@LMdT-}A64ScvR#bwd^AC?(mgc~4q0 zfIU*(-SRc+`*=PNKM+qABcq-I1HH)mU58MZ79g(Kk+xC{eRq1sKMUEy6o5@#klrI# zv{=;e!%dUPDRoPZezY>y-mvb{D*=Z|^8|?i45JT?0`IzsrM3s{p^TQrUG2Gv7g!e_ z8|5=|hZpyY`wp1^>*}2CXt{d~w4(Od{O*5tvzi!rcxsX{_K)X|e%}7-p!3Qtf?J#L$y?ChZieE7ii1BKkS>~sCv{92?U&BP~`_q=Z}4?b1Bu4AzC=+RR( zn`=7)Qg`S1X;rK}n^^1U$Vqo*QnlJ5oJ!{72Q zL%gn$QC{II92QY3{~oyoABj6vN^)Q#0LfZqYT*q04ewMw;yb#5wAUZ3x|5>=Dpyc2 zmNRfksHL$u;L;Ls|4-h5Rm@LwPKLl8uWg>?F_H({od~t^rXy}XAk)!ocCB3UZaU*> zj|Q7gU(+mT7bn+|C@p%7I9&%`{tX}%JMgA^(^(}l5~lZC)%d6OQyNjTBN_WAhqejJ z$BiT9c66e^>-h#WmI`_fTJkDO99`xaez#XYOtkS<+dnRi#~j4bd-#!g*L9+7^3{&; zZW7S6o}&hmDR?#3=|ZIGL&N4k>?)crSl_cx-e=Biak-ItjSM+bgWBJ}w4Xr?4Te_E z61=kcM(z{uZn;N1`*m$ADLgMIK}mC93zGGP#`wXwQJas97A;z9-;g=;zKa_DPqipU z;_m(+?0?_8cfN50n~*l8j7&sTbv4>-ZC2J>V1EJx2N7s5&xw37-D%b9#3pU{i-OSf zqA_;wplfE+ix!j^)E>;1V4MCvprsztl+~54 za`?0uD5~(rmIB{-0p~Hcd$AFkh7*LOSkxf#&!*BTw3qw-e_UK7NsdYsau{cd`G=Ipk(M`a(hGb?cy>h{5sIKyvbN=2 zA9LSs6V%75x9vEmo)D8U)(StA+izFpE=5LXJivrSPCz1-p-C!op|49`V;M}el-lrr zdq@8uf2ZzCD>AZZTMN=R@Ar5Eoc}rIP05efJpxm04;PG!&hu7~=`*z1OW?Iq2tK(r z;LNDCXw zPPv^#S3kliKyib<89-cdd?p`)*CPn@rpv@^a4T+cKUY9Qc7!e!GBSMe$fXzQ*_4@z z%9mEKo#H%upzfygjBI}2*S4eT+$EgP_98+bJWYJ-f;Q^KdwH_AI zC9grBp;N(0DuI@X=H!uTW|=eqbZM7)JF;{eu|CUF*WG)+V_GLJ-%C2vM5`8rkc)?*adPryrKGn3WkT`#1#&+`>6KyF{?!=S>;3(C=Um!Nhte?(KF>94cgeF^_pMt!MQiT1b)Z|w zv}x3-DzC4vpAR#)u-Lq+D>2?spQKmHHY<5pX5b!-7p52E%4 zDbne9&D)9Q7*CLaPq}}G*!kadGAo7qrpF6BB2AXLSM@l%g_Tcy4|^=`ssvjS z^@SRM`7+`}LXCHs#XX^Q8D8{_=HTue!GL@qi%J%qO08F!VqT|Ca~(PDP!=p$$tl3D zhm;pF4hT1(oqAFTyMsFN$HL4D=d`9^jpVAx`@$h@2^Y*7%fvF?27(cM+`PS;?J<=U zr`t0UN6uOR2O%;_Nx51SC6!nI@*^@(sbkIU@9%e?d`pF?Xk=uRzYgYWb<14_TZL}G zMxik<{0&*4^LX|6kTJ}6rOic%3hV#YYN=Vn=XQR1!AOnhxnmNA38GNp96Dhv5q-D;HC>K{pi0H(JBEi*ZrGR=DNrPEwSSDB?o2+ zC;K8r4I;lHNx3}T+wa9b-ilLzNR$BtV2!@!K3$Q<3kSm*ba}86m;?ln`iBe}pVc;i zu3b1ys^(O#HwofhZt4tW@x3}C-s9f*JT7EDdRZ9+uLCiw1e(`AFm-)KUjY&DIQ~3D zE_7MiZN{+BqX&Vx{RaOT#II7vZ8IJW8cYBi5EXfOrkyLppovxTH4D{69=Sr}$=Fc$ zCfBhUL)EEg+eIsb$24wtjuNGBCtAuqOH}3?pJnG^?`%0lj(3^~5eT0~qGq}P=uDHi zoPWd4=&0U0av-h({AiTx&nYwxsGmBwg_QqZDJg2Knrbhya#o=mJ^SU$cga|7zOjd^ zOf7}gA2*Wxm5cnh)xcfE_(306b8n#c`frG(A_X(h3(Van@<0?#vfHVwa$~tt#~>vB zWKy2;0S?Ub54ECEj}{ie>xsiDdADVp8WPQ)|E=|`ESeWYT;u_0tj6ukm=Cx+Hf|+b z1Z|{)vGHJAhf;IZ2MMP~_u7GbL*Tp!MDgOVCfcunk+8(iIUV-Kt9TT56Rqu}P{sXU z`0&QUDDV&qDlo~ZVxyi_P%g5|1@!moRA8j3_s9k}i0tTwk?yfh%NHDgsrbH-fE%x< z%oyd#S^<^7DrJLV)w%k$$BdTSNSmUATlfbqiE&4?xt}rv3__O-1bDEa2nd)OYf#^{ zUksX{rQB(WGClY1QNWPXiHI7;Fe;v6CIq6+vn`c!88b;i~HMYH#nR~aqk z@{)i+`s-x}9r#J}Nulw^nz3)mjH&UVEY}vj2`BDCuZbL#xZSOu939;~eWv9dP@q5% zKHKp7W&NX}E61)`WfK~vsgtY2n9udqcNG711McY_N-r%zit53(0p%|QWV?U;R;gNw zOE?}aaIu+=La`(vo0*wyUNXXRl|tA41D zInzi43XeL}tr(Mec>)xoY#dsz_K0-_Kjjt|G+!-Qk$eiZC@F>ua z;wgZMuO=a+RTB}cVG5YE`Y74K{(f7z^5H5o^lSE3Pyv~NH8JD%joo*KTbSR^rjg-c zp72}cprdnPILlgZQI{ca6tTN*)}@Kgd&zm1;Vrq=`u^aUmHyN~8WZ`~uf>mhyVu;F z^(U)kU^pZ&EDWjj@xcQKH%USwBZG2sUV-Ek%52b`>GSwtToy<7R`(9;we=&c86^jF z_L8ag(ZRFj$gb7#_v&M4m11R-Yx6Ss73Be<%C4oIqRJ`BTUNptpD!>BS!UD_KYD$I zdP%pSFFj$FDI4LVf7>>bZ3TtI@;sdQTK736W+vQo-v!~7EwdF0^tu-zPcv2P-<==_ znaHEm!z%|1Z5GL^qWb=%{3cz;`kIXF=a%tH@549ZT=ku%gUn3GDsXavQj^`r1tkZ$ z+eOTlOB{r4Ny?7W6bRSuwgkv1XsiZC!r;fSwEmrBHt<~_0oxkbK{FgxPfyQsrs4_s zm`6uPy@P`Rx_UFpxIE5u-Z#6H65FkHh((+rP1`y-iD+wUe{$%x-&o`hJLmIx=VYW; zx$b<_dSc3hX+3oF&iEJsijz1toWxo8`zk@-kJdaDc9Q0?28Lv{3ucW)IWis|0;mcB`7trJ3x4OSjh4yk) z!&0ZTvbrY}XW9yWWhn2+8A3;}A84&Rvo~m&V=>Y!ym04>TZGwvceYNYmIDyXH6$u_2YJM^KSvdl(|ZQ+Zhk?rL?JN@ryF`%TPKiGLE=JKPtK3=878z)n3o z>eLvdz3|wniv66J7T%mf2~IuU0q@;o+gqn{4HekSB^CYcRkXJhJ_ijKux{du95@=3 zQ*Y_y7X@lsUMwVE`l{90iM&BW$N0QSBi<0W;iN})e4~)VoUgfhvU|plnf?TvA+G<22F{SzA(ZsNEG~WVZBof)RXd-}r(6ST z#2-JFyOYNA(FwdTl3dEVnBP3{6&Hf~xr#8Pl%W?*n|Y2J1n!FItw3lf~Re0rgwY3qmz2`#X4(_rI;#1?VJuF?=R#B|Kb}pbx^!)jauBl4GQgm-2=1x2M zq~{S87A7_3^8h5$Y6o|9OYTVEHFR$dB-(8c6T!f!Kj=8VPSe%;_|3rLbN#(qL98`4 zPtR9Tq-!UFr*+&`VXZ5~^~sRe@I7ni`v<*oQaY#?uaYxGoR(U|g{SB_nGiaB-$DWR zi#`oEPES6JlroBywVL8yiRMp&xmfIl(4XOa3Ku!t!TolzdHHyRV#v=%QSB1& zlM<%Uj~DZG5C1RT2IhP_lY40xwCn3<50YeLEcpjJIrTN?1{=#Q?>S#0oS41A^!>a} z>hdkJ)E-rYQvbU7uy?r4M@j5sFl$1~*wdOj-a$274Dd6HoMI*}1MY`Xp0wNE8SjiX z;51b$9n7c8bj~wumRpB{;v&WhRY1&wc5!(?l@UFaA-}pwTbZ^()-GayB43}j{7~$| zb7yafEq^(%Kt9_*DzHjImcp_ij^hI>Fu*3)(#@c%ftSH1<MbwcM0LQ%4to3(+@L&oBxksKAJVr)n=w4!4vzwZYX_N)wb0Hw`zo6vn{7J8@La_V7n{v_Xf4>K&$SuS}8P zBuoqG8v8VZpMo55^gv_6v$GP0?2n~#^`J*$D4;@rbEj>n!w)qgx|p%Y3*x$FHWk@n zFaE;Am6WLivOx8~K^pq#KE3i0C!ybhdn-*jr#;F#&auC&&tqOVT*s37FZVaDO~3F$ zAQxKMyJTC*nagaxmOf6Be8QZ)Ot&SJ+i4GXsKN911JNWYDe0yLagr|u>oXUZ$4?tx zQdw%-_z_6Xh^{pUXl*8XzCZI%GJ-p?6rO!)*nN$v*QRyg#xsf1$%7Fw_RZWVc1ljz zkY1bUaIC=^oqHMu8ma*C8z-L9OeQyGTF8qxvFPQe?|7TTb&CVw9v<6bXwY{H$Z~T& z-=Wf&DM{Wxa8Gs-JYrxdh{X6@!F@>d@=LW43-4aocM*;E&FE~6aXPeS_PKKNtm;2G zMk(Knf-UDm!h!c-RS|(aU9F^|5O6<4>4_o_O8D|61lY@d07By4krA;d zV!jhF_Rzk(+$N30@do?|i;8{#^N3$qO;8~dNjY4=nBfT&m_&eyr>#!XFP)&Si>_aP zlS`~?bbew06-}MCI_}Iw&j@2FkxeQNY3;znVQZuAqqQY$XlcDE02$hw9r=H z)R^!rB=5a7PRCB3iS zHA}m}zBzxOPFP>sg?TG@u{UYAH$lzG!;|Q>Y3#WBlbp?Z0oS|+-=_691u$EHmXnhM ztH{=XKf?FB-)B#tfSr>FaDMCG;9Fc+fPw;oH6wd|cYC)}wdneLoZPe-gb?=}yM}5? zPaf1@x!bG?hf65=>%fz?0dPTb-h|o+I~C#te8P|!eLc|=x0%>9IM|8x6hIT>vVdND zrZ<~H#pCODhZEUY$^dYY2{}E&JrygcM0Z$)Sx-TgT+?hq@zcA;jr3-hR$$?S29Z6D zeuLt6?>z)1?&YQgJ@rG5=Rp<{JgzG}`t8HrLQFy@5D*BDoDs;G>qPI#$4TgHpOi#X zjanN`kX_aNcW959h~As2dy@Xmm^QKliHtobt6d86@=!_AaA;^~hK7cM<&2i{(kUD$v6ONQ)mH{W z{de}(sqfZm-FkW3if8NsgBd-Wnvst={9meZKcTV|K5QY3N=lJior%c(pm$~2q7O-l z5ke)!7*1%6nZiGr-)v<~ht;Ek*Xe0$M%urlxV`pzS~x{-v7 zmSrnQoZDhJf`SLDZrU^OTTVBvPMY6-mgEE*r&^&+P`|WP>DX#Sphx9H-21223)Lel zIoruvlQc{Ym0JtwNr-<^rO*`QO1*v!Q)ja*YHiIxNJtnODR6sGJ=5ax#{2Q!C7h7! zIaCFhk#gLdcs7*G-ZhpZHqd+$x&pjHu7S*@(&NhdH6>-&`1lK}xo;i7V=)4ih&yiA zqmtB!7(}w$U*4}R(}GiX)*iW3Zvr~Y=Pe2*?znmn4#ITnwiVyM+PfZFBT&%Uhd1~j zY`|9aQ*?r_`QvktiItHFWEa2JFW2dFCuE@>#c?t|l|A{`zf<2?%Ho&qLy|s3_WP+m zm%epKZguxUNOs6X7p@srlQ z=2rNABy|c2rLx~&k{Uli+q80j`u|1ocPq4xSePs;Th)|Ar?@7tbsc)PCS4=(@1 zTjh!#XI_8UosaUl`pC6e*v}(|*A-jv{NTM2^V=(a(JgN=fqz>=%lg;z96O6y%HB`> z#%x<-6|{cvp2!|@`|jPRd}ds~ak*^>Wcg2sghBn_EUtxYrZW&851xAfMZpd7>I|%V zH6o2zQE)ng%4xfc5+gq{8?Fx!jqX-SD{>kuTUEtfk6B4rf{mUqo&jT9;ACuwtBPgCo5ukxxnGKU#{x~Ts>x+kn zDK~h04IBV38tBeeaP%>wo<_HL`Et~;Imk%G4BO4#4 z8bd_b(2PICGDwNJj}s#{oR`WDy(VI7;@OFC>}bEb$1k|rtv8)dCc2ppgkWUfMn39= zlueVGjl6qJq!AqLecV-SB!IxeubdvTV`iko|50l|Ads7D%2tp!VAvno=`-QS9&Uq4>GiMgcv$U(X2>i5#$BsLv<**X1U zas|xFvo$NvCUAax@-RJ)Lf}ZhppPMpR6w4#GBqBtn z(4`)gdIw#?@BNhafhSPZWhw_Pa(TJ?_|d52NfWOJS;;NJxs~pVVadRxq8dv`L7tG1 z(#)fV(RA=)V~O)U+tw^6^xJz%0?(j2RjuLf59cIJSyOwVk(1|6JymbqUyhFd2#=qI zlLFyNr2Rh`d?A|bOkb$Tuat}{HL?yo;Jp{Axm7a zV8%&PM*_!V5|zDvg|7hlq}6uhY}xpMA3)NXKq*nq^GU2(fsGUA-s-*qsbsuPvBG*@ zHX!vLWhA6h)FFE{tnI)f`|SWKE-`U80paTpkgOB8u!Hm~y26d&IVgg5YTyhmnbQNP+zG3 zi)?9U+6C)pdsD3Qz1#VheR0Dl){7)hKEL?vvW@t>ZdhpRx6Y4rq19ES#(?SsZ;X_n+L6_*nt5ax6a6%0hSyapq)LWp4#e95&m#!OI zpI)3&a%X9pQUwJDuQudOt{5_cO_CgtZrbp3`&hSAfd$P6YD7gF0@U7(lS5v_2Mim* zGDj9UyUl~`wC$7GjwP z>gr#p_p^g5NR7nvRafWGUPTi)G=Aq|fyV)tE|f?Kf(iBkCTK=8485I~3C#{E-L`(e zm8Zn%GGC8Djv#m}s=}Nl_KG80@)M3cq`Xti;#L!JkI%8}8z_YsX za9v6SLBsVT+~pjMJ(vm%a@fC6UoH9Lk<5bn1KzJPVkZ1BU$m`NjZ~pklTz$SPv#OU zt8;5*tVnHkqmSS%myvCWPV9fS*QZ{a9tvP!oY$8=+6isvKW^voWP(W=l*USyEz|VJ z+mABX&9dlKu?UTv?}IVD2-2Gm4xH{z^q&*74t(TMcEkdK1B>V%Ah-v-Bf7xo+v&2; zx*}3aRdRS5^(9eCaJdsJj9V+bzza2J6M2?>7)Z^T1+Fe_ubNgU*|B`OfySPrm+l zoX`8smujUQ?IaxQ!qUGCwp(XQNkRTKKWPc-mlGL4`Aq&FfWtS&ni~@nQvocX==53% zgHeh6s6x?UAgxj&9%o@?WlWG5;Q^w55oehKo{#bVTy^34FhX$zZHMp4179~j;rPCY z<-;SE0*;iaFyC}8e)pAWz?`Kp9~PwbZD4Jg*!q&>6OXsZ+NIu0^9eWuH+g-$&Qr!@ zp|IOxPp!$5u9Fj{b8}4d@p%qNv{nJJV}jBCNSuPf_Bt8F#HLm z#Dby1M0DJrLQzmq2nh|PiTz-ON1%B1l~CJvhCEh^Pr$I((X|c_A+rLbW>*>+aw?@< zenxB|LXXx-u!l#PhHEyjH%p)Vi5)_Fahwafd{6OA>IfnhMac(+)KcH6_^*SunMiS7 z(e2!l01qUnW0Af|+ICr=3LTM;#&vIPt zIlu?9asg!i{uE<|z7{H@tgp>Q@hl5h14N&>lSV>g+#2Vlf$Bz%*PCStv5=<*fN( zI@qA86d>${D)yMXshh}U$xGy)=rgae_q(m_Cu|FTU~<_@_t&~M5d8cXhK}DIuK=^O zpU?#E(1Gn{cO11+sH!wZJb7J(ZExK0SNEVe>D}=Ay_bWjm3iMqC(ee7Ru7LNu8~p6 z$ck(i_G^Rpg9)O)-7>zmOhWQ{t2?Gnpe=&5N}-yG^?;$32^E@Md9U`O;?W$sS%-+J zUWFdnN{Jwg-+Jr%MMwy8h&xft75&@0o21cg0`K%ho<`D&V~-Q-d+0ymAfavJpt(oR zT`)-U{#pXkNfE}9Ph!gjy^Y~Co=RXBUZPPAhlht(Uw1;o#`cYF$*&+}p#yfQ^!An6 zNN3OR;wzhwa%>_FZiW{y=N78WaiZBPK11q%Pw})ci=ckUA7dK3SksMzv9UCd7!LLReIf z6rywaOu5n|qML^O*gtby;A7{|JaT+euxfsffLPAyMbL{F-R;#}HibXUAmO|VWi3#~ z*P%fQb1+r^IW<*#A^=n`T}X`0knsG?e6naTw8m(mYDC^ymBQJ3ilk$2=J~evJ17bR zz{j?=HA6PxBmPfn903)&*PW>F2DkTM|a1Sp@!lfF!KNK`cTBuCZMLk9Ypp*PZYqq1gza zD|3FlOyUoZCJMHwkX8dPqW-4@ib4FEZU|wQsoe+%B5JArFAzh!|5Gr^PlU%jgT8rF z9(l@#pQUN^5Z2857ZebRdbP(UK6m@($THdL+h)9e{zr#vTT8oxkAY$zFfw%Su| z6$o}2bPbnDlSqi5=?m?(0!Yc0+Kk*_jsn$=Va_IstQ{Ri$hm^~?sM?%ygKi43HQ?B z?3V~C*400VEl5%-Ha>j*(cAx9jzxGkCb2hyr}uMN>Wq%;r)m}Vo`mP*o(E@Fw(pep z=Ugjg=waL&zkq>5NOE8`vPS&*0=@2+gJJ5dHQN>_u!|tKz75~#p%rVgaK>;@-G%T` zr@??NFM0rdvJqRhhhp7Q+x*kEDgK^6)??VmYDI-LLKeRN`IP43z#x|@UJ~51mozkp zEgn~Zg3O`b&w12I`35+?Fs~ zlP0((^!VJ;58Xx$ zFsxrHA2<_;8&w_8h~(>KmRD5SPv07{a^%oJ{eKTfuU-KKY!5vZ90K~$KysnycZGeC zc))mZmn1Z|_JgBFMje9>@h@C+{#v*NweL1@b@=8ubHHQg%x@62l}M~z-~*wshsi;s zD*@7_UT0ZYox)Gb4na8+WHTiKA-J85f4~WoT1mTHNa+BcTcPaQk98Phvt4i^irXKuA0U z#-wJ)WP%d};naZp?p-OgEuGveTdLg=QWqxB|pJ8mOSyP2q zlvCCI?@PmxO9unqG2b0>p_rYW{L1;aj^>=v@V`@BRWyMIcAcsxoXUYl4s+t(5c*vw zC8(F5LXGa6&v-vzLyAzo#C0icb}Z#SSfTj|QRRKj6DJoIqu((QAi`&5Evz8iJ$o2( zYeK_@=YrA=!vlNQ7!jy9J^fONN{Z9}0>Wc7>ufXIO#lASmd}Q8*%|DWoF%0G66P(F zP@uA^YsL5}xDuNGE11!F>pP`TGUOV=0u2?X9YW=v|=z`X(bt8%C?TVUi2Tiwz(*s zTVraHKIlluMNR*Y7D5AoBNutIh#mL3%Q!Veg27I9T%H8d+#tELF#Zd<%4>`lvDle( zS-w8jviz2okQPP>ecOywu6j?(TS27fs~SviV)K5?cVNb4Q})v~>Z$XO(UUpcg(7TE zZ6cY=H@v9eY%IoZh)$S&{$@gYWSNTM<-4kyVQ(S_b9D<9uX$c`qavu8Mob$iYOs|T z8zMKtj(>G!+sVV1qAx#?ToJ1_SSq{Q_m>C+LUamEg`!X+bp*v#YS+NAPLcHd6q-h^ z{b6e-uWwtNeEqTtFGT?h>aUAvNK-VQh<-JqyrC@sA|2*WC>1zw#O-J}QNUwYSM0PK z(eZ#A1l$-hL8Ke2u$@$PrGK=4WKISZhpf4^y}|HH=h>d#Ca*rz!Qu-j7``jw18v@c zZ{4AyphW-x5`?CxX-X1ZPnVT!o^2r&tu1~k=E1|NES}f>e@`^|Kp$iK(tt~JmSR_z z#3$7EPdts4VgEiNNikHR;ykUWptB8YH6=tF0sCaB6rQYN1ZJi|nZ>s=T{X*+e6c%o{`!0#aKrjJ|w(P zF!J_v53n?|qRpJ^{FRzrS?1c!?7UKqX}7js$=PoQ!Hr#}|DQv`6CduDSu*XzDnM z^2t=$?$_&KPCXTmMRu>2mSlrgt-4NBuSAU(YvcOS%E9gZPvd7Mx(eM={PWkfzY=1< zl5Qva_tMm$16ufqZcvS8_9JCQjWdBV-Z>s2O>lCfaCIXfCm`QCkF`?NfaPub zM{M>J*^%yowN3M&yLxO%1fcDZ%fjj$TXfJew3t77n5Ih$+A6j+i)v; zox6Svvh`XoyWtiVCXbjIrI=xC5$LuAZB5c7&i=t6_&VOG&*iLxW#eJhYy|!^d{&?p z@;g8iZ*4e_w#{V&YK1kkEY?*srci*ak;ek6z#5(XwN z>yEaK6i)yB&t3?l6z=Sp4koiB@wglbgUSkE$*jqx`jPfM8}uG4!kFP%kMT(g-Clz?q?aw1;d;uZ9ah5 z!0F9e_}%UOLpmt?CN! zT6^d1&41LQj1IHo$nEcz#_iPU^LQT&ibsR?k57!$DkDA!{r6)TTK|?{4$opr4uPq*Jl;|rSJ*5Leu|PtvWaX|CaKNAMoe9%2ej&*2 z881|+zwAQT=uTz0upqp%ATvBsu2;4(ZtNVS<0*#{IOdMhb50X0o4a0CLh);1ZjfXB zhK5wJ7yn+F`+t)3L^QY0120f;Ax4P$R~qzu=#3_I1>SC-latLSqJbirTG#Vb1et&X z83~*2Z(_!<@b3&6Du^j!n7$lXA3rW1?LKe10GNCjK7nVK=?`DI^+21X2J&GZ1~{=& zwAY#&edC$vnmORu%Acr--6~EGETHr0F7%Ipfc?I33lWk0&ZGIsGDFVGYFaOBdWlxi zFoDOW0FqjeWXk4Dn}f=EmW~%OTL=QHFhoTDeYikZjNS9v_6~S@xdOfQDVV%@0(EzP z|2Zzsa01e~_RGj1>Fn&Z1qNqTrE_qVq#_BSfAzbH!~sEkbaajNY<}zQ1nz@AEs%cb zT}`jf|0eIn{JLy^e0Qi!x8Q?U9I3+UiSOK4rhP?jB2hE#$3a&HmOclk!1`CMZ zSa|;qHGo|xQZDqnzIJ_0LDAWhe-i8q`eK6bUxOXkzCo9i233+ z!S19(Q3J^gc|dJ5gK{^~aTw$*i%7km&ih0=4u+cVZr}_s(eWaWyeWpmfnSfu2c+1H zCkuX35%jf(N2ZXvVC7kerN+%}Y5nOu!Pj)X!+4U_*@C8Lv@|~MEg>lR1}7KYFG5;V z%-D4MG(Np)V*lQI>9mqr!73DJ`0Sc1;=kcMJ~t>%u`mFr0$EsCLL>Q#NH;e(0~Z#w zH@3GEB`Ryf|NZ|bxY?1csd}S2{glaJzJePet6vV5cZZkaKi-`&1W_K(B69g-k)YNA zp~+Y1`xF(>S*SUcu6P3&$3b-;LV#je)y+0blwTG($#|mAB7Z?mZG?DYVV3*wZnuI|azRnOsk_XyETvwsCC6-S^Njd6Yq3>b3AVwA@V ze+Eu-x)+MKj-LPE(%WAR)n|wmd)H2o)PS=#E}42IU4c`j3>zXt??&$ZHyTP&6oy(; z_43VK^ht1UttpA-wqt!ZK26jEBwnHhm(DEVmTK@e&K3wZYGSO)VhJG`);`b9c0t;~SAD9k9EVm+6y_7+Gj~rJ zq@x78XNjV1k6%EwPmU44nxE{5l<=zDO%q49R(TPL*f7@YU{@GrDf*=S(-%;*`Dp*V zeW5{MCChDncM4W+#({`z{eG3Ar2m|OGS^_^?r?|Qp+$jJk^%t$GzwHt7AU3Lx(}?O zfBubBaZOU7e)D-hQJ`Qto`>RmF!KZ+KKO&0_~CLJ380(g3ze?{dx(LDXBM+&^-K+t zdWnesu*har<-{GOK6zvUt3_j9v=%QBjvg@xJ&XBIf3vEX-vJUP3{|^3J^)#w-03ek z@0o`=n>&c)fnI$rWLh0V-cn}Yvx-1`Qu(QfrBthM_S$$|qJ$SaVy$)e5)d#~uTruT zGQ=9DMW=lK=k1z{f65L4{Tu@zg)yBdzz~fhj_8Y}e8tBH8Vd)4mX>rJHcR2`7E^-7 zlVmI`&w;>$-RRypHa$I3quTgA5dJ~5vYMqqiHSJC$vy~da>VkwIj0^6vW9=M!YAQ0 zIEhF4?zD*NtdS9B+!v5$05NPEu>q+Xx4>_Y5VFWc@y_sz9y==T%8ByZ`*?<(NviWt z%Za@|cYZn8dR!y|-PnTZ;P52U+`+_iz5T!BdrG2wo!ija|2lUmVWk01>BO^;zuW zj=X|Si#rxr<0b0p$qOY&5!(Uwr)FxfnNLvNf&Dc`m+AjPO>{{Vw6LcZZEyEJPl^H+ zjsVQXEmV5MtI&Hw4b3n#-Hz|%w^DpQThX*|(tj$^%VuBmOVdkPnn^-Fl4rQS1OdiWLqMJbVw0nbpXq2CB z6nXM9``R|!KLYI-Q1)3aw-SM**1p;m{5d)rsznzKsC+?R3-Og$q%hDr^*2r-<^XXX zr~L+kr>7?qnA|zvn?ynkgzoF>y9Vw^OcqmgQ0-Ml15Bo)nJjK{9J#%Ue|LkXUT$|& zf4@J&Q?|ni@7A`cd3S=8(LUC;^&-s!K*Z*|`C-M8ZA9uSSoaA0)BQGDPt^$_y;DeY zZUu+mH*NfWKNKXDPn2~+K)_pEN3zh_N$O%teO#5HuIr2K^WgGK0Pre~@gOngmGe-D za0mhPB{vC0>5lvF&*;7u6j$2lc18hIhHy~o^|qwoYZ|lVgY!94X#fw0f*)WskPzQ@ zTlG&)tilGZ4nWbPw;l29DB}hBvvAS4;6{E-fAn7%Q~BxU@C#|w_7@B2YDA?KEBD&H z??&er3N6vJKH&OJ=Z^FH(=af2W4%ysSLpzZ*n#G6Xo%i96$Fu9 zYy=iA&pFYo2g>;sB7R&~+ju|FoLTf)yPL2U=c|HKJo;>JS0Sdn|Fg7A(ZccC0gMA} zQ&wSS%Yw$i_R6n8obY6YqYIv#rHu%3jvTrOUB(ZGeFUZgR#;0K?(GG5Dt$kmfSC2& zR}K#37J7~xNazZodCB8|qi+VZ4q*>xg-ZGIZQfovT$FO@F!J_Ykm%N%Fw@{l_Il63 zH5l#P0zmi?TuN+tL$YrNv#+8(^|^|W?|?eNNy=#$Q249L(kwu@BH(;wYOHnR~|?@idB4yXNqJoMGjgqqww}4*B1_zbL>FJ zzuU1aiaBlHf&4WHo)`+Slx7313OS+@+zlWP2M5Cg;4|9bU;+aVFU3$nwl`Uf1N=nJ zdrib)VPRK+T%ka@P%){QvhLuO)DQCRKXG!j zzW||w|0ro7oxi|&VqVw$#V$r4*MlV=9GEBfw^uz;0uKlzBpLZMa|=gHEn%%*w`)L4 z28H_7_;BHe{FOtZMF zoUqsdUHF&8`j2jGuX=>vALtC5-Oi#vfA$4AgN&D#(AyjAuKxaAn2^&J3J8Y%^YbYo zndt7$GnLov`R(Q5Q>ckzH8dci4vLM9HG9amj{zA7$Q9Nzd>)uN!J_N2Vl_sHrf!7y zEIij?$$}e;5)2%am6e5rg?VJVe;OqJYyN=d(Dy>PHoSpUD_gcXXyan}rNCD1*cX-x z22ZkOf1!#~>^?MF>Nir6+C38&`6nxsOEEsD3yP?(av2^Fwnz0fDwor);jrxx{0HUr>9fZ7xIhw8^ha%k#f~2H~l!TxX5(-L4gOrq%h=eo;29j z=l30cd+ojUlJz`yT-Utj9CM5@-`wx#_&PW^0bwmTEKJJY{<2wH`~xrmSKQJ6Mf&Pi zawzOr(XRXaR^J|i!w%X0CI&V^8W66Ld;>g=J{RbVspjiaYUJzIf>B%$Cy^7t_PwC6i!aKI+?ny-e(d*X4dmgHPUq-b3j zCC?j+KGwCLgPYL~;5V$lHs4PO7S*Q^@+nHQ%2O~sJ^gcqn*)uY{fl=tItVG%55}b~ zgy@xn8LGl&jMzyW?l&e<4sRx)hb1)8166M)XiqvCZB?V-ZROJs2rB#(DYCU4XzI(|d$d4=Kg`YkWE}xyYL{NZ zU##`jepw4Il7}AZw2B7YiZ!$ys;xegLvToSt9CtQZg0A3#3ZM8%WXz*uW%Rr7tTAG ziEv3zE~J9ZyW_0$oJ54(VJ5Ko^gs9v6iqN2I0Rq?hKm=QMzxe0VCiviaGb=!sf%UP zK%qcbZ|mUTUsS}?-P4nrZ_kq;?7}hPI)U%%;lU(0utlJU`dVto1e_*()b;DvnZI=} z-+S}dap(jmn%(`WndU0bn%^=aZybPwLnD6ta)@%Z7k13KRO-H4%YKQ*9(%&P^;vat z=vgr(?T_D;)I*ET(8fkT zhZEC};TlIAwI%n==qL)9*}yoID|qKu>bZRxl?J`K^Br!smoC72r&2~*l*M(m1ws}P zyCjt6#bX;eoVTOAGw)vFJh+_iNgp8Bo+^2*^AG@bi5E25n3W%vS4}UY61pjoo+X1l z2t6D`Sp>&st$70`!W8;v-yWI6MuhtE$ zrk|DmN={$Yl#L)#piUa$;%0ptkC*|9~DSJ zfBpI;At8ZIKqKJ0yu2*iF4=|_o& zItHJg;H)^;e>hdSGOjpi%g2Fo=wEquxfLlwm$iRs zD!QsrK>l`d65sPZ40hw$5{Ed5@KB%?E1n&d^jreDB}P`l$JWC014S31I}AsZ-B?|~ z>6tqfuEY?;d>Ky8_Py5VT7j0UWI?}}y_1r3{~JR|)HNd`Mi`T;x$x|}_b&dmF+LAN zL&GU`YB=qq-$xG!Gs3RR;&&qfravFerB{^eX+!ZnrA6May6osRWOKdQ&}r)yzn?oF zTJvHGdu6>bChASb6{Upc2+`0-JdELqN;oe1y{`K%5AjkgqT893fR-ph^TBJMvvKcz zQx@gacv_7|W#VY_6Y^!bCU;92B+Ve!5(H8pq-Y$_b$nRvvQ*d6K>$^bq^qmI-ahW; zkmB|0HMRcOZ=L4V@4naG+TI=m3dQpJIwnftnE)|xU__9R3y<}R%QcFIGbSJIKVT`~mo50-Kt3Naw!B^rpo*}do`pX4;rtgOc%h~T zBT}zEcHa8~4ny!kjRL{N6-bbF5Ju}ruC86p_QTf1qLIgPe(DFv~_lt0pfEPq_!vbU~0vik`m6Czf%;c$fwRm7}f`Z zRlTdPD}OOvrFPMV~%OdDROM?5ZJiOJt_$-6LYCaAAxlDTw5c=b8_L+TwRP&jb z4;?_dwZ&G0w);Cyfc(@qHDM#njW>q%%TPE`?e8N&Xl-rHBKF0$keXSD@#O^H?0uc| z!g8ip1tNNmo;?Ygcn5a4XTlb7TrN(L^vkVrDP@Y+eQ~?~^%qH6KUG*+|I)*#vpvsk ziz<>fjZ_K&po-D-+S{prREa`Ghrb@*AGrxc&Wxe^Ovuhl4;BA~1O3Q+1X&gqIs#7U z*#C!%F^U#5M)hSQnjeo{*pm#Ju0bFJny^boDPHob6yCdfX=z(Tb!GCr43pWI z$L(Pu^WV97%vhhrwX%_3^)vlR1(DqM9_^HI2kA9zUD(zpwdmcX{iEpij3k%To z=jNT}o>>$sZt+FNrVI5OOI?@zg|v$o=GK87hgzAh3dmyTqHxF_IQ_i7%CAs4Eg{F_ zasum>=|zDceV0_uPp$rm4|hiEp)4*|_odAF{cM8SpcDER(jpZ}4t@J`Kee``3Y?Y; zFb@}v#|F-!`zcuSdPB~Rmt2BRnX?=2FV`WrCsHh&_)vR9-NHl<#()QFt%=&FM^e#{ zlggdgaKi=_31_jmHeJf((c!jPNxnQ$T-4mg#zj>yHS>UQmsfJrJjWK-p-^t`nGDi* zJLR5FK~(dT*VEdFgctZbKGTfdsaxluN9*U|lwpeyEgE793je$lTfhgX;V390GM+>sW+|-aKlCSZ+ zm*WHl>dV!9PDtH)sQcrD+Mek->*Sy{l_Xyy(dus=ST!Z{_m{0jP?rM=XSb5qbn#Nk zF*QhdLL&xRPI!vFqt&~s)bqlJT8rNeP}i=HV+NQ3~#fBY0#Y3qW)J50Bqg#6R85|Sp68%6jMc! z8_)VD(D~o{0OFlue}J|yP~8rF6f3RHG?jM!28)@UB{g3;I^(Aa86Q1$wkNS> z-z61N3#_%nK1|FK+M^c0?B`K5Y;f&ehwkV?DNzt-e`-y@7H@_fWLtVZ8Q&kIdPZYGsMIQcr^%u|c!zwNO#s`so)g3*V zYXv3v>Z1ud|Eq)q*&V7TdOl>!XxNUX%)5Wqi2w&VXvh!ezoBAv>AZ{?{|i`ADMv&^ zfV2Q4Bi;)+%&ajg zru%G8?K8CY*Au(Xe?E^|O%N=;v|6Y4k>h|UG54JR#KuVNN!lGU-TS=!DyEHpFm}oO z9dF4mTG`i1c51I{Oc{5+6qYH7X+0tJJ|_Se>%^l!bW_fgs4@S&e-Y&q7Z(w0EY6P~ zKVIdAgx11_W&OQjUw9-A-7Qpsz7N{t4Guq1v8KDaAd6pGuFV=UC9T?B;xn@mHs%`1 zwkGoSDB~$%FA@hj`jFoKd1V3)eyUXh`mi+bn9yRM+>>yXczcFCfTPH}-$}io23na{ zS&Oanr-}cSDLb;6L%AFs8yf&kNR>WGB?yJ#d(ce_ZjiM-Z16T-So!!YNm5@ zt2ljO=)OP;mbZ{`%AtN}wImj_ogq|2dCaF%yxEPD9Uo7@YxRv3F!_()-rs+Idzs_& zuR1cwWL#yjkDCvs3Dn6#sxke>b4su z)@@nOzMmE<4`OT}|Jb=~($`4pzdTB10B7tNsK44EA_2>e>Xx3_o16MKO#FPMjX$Kz zSB?TRw7KwK$B-~fjOk^{BS9L~6&#p4R_n1;83fQE0|Ucobir%om!DL~j+l+AdXsYI zTh4BjqCbNe*O%qaStmvMtzoBRXY|(!x(iQB)ST6#z0_6m@#Yrej`ambbL*P!{_>1l z2Ip7z(M&Fvm2|+-S>CM3JH##VCoM@}^{A+fTY8d1iHUMolB$w1A3tgp1UG-VZgE{L zPy79>n7Nsm5eRM~y}}tz!Y1fuJ36>7GbKozN_FO}D{sH{f=(ysW8P}otq2w>a)$Kq z1IBNoj1g&APc!iIO|VAGw5Z?OI%>Sj88fdM&sGQ#!arTyE?GC7imDuQp$369(Pa}M zQfpDOoP5Xep0L!&4Q5K#_aUrjk>r6<2U?8h4PB};-SLs+S4sM8Gug6rAH5#?4&(l$ z89av|P+s6ozmwwCW~9m_3N>}J;+n-~pAMgU7##R~nkDg*f}Rnt(U~gjzB~u4CP-3G zE{aj=Z0 zMqv9}YfAnTei9)-olmn58Y42f%%isHY7tH#W+~<`IM|VPjcxx`U2UF4T|@22iP+C6`llbBFZ;6_Y#kW#JQVhW@C|O|7dHE2aAM@(59!S5%r`vb?OvgF$u^NgZ?DhNS?lL+ej@QS+{@pE$g~G(^QGfPh0|vqZ$}$vYxn^ zqwdozxuj66hB9?+S+*0un99L#gp*ZaZukV}`vP3yvavJ%|LrBCN`5f02823So-I6q zFxcDhwb+0r0hy4|$6(^?ii$x;0s#~rl`sx80KTMC? zVE8bBw_tZ@#%|?|Gk&;F6*A~8K6Q)fc3HjcSHTr|bGPP-H{@?0i9Ni))-qaJ>B1!u zh5n|~-ZXW>R;NatfDGp6~ zMuyUR3pG>#J`42O&Yh#MCPCQoh?a@oYmFG-0PSM6L59DYXy=m8Pfm9y^WtsaV>^9W z)mEA=rJj4O_LP966|cr^ftrh+M2G;~L(zvv^?ouEC(OD$n6Fnt(L1*ID2lw=*JF6Y zEN5Xw+2Pj;D62H>CKJ0f{S9n9h0oCdBjS8=&uM!})hXgB#3oG`%OOUgZmhzSFIr zep#ga0QyDaFE^YR;#$t74l2V%NB`%d0a!@up9j{CwqqMe?IG)G{`ASOsA%lVZ=p37 z{w4Rz|Gwf+H??nx2DT3dl5n?3H$7pld4}~Pi>mxoVq1A5=i?ToY^`&^Rt?P154X?# z@@r~DC^9}^iBXLE+i2;~0LfW@Q4q79kM|EEO zuJ`#dT&mK)E_GOjoQ5V096%?hrpRbfP?*zOul*lgr7+yGvE;4}Ao-=4EMj+U|&hM~`|u#&?zu{aF_!1U0PO zpoi|lcj(*ty~?0B%)a3Jsr3ssxo_IDJj2iE+guId(9h?#kii6{XYKvY@kotSuN*Vd z;|=B((;H<0APvMjXqh~USYI5bPG=qmwn7v1l+u*5ZNcOLk6x6n&b?>r+0h{%Pe&5`EvAIQ@oWF>9d#bh=gf=~y=QcHayZj{yvYP%iji^O zQzlMssyK53iMmEXUp1JQ&>Ee7$04?)BDGoIVOwR#pZ*X*BDRic{i5 zq0=CCc&QxXwY%{g*a)WjW#MB_y#PS_7=O&$k#C2FMts$xfcKF1m(i`FbwBRIsSewB zBvIb(Y^O^O!YZ~}8&c}utKZ*pqQ~vm&chRp9ApHharX#jus%OYdgHR2JjWGnxMPjm zq7^&2e%D(%>do&*jHhOF$r$6mcAZGdvITZCN6+TFUg+~8+OvO3eJB;=3i>BN8VA6a zY6$!zgXl1TN-T20kt_%OFX;6XTTw(^yBkYxOxNV@k^vX5ci?;dZ?edqIvuA!j;mX%NX?wx|2Hubw(1XdEw?Q_QcE7R$fpXZi*t`_{5 zd0;y6f$4M6a?yL{(B>+8qEk9Pd=?c$6BMfC2g6M@8HSrU1HLTx!Q^56v)sqL6ap{r z{MvYx>lmuE8&_!da+0z?$d)HUe+$C!SWpTHCaRJuaC{XlPhsa7^ZjNpAUGCJ9k)(n z?@5L#1pQs+nzNS;IQ-y~+5bS2@(ZZY-F6~G(@Xcb_(3Ld&T^kpi+uBp2L~ik zh!a6<;zl4t3D3RZ>GT!_iG}XSC^zqtG2b=$2J%4FI#M1HNkn=GRI+a?;`)$mqNwoDy9@PE)b!4<0^Y$r+;L@x&8e)9XoE(yxDl&3!{*AQgnY%4=HF>jA$7Cxf<gqf9K1psmpc1SE&i?M;_z?*$+%Nrd*ZS2bIk0MYN>>%w*-s<=LC%V68gf?b z=ytBcI~0>(p^eP!b8NojZT=gPqYOVl3c1Dn zzxMT)#H8E`^sp^cVFGFD5xfmU6P~>S9gLHHx5w|}c9S4x882vb(Udays6dqW9Ee?6 zLv|++4)2Gx%P&=C#y93DuM55bI+U;{kzY#ci8B93l&l`_UWZV++6QV*L)O}!8ft)h~YTF~`LWbh1$a)0ijE1OHjDKJ-rU+UGdzrgQ=8~S?7 z%h2wEQh?v|rYhdQ9wqxEfO?)KQ@m9JuHB=q<=@b@CSo~ko!)WwxjDKGTpy`E0lsKx zR<{znN*OfLlpj?=c%mW@J!`65t_niO(z4|{lNWV)1UOtaq?$pqGH|Z$l9Py>y)&Jt zj@Jpc$UFR0_i4ucZPM!yy9vD5*tRo}qeXgbsGc=1>wLmj(mL+_|K`N(r~@GH3LUn#B>qm-^(RTtj2QzyObv`hi~up9WVWgV z(FsR@PI_lF-x#3P)z^=WjiJFgg&}6fhG9$HU*A>-HQi&Ugi_Z z#M{H}AJ4@dOrKAvWMXxpd9ahZC>2|qT0b)XoV`q1q&uRZso{r+-F|<6w8y-s*8Zu6 zPx4`nQ%hK59gPE6?aa_JKFjI-U@pTa+qON6?bMH1eh;P#mj-Nl!{&wsNO#5NB9&<8 z)SLe#VDBVwP(VxI@q~UFq6a_*4L`cW(pO@mkN9fLx`Mv^jd^E^Y?V5xG#;%Gsr=JR zXei))NOHt6adHx(kZo!SR*M@OF9-Qv@|H>fyK-rk;i#w8amK-cSqP2Ed*DHC#cnt# z^RSW5PRgl`r=FrpnbO}v!mkhQko-Dw-~o2&^3s72Fb=)OKZK8H{7H6XAlfNsU`hme zVNTVzU*>6L`)suFr6<08f`(V9vR3ouptyOnT~fy<(q{FVzOd*7@WPS%lB z5~4u0!Zf{fqq)ra<7jExZ4eAe0lf)b0#oPCyG`Y*Bl_MbZk7S8rm4$l8N&TdAA=ih z)mgzlh_^VAIz6-YubPnc?b-8e8JymJ464oS3ywmxlfQNcpCPOydY`=&V`^nLH@A}n z1Tq#DtfG6%k_xG!!v>xQd)pKu?!0hvAW5x{&*A0lxPP5*C2YY7+hpRW#k zU~E8vL>TjoMIw_;G4pvKrZ{b*9CH+!lZVF6oFVWMr#tEhZ4U?9Fctplt_j#>tY$Bb1enZI3h@%P7 zWK&)5i-;4AYQKgy!+VL2a1~_3UFKZij*lVtq2xbw#dYy~U0XeW7fcBJ9^Q0_I(BSf zOFJQKYG|chZ!Rsk)pu=n*=AJt|L-784xX}_P}r3v02TNOU?TW*@uJWj{u z(I=e{d=}P`m+}F-v;BEGzmi`e05{lChE5|cf5ZuiP+enYlpEEJdueXh6x$aSr-&=7 z(^b*Y5D0Jg^`L8>-dYN7 zYw#W^I%AV0uzA$1Oj$sfG@zqsycppCdQmK6@mO6hV}tvSdHM|seMbc+vDT|5(8msy z`9oKt+2ry&tHEN`iwMVM&0PceEyUpKaRSD(*ja@Jq2)9XaPGgI`zcXX3vP4mz3?xacuL0{Oj>O8A%wzM>b-86qQ zrtqvwvwl=Aen*$&;kL)Mqyh=EDoPw)v_BaVk|kYAC@=R2F^(K6G9E0p67b#=0DkQ9 zkIdAkJNRPzYbw1tYPi@0G&LZbr$6*lP84!tg^Nv@Jp?j?x|tIZ^5WDSIZIxL7hv%Q zB`+KlW!(DsYP@0FBPu?dwP#aV6-Q#KDrT%_W?#8l=F)9(GOrg3^RB0%>q^#S%`m>3 zC)3VM{*qSoXAgzE{`^BOAHr4A-;ek_U73W%kM5V?p0ZT28u22C0cI;!>@JqOPQ_}x zF=;ssK7l{7aiP}JBTUYBbkRMt`K@qtL3iu(ctu6U>6t9m0R81Vy1M6*h1u9%e|+ol zI+L#N&bgfCtxk^P{!&}`b7ER*hUO-cg1stSH%x6*&HERZ@V1>epGM3-(DtS?-SMWn zXntcg_H+FDVJFBrCMsWI+paa<$#^SzGiQADAa8Vj(F6>`$p2LEM-BVwXtoyw^rCpE z+2Qh%Ns{j#?ZEy?P0Jleyl&H(MY|5yXS*^KEJ`;$vphN;*{iK`KzrBE6^<}B&qeSXk1~Umuhvj; z72V`LW@iU~YGtBU2Evf7y}e&#WaK0XP&p}qSQGq+XT*Q_abLCyHp&(r@9CK#>T>=# zB(ya@OMIx_2$V14SvMa{l@D^^XbyaaO#y|w6^jrx6AB$)zW`*^78jSZXpifYabdrDr46CYD9V&I!gNDSvo#|I(Vv!Gc>$4~Wl^t=C(N@&#tLLI$54`m~ z*K{2O2E?{E>5&fw&e;GwmXVPWzyl?moi{RGk+A{AHz6Tm`PVP^@p0Dl{5$1!02vA; z6Nv3C`*&@@49e)IPq6?ng0pM{kL)N;l68Ksr#y^U3Q+I!dKB?vjd5gWS8T6z|B{rF z=f~Ib7l9V|Op;o}KEmNH6Yv*2=QdgKEA>96@5gm5uBvG|j0e`nCuN9KQT1ZYUzAF> zHh5{D9fZ7s;AVA^!n0~6)^|`fu*c4&y3T*eL;1R|&xwd@y~B zycv)9J~2A#3(^}5RMpNL+upBMPUr`M(&@EEK_g^ED3ndb3RddbqqZwqL{Q9W`KI-2 zQvO8#6Fpz(>Ywz6QIX4clxRO>zcv+7A6~o^BPfu8l#kohm{Dvu4?SOjZO`&h zF21YF02)YUDZ;>&h;_rOm!9cc-vfmU@fBx|AFa;C;K)+^Ur0jsF@&TG$a*xw&bhsV zoACB|ZANHjI+E*wVZQ>%Rh(L-=`BIwCnlJBtB{UY@q8Ie96`N3YcBOZw1At#AMyT0_-4T&qx`pCJp-P zh_TgTri;~STjX*KF18OKe_6KXAxa7V5GT)LvDq*gL+iOQ_{n1crIjXffHkAQ-mF~}hLx{BsD;wl4dT4F!<= z?bm%MoRT6HW>UmCvN3}8eqFZ}?%%Dth%z_#c}1{h#q50IX6YdNj2m8We zMH2Nm$CvyyxU7D1n~1DEW9FCjKz2LW6(x0suwwluB+o_bYz(h{yI(K{PX~f8wv^IC z=1|1Q87Q@2TN#y_s_%G}kr5qgs^99oWZO6Rv=YO7o2NtvA;ps!F_Q9*R( zOdxQD7?_!x&joxF`kRCY+Kx{hWu>Ih`J@K2f{YX@=zHAcF-F zD8KuyClEhH-SY(li?iEjzz`kzQM0wYe)Km=Z*^%FRyMy{{~k{go<70DpDJOmPG^Q^ zfnV0E*^i{Dk$-fjvhI75O=vklL@RbqcXeDP$1Hg zU0UKZ*e<2ZTAb(j;R$IQ+`943H{<4xeu}U%;cpWYRqfm_oUEEnBog-Vj~|@sfx_kq zqoWTmlp9XT_VYO)Mkp3 zwjf}_gSrMf`_4m=(0u*Ehl&mo%DCBoyIW{esCy?ocCEOtGN z_CJmUCzg`|e-MSrL1zyDp0p4a#N4<_X5EN(HU}e>6zm{azFm!XZ-x$`^}4fSdzMMUpq`l`~wtSMds!s{)U&>Js_y-&MUppeygI&iT|L zU!8MY!ZR_1?f6-whl1nC(ciL?0L$mb49U?2MNj>+Fg`sDis^o_dR#&_+k;`i(Q7e9 z%++9Cka9c7i$OK-`+xdC4uXwfi#tWC4%j6*r-0~3qH(-Pq{uM;xh^wAbqncQcAXApY;4 zn13G3@841Y8-l<8{4xvQi~nyQAV(yv#Op%5<_bQM>CAO~eSJeOkFVzV)+p^@lnykt z-x@nl>Eeo<6(?%DgWg5b($Rs1;!_DZcWgh2g#Ew&YSy+3ASF&fU~m)z8(V8zERhz$ zzYD^fsZeMIN_h?_4^l0lz)fX-+vx`FG(42#V z6~g&*PR9}yfxWiRDTYErP*@>^8N|@kJk{$AV3hzX`zu_SkdDnd0bpL1dLGrr;_xHr z`a8r)nh>k9k|B~UQ12j80T{xJRl{j375O+dH4-xW3>Y*)0lEoDqzEhZI?Xt`n1GlA z0>{VUI*cK}(*`g7?(XiiuDh-otV%3!%@r`&xw)l<+l58yOu%=uU`tg!Jvu~Yk952t z8-(%zpzg+Z_j*vI`A--bB{#iQZNPf~l0-}}ddLvmdyw25CHGPSxQ3te?+63bdtC}sfX6M*RtzhuWf5_^1wE1-|m@KEw=m!NKoyW$)*!#d)G_dIBtlDCHr_Bx>C2sIn0jgR2chg`x1ZPLY9AyqO z%OIR&ymSc<5e$MW=k43KYm$P3L+zGi`hcZ0+C=a3u)4asi$>2USgsLSa$WQ5$S4h{ z6_CMDuWlwukV-?h9q>3rKm-N5IOL{)ew`2>4R~Uiaw}63UIGz0c@S9BAYX*y^4PFM zI9i~Tl7&CwvKyDW%S#1i6S(g1Xxuc>gB6WDB(O(b24tM)AODmZ=8Q>qde&d zy6UaJSA_#Yo~BWthd4Q)Y8o0a5dsp#GuwkbH;__d!3-4a8a!gO8srhdl+G|Z5vs$z zRYE|5xQsrYx$WAll7Csn7b3Dx@Uj6M z?Rs}l1(_j|rBlLzVh|Cb1&jjL!9;PKR3y-%qr;V6_kmbZ0<3aD1BIDpE8T=R2SFM= z(HzZ$Y^-nZ?)k%@C`NAXXmaI3H`tyOJZ5;$l9OQun-9b}T!4!R8YAfm%XA|Eb^-O+ z1pI*%=z)=e8fCg2^w*kSC4)j3WM(xk-@Sr>eoRig1SNr<`;YUNz;dvwt80iLaP}Kaz`O9fNYm*3psIxIt3o=ySy1t*gn@dgc+{S6JGGTYZ<$t=n%MG|*LfGby$3Iz^nQK5J z7rwUU2y(f%vs=i5ND=Xncz!z#ainlrqyaA48>0ps7%}LZa5hQ(_HAD<UQmcf ziHV6JFvRrqIF$Ca!b=tYNSi5~e}v1WOv(%%=L*=f7(*Wepgw-H&J=up{*rXUctB_% zayP=#dJCV=Yx(u_5U2t)L1lai8I%NSYX*35`n|z)T;?4#75f{5vm=$l@P)onQACyl zg_uYXddjX7_y7YN_98k=O%$*lMZunHyK`A7VAWP-4vaCSKwx;JxLe-X;Lr}aZt{pu zHyjrQNDJEsC8xG{?pYW>bLNt!FSL?~Sy)&EoEIqk{QTN9xudu5z1OxJDkaX43HM7* zriCc;N-;HpT+bB^X4QdlPP~|pC?MO$4+8=WVblTaU_=_ih;HCYUM`dIK;k)|*#tvF zL(33a4d;5Z&GMtNjDUFC&=^ilOhKW!mVtG9tF{;77&alD6rjs*!4tb;xenJX_eqvf z_YX%{T3?II2|u_jMc7R=j+HM}@wjM7k=0cu1)xTuVcQ6=C>mgLa&q#?$;nYf*bTpf z3(Z=@uo-!{Kuc39GP}v6aZA-32M33Ugv1|0hKTcG<*Qy;^3VI{--5k!^OU{k&W}tO ztx^jJh=Glb%{%+h=0p+KJh$a)8+g*l;)CzgbIh<17e{+~Ny`rmBMd)1VcCLhoIqZ9 zgK%0Vv&V3vF+85j5FI&40b?o4qI6oqDu*e5#?Z3(3K8U6vDN*K?hsa*5jk0tiN!mr zOII@BsX{*!WpH?~f18;U4kI4(5Gk@ySoKGG$7e$hON(o8T%e=C=}LL=J+YRbhmO`4 zz-D3DB62Tom=Fkyx~46bIN^KcjPoaez(K`-7p86@%jn%ZUSzMsJRFup4-glA9T=!> zZEX!BBZ(~l8w7B<$beY01?>nWXs^n4#I;T#2dnq~x&{)(;aKpit`Oc91HBOJRHf8B?OQ?o0%NK*T%*Mg`oX8c)LL`)`2gh`PGvr)DU5zHGqUy zBuE|sJC_r3XJz_e#*x9s!T_>0AaP{B6r;43EEizea z*6WWOZZlF50w>5>IMonNiR?Y8Gxh!blKw2Nsba2U>nu zh3fgbAz+9_mK+Bw{|3BYYPGmaTF4|WKxQkh%3Ziz-aS1%(n09E5}rfNZ!PuSqK}A% zMwjCXiPmZB0a+7_MXP<~?(RM8883+Lh=&3w&k>t@JPIz9?}})T)R$Xw)<_6BFaMMQ z+`+&kL9*I=|5K4_I!?yBva+VGE@E(shg2&7xWsNxuB(ncd6MKIhK0V#+Lt~C=IPa z%+n51iVJ|nVnBdHX2hW!B#V(`5>is|Ye3x-G)7{ZlZi40h~a5#7xY9t_!anIpcu`|K6-3+TyI%z(V7CmJ5zmD4^1AGozMe zozlCfW%AU4D})YZwH#8AkjdY;Ar6cAjnAPs;w1tRo8tqQnW-r<_DB<4A;Fdbaa>W& z)9z~{BS}Dpf&3=)syc(UC!+gDUgU!Z4{TjrnjmDpz1ziz1fFS*$gDayRVEF{tDv5` z*3yY~zEB!u`~#!x!&2&U8C?b=6>dnT0>{fE@YcKr6sF7`@z;Hb9Sbziq{FGM5W$m% zm4L=v;qTw9{)bj7WhzT54f)w<*-T3PiyJYvu+0`mDyd;!lPOPV)(Hsd2oUbQJw*<_ zQW$U=)Im&rV?8W{Y-&FS>zAq2S6I(!~ASbZM0uKsOPK57>C2GSWDLN_cEy!XHvMbl+MGub&GzyIq@&9VsHX zY0KC|j5>%5@3G2)NF+tC0NY?Fm_5Ht6%s+jaSn=15NcsKTP6?SODgx<9XU?e3;J-m4cc3}EMd zEpxaC)i(-NP_@HK#cv%pI%JM$XR8So&?D z8U6Fmfufy|a0+s4aU!6gniSU2iV)IWJqczhP-s*mCxY-FsHiz!uYl7+=KcTMcC3X| z1t5S2EGn;{fNWrv-!qJ|Y@eViyLjsv*WF1RKUALjBPf2-WWuRXTNy$FbU-b=x`oHW zNc{?~rRN7+0yt!Xr~tNMsQ091g)28y!GUmdC9?#?j5@zxEh153f81h%By*2AtAf9N z)kO9qlEJ|^3|~+qoq)4f8fr&)ZExMzEdUKz4!Tz8M#yi!33C|1o5C2Ddi-Tm9K^2y zyrXh;%T8Ax?A-6-dt*8Im>rgF8-n{&Ux;ZLJ*Ec297{>HB!T9j1lT$-?%u9kk!4tJyhCk0^Qz&D9eFL88E4GI?r!U?@dP#odZf7Z! z{@0U--ZQw$lHa)vldj;DS#G4ZL5)VzNHzTFt2L{`vlZD`bbxIGemiY#ZRzZ6(n^;; zeFU5VnSloDD^9jW%B)G|m}26#OpO|7~BG&4NnJ02MUL}4A!Jfx%1CAt}8~WIjSwc z&ps0>aK;M0U1o2JScsu~CK*`S|BfZ_8ju4Qg4m+6&l}2h7z9gtC8IR&G%=hh|Jrq= z`&m`L_Z;douwDOe*O1OrZb05bNK!K z{aHJJYr(!qyTtk!g@)QJ0BMe+d??Pb-hc!Jc#uJm_JG`1iIf|`1y&lVQSr3t0FQv8+N#O#}5hD+oRgT_ObW$Xh^3j{^o>NspYV+1$Ou$5Bz& z;Mf43d0AKX@heX1jXL5)MXQ?v-~{8I1K`+E(bR;AHkqm3I5jbFD(k;G5(hHc_kg_Z zs6nJn2DN93>-ed;xjDpekScuer!(RjTu{IXsf&h}zMUO+NJz+#+f4F_6DK|cs)|KG za~+Hf$vO2b?Qglc3EsGI1CK@^3J#9(`FVK|C6_H#?IPqr1n;13zacMQZ}j$aw#vnd z%F0s&1g`*~y}G~F^CB~oKeV&0Z4$Z$CcxZ*E5-x~E~M}<#S9tl&Cw4H6DZ6akhVZ) zXCxqvaN9Ld)o7JFvtNh_feega_z?0B21Z6dutA=fn2-dZAmOUpFSY7<*3_rHY%DCN zVp=|e$yaj9=3HOv54|_j-jLu>O(8irU_Wli%us+{pdx_Eu*iN&Zozr?N}l}DvpE=d z${;yz|818vG&UAOnf;|ezY}(Pa$^-FE4k`y5RU6C`N};Np@;8g@qzF_wH$UIf z-hQ5+KMn$3A(SL=TQm-G^76>DfvOLKgTEQZ&~^?KnFEZ82Z-b-LXc z2!RpR7#J93t}Be^&!2~G$g_#2NaMGfG!uMD?b(LCRbF6EQOR=Yk_q4-FJ8U`K3W}^ za}M6U38Sse!NnSBprCwwe86bl6(H}gRbIlJ70VJu>u&~sLbVpq+pC&sLrz8pq`F3O zeNTj<)Bwk=?w9qY{Fi#|Wm8Xevhwq-?Q@HZpF$A2(1f5NKn|;e@*V|#rD=tQj~5m! zA#u5J^X7RGkz^oUmsN5cEm^x)wErx2Ji4p#;F@MP*-R`u@lw3r&I{^Dj_=R`xMpH< z4jk5Ct@^{FW@Tfe4G#r`GdeD=Nkvg)iq{5mHej7~mAlw@Z;fFuI17_==%(f5gaYc9 z0O$assmNSH0CZP(cjVPTV}pgA{pE)biEwfhLT5!Lg678>J`DMz^H|_U-WNQ77B;qe zvm(3cHcG(nx$*pe^kh0ffgm&Q4sePf5N|=Ij5yK`(PrkYJzjSAFvrKo@oJC; z4qiPM0+Ueqa)aZutip{OAy9`I$G4kheE9GUAbc(q2G^}r(VuHoVCaIu^*AObX>SC8 zuQ0%|F}RR#?t})C4XP1Puc1%ESFZG-@E$dEcc15H6BQjng~1aQ5b5mfbjjTWsO%mu zC)iekyCD~9;sG|@UEY4En_%#z3_wRQ-|u{I%toVt1?eK(|MW{M7oPE0T>zHB$A*S7 zvTE2v7+jDwh5_~`5ROkTI)n;S6x)B-QU;#HywE(vgNQQ$kIQ-n#zn(V$Gd-G(Mymt zo`(Gj$Qd^tMyV%n(y;YKAt5S#{qJTo>gwu+;0g+>J^@Dh#&DYzB_QV~Qtvmd!c;hn z+nSoQ>M8gukRxYQRgL5cYOH>iBt%2Ng3K8JzA+GA)rk0NYTmm<#*Jb3ekH@!sprBo zGB*AJ^*{_a{Gh$Zl*jCwwMo`$@snVC?8)!wK-Ata5##tVRUF|n~d;<)oc!1=}< z7GB;b%geS9IJMuc9!HF4KL~yY?O14V)M5}`y#}2f=v^Yd^49eIE?`<`VPUbnwl*0d zym*(d!>kC+c4zHg6VIat3D9H!kT3;srj)caF&$k*b+u2-5(S?n1>EYhXEflc+YZfQ z3Ss9)@~f`0x!}@n}KDp6)#Xp#r!`ia@)*R5+o9rV|w|0Gf3ZI{x6)J_@$-O+7s^ zV9WwVg`}n?1uzAicJ!pA;2JTZ@a1y*Wdh9Eg)NWe(b`LQKJ>DP3JNAbwQ!n(q8We` zW)_xOvmzy3T?Hj2VuW^JOQW`Bb2qC6`tCzHH$a;@7O?QpUxC{)vwROX(6iYo1K^M1gIWnI_%(VxxeEKtq{m` zr=j8f|8#ccPfcD?^h0bdb{xwf)|Np>I>vF@fie|rWO1klFc3gcHj9b^vRKvtqZMR? zg21pTlqCTHAs}E_YJ^Z$3nBy%3rs+^6mTjHAti!TXlc(&;y=*u2PVlt^6~QCz2}{C z?|B`80tL9pVR$g7q^2fkWQ6PrG38vvNqu7tz8^9e4Dz8kv(Fm_@d^>CZB8G<@KApa zn#rN{9Cvcc9R3;USM1{e7!xPmD8LvNT%UHj<5J-ubYB$&1+;#96@uerh2!1oK_PUOWrd=Y$K&;GWPA2OuRvo8 z#(zf#l!MBus&M+{p<*jg!pT4g&Tei!)2-j@UjjK9Hb0&%Z~wQg_jl-|r1GYETuwk4 z%Mv+Zn`9!SJu-i-{Ws4)MlT>{^cR|a9ET>wJoLC7(VM8_fg+8-ZQjTeD2KKQUD(P8 z4+D3n{s77XyM}Jm7EcunL(JG4vvP_ex^-^!Hq1A=th8Z5Ht=M9V?#4HBc=NOePLh= zBw0&KOX{?nn^_TDQTE0mN?Jz{5eTSaW_|-T0UE?80i9SOhl@BI5p=Qwt{&y@JnAhxJM_h9!3ygoy80oj28AWWi}}ZuYx$wm~(> z0ru|zC zXli&X*SmiKWAcu$HCGmE2Rz9AK;%wFO?($zt8@=L@#*X4=UU1y_SWA}43K){y0$Vo`pih7{OC?|NaSP(3Hxf`%1-z4bAx!5WcqA1|u zdb_YN4JNP@+T(#)M50vCO(-nt)NwSof?!iwQ_~gP7=c)%qpRC(voEF#QnoZI?JTkc z5(&Z|=O3XWuLGX9WLbi+>50?Ogs9JJY2l)G9kgjT8c}*hqM%ba#uwLYzt3SV0~-)> z9ccw=fD!Gfu5BX|Hkf@>`TDO+|0GrHxJ41SmPf%;+a>JF$|lQOPmh(W1ItrZS#fb3 z&RGI$>3HUhE@fh3LJBni3=>mQwlB4=om;}J(c`Lid=F_y8&oyY^R(N!xi(l``ZqWO zegKyELn$Mb*#$&o6B5+%>XA;_GiD Pr*NHQIGo^rbK&NH;E`m0 literal 0 HcmV?d00001 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.csv b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.csv new file mode 100644 index 0000000000..41be4689fd --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.csv @@ -0,0 +1,31 @@ +operation,message_bytes,mean_GBps,min_GBps,max_GBps,baseline_GBps,change_percent,mean_wall_us,mean_transfer_us,min_p99_us,max_p99_us,min_p999_us,max_p999_us,measured_operations,measured_bytes +read,4096,0.4651459108747723,0.4575730796853841,0.4693601706398294,0.5003561378674665,-7.037033090622458,8.807020598308707,7.969522096093298,10.0,11.0,17.0,20.0,1703416,6977191936 +read,8192,0.9298092890276466,0.9221881856,0.941345620261752,0.9812618331638756,-5.2435081440323605,8.811106629409931,8.013249334778237,10.0,10.0,11.0,13.0,1702532,13947142144 +read,16384,1.7782468222725647,1.7686706921293078,1.783895403262758,1.8692500605464117,-4.868435753707844,9.213704500284031,8.118648933961119,10.0,10.0,12.0,13.0,1628035,26673725440 +read,32768,3.387195766571659,3.3589021572391373,3.4131252566997947,3.580697534218389,-5.404024377863803,9.674495168764858,8.723271754880093,10.0,11.0,13.0,27.0,1550536,50807963648 +read,65536,6.195293280248187,6.130367285926543,6.278296921806157,6.6105201214267755,-6.2813036425486075,10.579399178296963,9.470716866760784,12.0,13.0,35.0,42.0,1417992,92929523712 +read,131072,10.189972014228323,10.045166543066765,10.295315948704912,11.043801933350329,-7.731304167485931,12.8642379248727,11.83605650780262,15.0,15.0,18.0,20.0,1166151,152849743872 +read,262144,15.57920126418906,15.069092917521248,16.066298554943177,17.50306697106177,-10.99159198815547,16.8380823099914,15.63002464916631,17.0,19.0,20.0,26.0,891451,233688530944 +read,524288,20.956637275895396,20.906969855090434,21.031804707278116,22.964354957694546,-8.742756700537912,25.01792040630208,23.461093222246696,32.0,32.0,36.0,37.0,599575,314349977600 +read,1048576,28.261218819330843,28.23558427354831,28.306337674638883,30.335508406330323,-6.837827008584519,37.10304747423089,35.16696497616981,51.0,51.0,54.0,59.0,404282,423920402432 +read,2097152,29.45053827868747,29.396193135161372,29.47893375690218,30.524125696395707,-3.5171766372165236,71.20941411700885,68.56419255173735,87.0,88.0,94.0,98.0,210648,441760874496 +read,4194304,34.995892438855876,34.9799286851553,35.00656876108726,35.80223107745133,-2.2522022073180215,119.85133570670291,117.14410482338248,156.0,157.0,163.0,164.0,125157,524946505728 +read,8388608,38.708376605944316,38.6974209059508,38.724906641834835,39.19519339086779,-1.2420318483156167,216.71300961960713,213.98986482079127,251.0699999999306,296.3199999999997,303.0,303.0,69217,580634279936 +read,16777216,40.93107844401615,40.91943341071963,40.943402614403915,41.22154820968126,-0.7046551579954707,409.8894433999424,407.1583855026053,427.92000000006374,485.75999999997293,583.0,585.0,36597,613995773952 +read,33554432,41.46821969757634,41.20611417532886,41.96138510419456,41.55179880961917,-0.20114438950230573,809.2171413991467,806.4522700682589,820.5599999999995,1051.0000000001128,1144.0,1191.8560000000007,18539,622065614848 +read,67108864,41.750739333874336,41.727932774978136,41.76582546861435,41.823256240512926,-0.17338895427359002,1607.3697240755273,1604.6608439839226,1613.8400000000001,2268.709999999906,2360.0,2365.0,9333,626327027712 +write,4096,0.46760191904105364,0.4648625811923864,0.47117176805646394,0.5015755508396647,-6.773382741989176,8.759866576846948,7.819110565833217,10.0,10.0,14.0,19.0,1712412,7014039552 +write,8192,0.9404962554968604,0.9352708096000001,0.946777795066646,0.9906987270880243,-5.067380245730679,8.710517631427457,7.937192903774412,10.0,10.0,12.0,12.0,1722101,14107451392 +write,16384,1.8258288274562113,1.815263101684278,1.8398423489891813,1.9253342860587188,-5.168217245338802,8.973745227265733,8.101603920850668,10.0,10.0,12.0,12.0,1671598,27387461632 +write,32768,3.450667585516795,3.435800007399992,3.4631240163503936,3.6564099802635868,-5.626896213973243,9.496235144823347,8.39270974167247,10.0,11.0,17.570000000065193,22.0,1579591,51760037888 +write,65536,6.314624047241013,6.271210957746851,6.387091573890111,6.728749591086833,-6.1545691103498275,10.37913710049642,9.295521309054047,12.0,12.0,35.0,42.0,1445304,94719442944 +write,131072,10.182644845275183,10.170130487895609,10.196167483966503,11.10367177332039,-8.294796053484088,12.87211185730185,11.844358685238257,15.0,15.0,18.0,21.0,1165312,152739774464 +write,262144,15.362185238892762,15.148351982259207,15.57279046081908,17.075179699210093,-10.032072812660209,17.066411441323975,15.764241398968492,17.0,19.0,21.0,22.0,879032,230432964608 +write,524288,21.271503709129277,21.010883424949725,21.487849133463815,23.013176365697383,-7.5681541256694125,24.649560193809325,23.120335548486846,27.0,32.0,32.0,37.0,608585,319073812480 +write,1048576,28.181134574397767,27.856722306066164,28.499240224974287,29.7657023669789,-5.3234685109900255,37.21166410270688,35.2862103936608,56.0,58.0,60.0,60.0,403137,422719782912 +write,2097152,29.844568908789565,29.720204728125957,29.951099443380112,30.969740273355836,-3.633131420008351,70.26984777286876,67.59738659117247,99.0,99.0,103.0,103.0,213466,447670648832 +write,4194304,34.61660395945932,34.49596903725205,34.703608829504105,35.42831182874708,-2.2911277094189053,121.16529892746641,118.4634455655462,176.77999999999884,178.0,183.0,184.0,123800,519254835200 +write,8388608,37.682958879825634,37.59786222610384,37.76248359748721,38.23022561757633,-1.431502767535564,222.61083071158532,219.90022377255747,257.5200000000186,336.0,342.0,344.0,67385,565266350080 +write,16777216,39.43200484696949,39.317007479605074,39.497670504146726,39.780574458980695,-0.8762307149953963,425.47387567610497,422.7827926687437,457.509999999982,658.0,662.0,911.1130000000139,35257,591514304512 +write,33554432,40.07214159233397,40.06262967859092,40.07759983394703,40.37462545607061,-0.7491929902006311,837.3506286148097,834.60058353179,1000.9800000000632,1280.29,1290.0,1291.0,17915,601127649280 +write,67108864,40.15150974378873,40.14541884691976,40.15543048765147,40.218340715051085,-0.16617038414353802,1671.3908199643495,1668.6951871657755,1800.1800000001313,2018.5800000000963,2630.0090000000005,2636.0,8976,602369163264 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.json b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.json new file mode 100644 index 0000000000..d97c7cec84 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/bandwidth-summary.json @@ -0,0 +1,512 @@ +[ + { + "operation": "read", + "message_bytes": 4096, + "mean_GBps": 0.4651459108747723, + "min_GBps": 0.4575730796853841, + "max_GBps": 0.4693601706398294, + "baseline_GBps": 0.5003561378674665, + "change_percent": -7.037033090622458, + "mean_wall_us": 8.807020598308707, + "mean_transfer_us": 7.969522096093298, + "min_p99_us": 10.0, + "max_p99_us": 11.0, + "min_p999_us": 17.0, + "max_p999_us": 20.0, + "measured_operations": 1703416, + "measured_bytes": 6977191936 + }, + { + "operation": "read", + "message_bytes": 8192, + "mean_GBps": 0.9298092890276466, + "min_GBps": 0.9221881856, + "max_GBps": 0.941345620261752, + "baseline_GBps": 0.9812618331638756, + "change_percent": -5.2435081440323605, + "mean_wall_us": 8.811106629409931, + "mean_transfer_us": 8.013249334778237, + "min_p99_us": 10.0, + "max_p99_us": 10.0, + "min_p999_us": 11.0, + "max_p999_us": 13.0, + "measured_operations": 1702532, + "measured_bytes": 13947142144 + }, + { + "operation": "read", + "message_bytes": 16384, + "mean_GBps": 1.7782468222725647, + "min_GBps": 1.7686706921293078, + "max_GBps": 1.783895403262758, + "baseline_GBps": 1.8692500605464117, + "change_percent": -4.868435753707844, + "mean_wall_us": 9.213704500284031, + "mean_transfer_us": 8.118648933961119, + "min_p99_us": 10.0, + "max_p99_us": 10.0, + "min_p999_us": 12.0, + "max_p999_us": 13.0, + "measured_operations": 1628035, + "measured_bytes": 26673725440 + }, + { + "operation": "read", + "message_bytes": 32768, + "mean_GBps": 3.387195766571659, + "min_GBps": 3.3589021572391373, + "max_GBps": 3.4131252566997947, + "baseline_GBps": 3.580697534218389, + "change_percent": -5.404024377863803, + "mean_wall_us": 9.674495168764858, + "mean_transfer_us": 8.723271754880093, + "min_p99_us": 10.0, + "max_p99_us": 11.0, + "min_p999_us": 13.0, + "max_p999_us": 27.0, + "measured_operations": 1550536, + "measured_bytes": 50807963648 + }, + { + "operation": "read", + "message_bytes": 65536, + "mean_GBps": 6.195293280248187, + "min_GBps": 6.130367285926543, + "max_GBps": 6.278296921806157, + "baseline_GBps": 6.6105201214267755, + "change_percent": -6.2813036425486075, + "mean_wall_us": 10.579399178296963, + "mean_transfer_us": 9.470716866760784, + "min_p99_us": 12.0, + "max_p99_us": 13.0, + "min_p999_us": 35.0, + "max_p999_us": 42.0, + "measured_operations": 1417992, + "measured_bytes": 92929523712 + }, + { + "operation": "read", + "message_bytes": 131072, + "mean_GBps": 10.189972014228323, + "min_GBps": 10.045166543066765, + "max_GBps": 10.295315948704912, + "baseline_GBps": 11.043801933350329, + "change_percent": -7.731304167485931, + "mean_wall_us": 12.8642379248727, + "mean_transfer_us": 11.83605650780262, + "min_p99_us": 15.0, + "max_p99_us": 15.0, + "min_p999_us": 18.0, + "max_p999_us": 20.0, + "measured_operations": 1166151, + "measured_bytes": 152849743872 + }, + { + "operation": "read", + "message_bytes": 262144, + "mean_GBps": 15.57920126418906, + "min_GBps": 15.069092917521248, + "max_GBps": 16.066298554943177, + "baseline_GBps": 17.50306697106177, + "change_percent": -10.99159198815547, + "mean_wall_us": 16.8380823099914, + "mean_transfer_us": 15.63002464916631, + "min_p99_us": 17.0, + "max_p99_us": 19.0, + "min_p999_us": 20.0, + "max_p999_us": 26.0, + "measured_operations": 891451, + "measured_bytes": 233688530944 + }, + { + "operation": "read", + "message_bytes": 524288, + "mean_GBps": 20.956637275895396, + "min_GBps": 20.906969855090434, + "max_GBps": 21.031804707278116, + "baseline_GBps": 22.964354957694546, + "change_percent": -8.742756700537912, + "mean_wall_us": 25.01792040630208, + "mean_transfer_us": 23.461093222246696, + "min_p99_us": 32.0, + "max_p99_us": 32.0, + "min_p999_us": 36.0, + "max_p999_us": 37.0, + "measured_operations": 599575, + "measured_bytes": 314349977600 + }, + { + "operation": "read", + "message_bytes": 1048576, + "mean_GBps": 28.261218819330843, + "min_GBps": 28.23558427354831, + "max_GBps": 28.306337674638883, + "baseline_GBps": 30.335508406330323, + "change_percent": -6.837827008584519, + "mean_wall_us": 37.10304747423089, + "mean_transfer_us": 35.16696497616981, + "min_p99_us": 51.0, + "max_p99_us": 51.0, + "min_p999_us": 54.0, + "max_p999_us": 59.0, + "measured_operations": 404282, + "measured_bytes": 423920402432 + }, + { + "operation": "read", + "message_bytes": 2097152, + "mean_GBps": 29.45053827868747, + "min_GBps": 29.396193135161372, + "max_GBps": 29.47893375690218, + "baseline_GBps": 30.524125696395707, + "change_percent": -3.5171766372165236, + "mean_wall_us": 71.20941411700885, + "mean_transfer_us": 68.56419255173735, + "min_p99_us": 87.0, + "max_p99_us": 88.0, + "min_p999_us": 94.0, + "max_p999_us": 98.0, + "measured_operations": 210648, + "measured_bytes": 441760874496 + }, + { + "operation": "read", + "message_bytes": 4194304, + "mean_GBps": 34.995892438855876, + "min_GBps": 34.9799286851553, + "max_GBps": 35.00656876108726, + "baseline_GBps": 35.80223107745133, + "change_percent": -2.2522022073180215, + "mean_wall_us": 119.85133570670291, + "mean_transfer_us": 117.14410482338248, + "min_p99_us": 156.0, + "max_p99_us": 157.0, + "min_p999_us": 163.0, + "max_p999_us": 164.0, + "measured_operations": 125157, + "measured_bytes": 524946505728 + }, + { + "operation": "read", + "message_bytes": 8388608, + "mean_GBps": 38.708376605944316, + "min_GBps": 38.6974209059508, + "max_GBps": 38.724906641834835, + "baseline_GBps": 39.19519339086779, + "change_percent": -1.2420318483156167, + "mean_wall_us": 216.71300961960713, + "mean_transfer_us": 213.98986482079127, + "min_p99_us": 251.0699999999306, + "max_p99_us": 296.3199999999997, + "min_p999_us": 303.0, + "max_p999_us": 303.0, + "measured_operations": 69217, + "measured_bytes": 580634279936 + }, + { + "operation": "read", + "message_bytes": 16777216, + "mean_GBps": 40.93107844401615, + "min_GBps": 40.91943341071963, + "max_GBps": 40.943402614403915, + "baseline_GBps": 41.22154820968126, + "change_percent": -0.7046551579954707, + "mean_wall_us": 409.8894433999424, + "mean_transfer_us": 407.1583855026053, + "min_p99_us": 427.92000000006374, + "max_p99_us": 485.75999999997293, + "min_p999_us": 583.0, + "max_p999_us": 585.0, + "measured_operations": 36597, + "measured_bytes": 613995773952 + }, + { + "operation": "read", + "message_bytes": 33554432, + "mean_GBps": 41.46821969757634, + "min_GBps": 41.20611417532886, + "max_GBps": 41.96138510419456, + "baseline_GBps": 41.55179880961917, + "change_percent": -0.20114438950230573, + "mean_wall_us": 809.2171413991467, + "mean_transfer_us": 806.4522700682589, + "min_p99_us": 820.5599999999995, + "max_p99_us": 1051.0000000001128, + "min_p999_us": 1144.0, + "max_p999_us": 1191.8560000000007, + "measured_operations": 18539, + "measured_bytes": 622065614848 + }, + { + "operation": "read", + "message_bytes": 67108864, + "mean_GBps": 41.750739333874336, + "min_GBps": 41.727932774978136, + "max_GBps": 41.76582546861435, + "baseline_GBps": 41.823256240512926, + "change_percent": -0.17338895427359002, + "mean_wall_us": 1607.3697240755273, + "mean_transfer_us": 1604.6608439839226, + "min_p99_us": 1613.8400000000001, + "max_p99_us": 2268.709999999906, + "min_p999_us": 2360.0, + "max_p999_us": 2365.0, + "measured_operations": 9333, + "measured_bytes": 626327027712 + }, + { + "operation": "write", + "message_bytes": 4096, + "mean_GBps": 0.46760191904105364, + "min_GBps": 0.4648625811923864, + "max_GBps": 0.47117176805646394, + "baseline_GBps": 0.5015755508396647, + "change_percent": -6.773382741989176, + "mean_wall_us": 8.759866576846948, + "mean_transfer_us": 7.819110565833217, + "min_p99_us": 10.0, + "max_p99_us": 10.0, + "min_p999_us": 14.0, + "max_p999_us": 19.0, + "measured_operations": 1712412, + "measured_bytes": 7014039552 + }, + { + "operation": "write", + "message_bytes": 8192, + "mean_GBps": 0.9404962554968604, + "min_GBps": 0.9352708096000001, + "max_GBps": 0.946777795066646, + "baseline_GBps": 0.9906987270880243, + "change_percent": -5.067380245730679, + "mean_wall_us": 8.710517631427457, + "mean_transfer_us": 7.937192903774412, + "min_p99_us": 10.0, + "max_p99_us": 10.0, + "min_p999_us": 12.0, + "max_p999_us": 12.0, + "measured_operations": 1722101, + "measured_bytes": 14107451392 + }, + { + "operation": "write", + "message_bytes": 16384, + "mean_GBps": 1.8258288274562113, + "min_GBps": 1.815263101684278, + "max_GBps": 1.8398423489891813, + "baseline_GBps": 1.9253342860587188, + "change_percent": -5.168217245338802, + "mean_wall_us": 8.973745227265733, + "mean_transfer_us": 8.101603920850668, + "min_p99_us": 10.0, + "max_p99_us": 10.0, + "min_p999_us": 12.0, + "max_p999_us": 12.0, + "measured_operations": 1671598, + "measured_bytes": 27387461632 + }, + { + "operation": "write", + "message_bytes": 32768, + "mean_GBps": 3.450667585516795, + "min_GBps": 3.435800007399992, + "max_GBps": 3.4631240163503936, + "baseline_GBps": 3.6564099802635868, + "change_percent": -5.626896213973243, + "mean_wall_us": 9.496235144823347, + "mean_transfer_us": 8.39270974167247, + "min_p99_us": 10.0, + "max_p99_us": 11.0, + "min_p999_us": 17.570000000065193, + "max_p999_us": 22.0, + "measured_operations": 1579591, + "measured_bytes": 51760037888 + }, + { + "operation": "write", + "message_bytes": 65536, + "mean_GBps": 6.314624047241013, + "min_GBps": 6.271210957746851, + "max_GBps": 6.387091573890111, + "baseline_GBps": 6.728749591086833, + "change_percent": -6.1545691103498275, + "mean_wall_us": 10.37913710049642, + "mean_transfer_us": 9.295521309054047, + "min_p99_us": 12.0, + "max_p99_us": 12.0, + "min_p999_us": 35.0, + "max_p999_us": 42.0, + "measured_operations": 1445304, + "measured_bytes": 94719442944 + }, + { + "operation": "write", + "message_bytes": 131072, + "mean_GBps": 10.182644845275183, + "min_GBps": 10.170130487895609, + "max_GBps": 10.196167483966503, + "baseline_GBps": 11.10367177332039, + "change_percent": -8.294796053484088, + "mean_wall_us": 12.87211185730185, + "mean_transfer_us": 11.844358685238257, + "min_p99_us": 15.0, + "max_p99_us": 15.0, + "min_p999_us": 18.0, + "max_p999_us": 21.0, + "measured_operations": 1165312, + "measured_bytes": 152739774464 + }, + { + "operation": "write", + "message_bytes": 262144, + "mean_GBps": 15.362185238892762, + "min_GBps": 15.148351982259207, + "max_GBps": 15.57279046081908, + "baseline_GBps": 17.075179699210093, + "change_percent": -10.032072812660209, + "mean_wall_us": 17.066411441323975, + "mean_transfer_us": 15.764241398968492, + "min_p99_us": 17.0, + "max_p99_us": 19.0, + "min_p999_us": 21.0, + "max_p999_us": 22.0, + "measured_operations": 879032, + "measured_bytes": 230432964608 + }, + { + "operation": "write", + "message_bytes": 524288, + "mean_GBps": 21.271503709129277, + "min_GBps": 21.010883424949725, + "max_GBps": 21.487849133463815, + "baseline_GBps": 23.013176365697383, + "change_percent": -7.5681541256694125, + "mean_wall_us": 24.649560193809325, + "mean_transfer_us": 23.120335548486846, + "min_p99_us": 27.0, + "max_p99_us": 32.0, + "min_p999_us": 32.0, + "max_p999_us": 37.0, + "measured_operations": 608585, + "measured_bytes": 319073812480 + }, + { + "operation": "write", + "message_bytes": 1048576, + "mean_GBps": 28.181134574397767, + "min_GBps": 27.856722306066164, + "max_GBps": 28.499240224974287, + "baseline_GBps": 29.7657023669789, + "change_percent": -5.3234685109900255, + "mean_wall_us": 37.21166410270688, + "mean_transfer_us": 35.2862103936608, + "min_p99_us": 56.0, + "max_p99_us": 58.0, + "min_p999_us": 60.0, + "max_p999_us": 60.0, + "measured_operations": 403137, + "measured_bytes": 422719782912 + }, + { + "operation": "write", + "message_bytes": 2097152, + "mean_GBps": 29.844568908789565, + "min_GBps": 29.720204728125957, + "max_GBps": 29.951099443380112, + "baseline_GBps": 30.969740273355836, + "change_percent": -3.633131420008351, + "mean_wall_us": 70.26984777286876, + "mean_transfer_us": 67.59738659117247, + "min_p99_us": 99.0, + "max_p99_us": 99.0, + "min_p999_us": 103.0, + "max_p999_us": 103.0, + "measured_operations": 213466, + "measured_bytes": 447670648832 + }, + { + "operation": "write", + "message_bytes": 4194304, + "mean_GBps": 34.61660395945932, + "min_GBps": 34.49596903725205, + "max_GBps": 34.703608829504105, + "baseline_GBps": 35.42831182874708, + "change_percent": -2.2911277094189053, + "mean_wall_us": 121.16529892746641, + "mean_transfer_us": 118.4634455655462, + "min_p99_us": 176.77999999999884, + "max_p99_us": 178.0, + "min_p999_us": 183.0, + "max_p999_us": 184.0, + "measured_operations": 123800, + "measured_bytes": 519254835200 + }, + { + "operation": "write", + "message_bytes": 8388608, + "mean_GBps": 37.682958879825634, + "min_GBps": 37.59786222610384, + "max_GBps": 37.76248359748721, + "baseline_GBps": 38.23022561757633, + "change_percent": -1.431502767535564, + "mean_wall_us": 222.61083071158532, + "mean_transfer_us": 219.90022377255747, + "min_p99_us": 257.5200000000186, + "max_p99_us": 336.0, + "min_p999_us": 342.0, + "max_p999_us": 344.0, + "measured_operations": 67385, + "measured_bytes": 565266350080 + }, + { + "operation": "write", + "message_bytes": 16777216, + "mean_GBps": 39.43200484696949, + "min_GBps": 39.317007479605074, + "max_GBps": 39.497670504146726, + "baseline_GBps": 39.780574458980695, + "change_percent": -0.8762307149953963, + "mean_wall_us": 425.47387567610497, + "mean_transfer_us": 422.7827926687437, + "min_p99_us": 457.509999999982, + "max_p99_us": 658.0, + "min_p999_us": 662.0, + "max_p999_us": 911.1130000000139, + "measured_operations": 35257, + "measured_bytes": 591514304512 + }, + { + "operation": "write", + "message_bytes": 33554432, + "mean_GBps": 40.07214159233397, + "min_GBps": 40.06262967859092, + "max_GBps": 40.07759983394703, + "baseline_GBps": 40.37462545607061, + "change_percent": -0.7491929902006311, + "mean_wall_us": 837.3506286148097, + "mean_transfer_us": 834.60058353179, + "min_p99_us": 1000.9800000000632, + "max_p99_us": 1280.29, + "min_p999_us": 1290.0, + "max_p999_us": 1291.0, + "measured_operations": 17915, + "measured_bytes": 601127649280 + }, + { + "operation": "write", + "message_bytes": 67108864, + "mean_GBps": 40.15150974378873, + "min_GBps": 40.14541884691976, + "max_GBps": 40.15543048765147, + "baseline_GBps": 40.218340715051085, + "change_percent": -0.16617038414353802, + "mean_wall_us": 1671.3908199643495, + "mean_transfer_us": 1668.6951871657755, + "min_p99_us": 1800.1800000001313, + "max_p99_us": 2018.5800000000963, + "min_p999_us": 2630.0090000000005, + "max_p999_us": 2636.0, + "measured_operations": 8976, + "measured_bytes": 602369163264 + } +] diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/commit.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/commit.txt new file mode 100644 index 0000000000..f75798842c --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/commit.txt @@ -0,0 +1 @@ +fe0d23e332f41c7d54b5c217194dce5f8e751b21 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/endpoint.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/endpoint.txt new file mode 100644 index 0000000000..063a69eb87 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/endpoint.txt @@ -0,0 +1 @@ +10.0.3.49:15309 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-025.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-025.txt new file mode 100644 index 0000000000..e3990cc825 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-025.txt @@ -0,0 +1,71 @@ +slurm-h100-206-025 +CUDA_VISIBLE_DEVICES=0 +GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-f1e69e6a-387d-ad42-fb66-38e93e13e79e) + GPU0 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 NIC9 NIC10 NIC11 NIC12 NIC13 NIC14 NIC15 NIC16 CPU Affinity NUMA Affinity GPU NUMA ID +GPU0 X PIX NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE 0,2,4,6,64 0 N/A +NIC0 PIX X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC1 NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC2 NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC3 NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC4 NODE NODE NODE NODE NODE X NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX PIX PIX +NIC5 NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC6 NODE NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC7 NODE NODE NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC8 NODE NODE NODE NODE NODE PIX NODE NODE NODE X PIX PIX PIX PIX PIX PIX PIX PIX +NIC9 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX X PIX PIX PIX PIX PIX PIX PIX +NIC10 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX X PIX PIX PIX PIX PIX PIX +NIC11 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX X PIX PIX PIX PIX PIX +NIC12 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX X PIX PIX PIX PIX +NIC13 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX X PIX PIX PIX +NIC14 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX X PIX PIX +NIC15 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX X PIX +NIC16 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX PIX X + +Legend: + + X = Self + SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) + NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node + PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) + PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) + PIX = Connection traversing at most a single PCIe bridge + NV# = Connection traversing a bonded set of # NVLinks + +NIC Legend: + + NIC0: ibp0 + NIC1: ibp1 + NIC2: ibp2 + NIC3: ibp3 + NIC4: ibp4 + NIC5: ibp5 + NIC6: ibp6 + NIC7: ibp7 + NIC8: mlx5_0 + NIC9: mlx5_1 + NIC10: mlx5_2 + NIC11: mlx5_3 + NIC12: mlx5_4 + NIC13: mlx5_5 + NIC14: mlx5_6 + NIC15: mlx5_7 + NIC16: mlx5_8 + +ibp0 +ibp1 +ibp2 +ibp3 +ibp4 +ibp5 +ibp6 +ibp7 +mlx5_0 +mlx5_1 +mlx5_2 +mlx5_3 +mlx5_4 +mlx5_5 +mlx5_6 +mlx5_7 +mlx5_8 +unlimited diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-035.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-035.txt new file mode 100644 index 0000000000..8090c44d0c --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/host-slurm-h100-206-035.txt @@ -0,0 +1,71 @@ +slurm-h100-206-035 +CUDA_VISIBLE_DEVICES=0 +GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-1db02bb9-5b9f-7543-9c82-5650222fd182) + GPU0 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 NIC9 NIC10 NIC11 NIC12 NIC13 NIC14 NIC15 NIC16 CPU Affinity NUMA Affinity GPU NUMA ID +GPU0 X PIX NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE 0,2,4,6,64 0 N/A +NIC0 PIX X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC1 NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC2 NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC3 NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC4 NODE NODE NODE NODE NODE X NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX PIX PIX +NIC5 NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC6 NODE NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC7 NODE NODE NODE NODE NODE NODE NODE NODE X NODE NODE NODE NODE NODE NODE NODE NODE NODE +NIC8 NODE NODE NODE NODE NODE PIX NODE NODE NODE X PIX PIX PIX PIX PIX PIX PIX PIX +NIC9 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX X PIX PIX PIX PIX PIX PIX PIX +NIC10 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX X PIX PIX PIX PIX PIX PIX +NIC11 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX X PIX PIX PIX PIX PIX +NIC12 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX X PIX PIX PIX PIX +NIC13 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX X PIX PIX PIX +NIC14 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX X PIX PIX +NIC15 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX X PIX +NIC16 NODE NODE NODE NODE NODE PIX NODE NODE NODE PIX PIX PIX PIX PIX PIX PIX PIX X + +Legend: + + X = Self + SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) + NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node + PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) + PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) + PIX = Connection traversing at most a single PCIe bridge + NV# = Connection traversing a bonded set of # NVLinks + +NIC Legend: + + NIC0: ibp0 + NIC1: ibp1 + NIC2: ibp2 + NIC3: ibp3 + NIC4: ibp4 + NIC5: ibp5 + NIC6: ibp6 + NIC7: ibp7 + NIC8: mlx5_0 + NIC9: mlx5_1 + NIC10: mlx5_2 + NIC11: mlx5_3 + NIC12: mlx5_4 + NIC13: mlx5_5 + NIC14: mlx5_6 + NIC15: mlx5_7 + NIC16: mlx5_8 + +ibp0 +ibp1 +ibp2 +ibp3 +ibp4 +ibp5 +ibp6 +ibp7 +mlx5_0 +mlx5_1 +mlx5_2 +mlx5_3 +mlx5_4 +mlx5_5 +mlx5_6 +mlx5_7 +mlx5_8 +unlimited diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/job.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/job.log new file mode 100644 index 0000000000..477243ad1b --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/job.log @@ -0,0 +1,10 @@ +Results: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R +Initiator: slurm-h100-206-025; target: slurm-h100-206-035; RDMA VRAM +Transfer event traces: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events +Sweep: read, repeat 1/3, 4096..67108864 bytes +Sweep: write, repeat 1/3, 4096..67108864 bytes +Sweep: read, repeat 2/3, 4096..67108864 bytes +Sweep: write, repeat 2/3, 4096..67108864 bytes +Sweep: read, repeat 3/3, 4096..67108864 bytes +Sweep: write, repeat 3/3, 4096..67108864 bytes +Completed all sweeps. JSON results and logs: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.command.txt new file mode 100644 index 0000000000..4bd54fc14a --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=read --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/read-1.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.jsonl new file mode 100644 index 0000000000..ce65615295 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":558561,"aggregate_throughput_gbps":0.4575730796853841,"aggregate_transferred_bytes":2287865856,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5101632059792551,"avg_latency_us":8.951575566500347,"avg_transfer_us":8.119482026135016,"index":0,"operations":558561,"p999_us":20.0,"p99_us":11.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.4575730796853841,"total_duration_us":5000001.0,"transferred_bytes":2287865856}]} +{"aggregate_operations":565121,"aggregate_throughput_gbps":0.9258940612211877,"aggregate_transferred_bytes":4629471232,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.0269044956815954,"avg_latency_us":8.847664482473665,"avg_transfer_us":8.020116045944143,"index":0,"operations":565121,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.9258940612211877,"total_duration_us":5000001.0,"transferred_bytes":4629471232}]} +{"aggregate_operations":544402,"aggregate_throughput_gbps":1.783895403262758,"aggregate_transferred_bytes":8919482368,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.033233506756037,"avg_latency_us":9.184394987527599,"avg_transfer_us":8.093146241196763,"index":0,"operations":544402,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.783895403262758,"total_duration_us":5000003.0,"transferred_bytes":8919482368}]} +{"aggregate_operations":517206,"aggregate_throughput_gbps":3.3895598857760456,"aggregate_transferred_bytes":16947806208,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.7959434244991943,"avg_latency_us":9.667331778827005,"avg_transfer_us":8.724314876470883,"index":0,"operations":517206,"p999_us":19.0,"p99_us":11.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.3895598857760456,"total_duration_us":5000002.0,"transferred_bytes":16947806208}]} +{"aggregate_operations":478997,"aggregate_throughput_gbps":6.278296921806157,"aggregate_transferred_bytes":31391547392,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.161245815249703,"avg_latency_us":10.438499614820135,"avg_transfer_us":9.283124946502797,"index":0,"operations":478997,"p999_us":35.0,"p99_us":12.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.278296921806157,"total_duration_us":5000010.0,"transferred_bytes":31391547392}]} +{"aggregate_operations":390222,"aggregate_throughput_gbps":10.229433550913289,"aggregate_transferred_bytes":51147177984,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.204292683683994,"avg_latency_us":12.81322170456817,"avg_transfer_us":11.762235342958624,"index":0,"operations":390222,"p999_us":18.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.229433550913289,"total_duration_us":5000001.0,"transferred_bytes":51147177984}]} +{"aggregate_operations":306441,"aggregate_throughput_gbps":16.066298554943177,"aggregate_transferred_bytes":80331669504,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":17.241745626254957,"avg_latency_us":16.31639043078439,"avg_transfer_us":15.233451137413075,"index":0,"operations":306441,"p999_us":20.0,"p99_us":18.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":16.066298554943177,"total_duration_us":5000011.0,"transferred_bytes":80331669504}]} +{"aggregate_operations":200575,"aggregate_throughput_gbps":21.031804707278116,"aggregate_transferred_bytes":105159065600,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":22.58145455333971,"avg_latency_us":24.92834101956874,"avg_transfer_us":23.385072915368315,"index":0,"operations":200575,"p999_us":36.0,"p99_us":32.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":21.031804707278116,"total_duration_us":5000002.0,"transferred_bytes":105159065600}]} +{"aggregate_operations":134638,"aggregate_throughput_gbps":28.23558427354831,"aggregate_transferred_bytes":141178175488,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":29.870700167780534,"avg_latency_us":37.136685036913796,"avg_transfer_us":35.197529672157934,"index":0,"operations":134638,"p999_us":57.0,"p99_us":51.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":28.23558427354831,"total_duration_us":5000009.0,"transferred_bytes":141178175488}]} +{"aggregate_operations":70086,"aggregate_throughput_gbps":29.396193135161372,"aggregate_transferred_bytes":146980995072,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":30.590061350424307,"avg_latency_us":71.34093827583254,"avg_transfer_us":68.66292840224867,"index":0,"operations":70086,"p999_us":97.0,"p99_us":88.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.396193135161372,"total_duration_us":5000001.0,"transferred_bytes":146980995072}]} +{"aggregate_operations":41725,"aggregate_throughput_gbps":35.00117987032507,"aggregate_transferred_bytes":175007334400,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.85780912908312,"avg_latency_us":119.83321749550629,"avg_transfer_us":117.1175793888556,"index":0,"operations":41725,"p999_us":163.0,"p99_us":157.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":35.00117987032507,"total_duration_us":5000041.0,"transferred_bytes":175007334400}]} +{"aggregate_operations":23069,"aggregate_throughput_gbps":38.70280227004731,"aggregate_transferred_bytes":193516797952,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.24756034614149,"avg_latency_us":216.74420217608045,"avg_transfer_us":214.01049026832547,"index":0,"operations":23069,"p999_us":303.0,"p99_us":296.3199999999997,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":38.70280227004731,"total_duration_us":5000072.0,"transferred_bytes":193516797952}]} +{"aggregate_operations":12199,"aggregate_throughput_gbps":40.93039930692491,"aggregate_transferred_bytes":204665257984,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.261433771693525,"avg_latency_us":409.89622100172147,"avg_transfer_us":407.14517583408474,"index":0,"operations":12199,"p999_us":583.0,"p99_us":427.92000000006374,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.93039930692491,"total_duration_us":5000324.0,"transferred_bytes":204665257984}]} +{"aggregate_operations":6253,"aggregate_throughput_gbps":41.96138510419456,"aggregate_transferred_bytes":209815863296,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":42.163103308240125,"avg_latency_us":799.6502478810171,"avg_transfer_us":796.8739804893651,"index":0,"operations":6253,"p999_us":1144.0,"p99_us":828.7599999999838,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.96138510419456,"total_duration_us":5000213.0,"transferred_bytes":209815863296}]} +{"aggregate_operations":3109,"aggregate_throughput_gbps":41.727932774978136,"aggregate_transferred_bytes":208641458176,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.861813126150885,"avg_latency_us":1608.2479897073015,"avg_transfer_us":1605.546477967192,"index":0,"operations":3109,"p999_us":2365.0,"p99_us":1613.8400000000001,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.727932774978136,"total_duration_us":5000043.0,"transferred_bytes":208641458176}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.log new file mode 100644 index 0000000000..17107c039f --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-1.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:09:33.360158 2231481 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:09:33.360231 2231481 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:09:33.361199 2231481 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:09:33.366312 2231481 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:09:33.366317 2231481 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:09:33.369740 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:09:33.374753 2231481 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:09:33.374759 2231481 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:09:33.378243 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:09:33.382848 2231481 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:09:33.382855 2231481 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:09:33.386371 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:09:33.391407 2231481 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:09:33.391414 2231481 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:09:33.394940 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:09:33.399582 2231481 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:09:33.399588 2231481 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:09:33.403115 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:09:33.407846 2231481 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:09:33.407852 2231481 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:09:33.411373 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:09:33.415992 2231481 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:09:33.415997 2231481 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:09:33.419555 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:09:33.424662 2231481 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:09:33.424670 2231481 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:09:33.428205 2231481 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2231481-gziMb7.jsonl +I0924 19:09:33.437346 2231481 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:09:33.437355 2231481 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:09:33.437359 2231481 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:16828 started successfully +I0924 19:09:33.601320 2231481 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 160.915 ms, registered in 3.02895 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:09:33.601359 2231481 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:09:33.602356 2231481 tent_backend.cpp:356] Opened 1 target segments +I0924 19:09:33.603952 2231681 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.457573 0.510163 9.0 8.1 11.0 20.0 +8192 1 0.925894 1.026904 8.8 8.0 10.0 12.0 +16384 1 1.783895 2.033234 9.2 8.1 10.0 12.0 +32768 1 3.389560 3.795943 9.7 8.7 11.0 19.0 +65536 1 6.278297 7.161246 10.4 9.3 12.0 35.0 +I0924 19:10:03.845031 2231682 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.229434 11.204293 12.8 11.8 15.0 18.0 +I0924 19:10:09.881718 2231683 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:10:09.881776 2231684 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 16.066299 17.241746 16.3 15.2 18.0 20.0 +I0924 19:10:15.906000 2231686 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:10:15.906028 2231685 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 21.031805 22.581455 24.9 23.4 32.0 36.0 +1048576 1 28.235584 29.870700 37.1 35.2 51.0 57.0 +2097152 1 29.396193 30.590061 71.3 68.7 88.0 97.0 +4194304 1 35.001180 35.857809 119.8 117.1 157.0 163.0 +8388608 1 38.702802 39.247560 216.7 214.0 296.3 303.0 +16777216 1 40.930399 41.261434 409.9 407.1 427.9 583.0 +33554432 1 41.961385 42.163103 799.7 796.9 828.8 1144.0 +67108864 1 41.727933 41.861813 1608.2 1605.5 1613.8 2365.0 +TENT trace incomplete: 312506 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2231481-gziMb7.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.command.txt new file mode 100644 index 0000000000..27b710173d --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=read --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/read-2.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.jsonl new file mode 100644 index 0000000000..1d3192db4d --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":571905,"aggregate_throughput_gbps":0.4685044822991035,"aggregate_transferred_bytes":2342522880,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5229682842074607,"avg_latency_us":8.742712513441917,"avg_transfer_us":7.917538752065465,"index":0,"operations":571905,"p999_us":18.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.4685044822991035,"total_duration_us":5000001.0,"transferred_bytes":2342522880}]} +{"aggregate_operations":574552,"aggregate_throughput_gbps":0.941345620261752,"aggregate_transferred_bytes":4706729984,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.0364745392195176,"avg_latency_us":8.7024359849065,"avg_transfer_us":7.949967975048385,"index":0,"operations":574552,"p999_us":11.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.941345620261752,"total_duration_us":5000002.0,"transferred_bytes":4706729984}]} +{"aggregate_operations":543877,"aggregate_throughput_gbps":1.7821743714256284,"aggregate_transferred_bytes":8910880768,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.028121362582705,"avg_latency_us":9.193264285858751,"avg_transfer_us":8.119102297026718,"index":0,"operations":543877,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.7821743714256284,"total_duration_us":5000005.0,"transferred_bytes":8910880768}]} +{"aggregate_operations":520802,"aggregate_throughput_gbps":3.4131252566997947,"aggregate_transferred_bytes":17065639936,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.828098596824252,"avg_latency_us":9.600585251208713,"avg_transfer_us":8.633378904074869,"index":0,"operations":520802,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.4131252566997947,"total_duration_us":5000004.0,"transferred_bytes":17065639936}]} +{"aggregate_operations":467710,"aggregate_throughput_gbps":6.130367285926543,"aggregate_transferred_bytes":30651842560,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":6.891787320792245,"avg_latency_us":10.69038720574715,"avg_transfer_us":9.655344123495329,"index":0,"operations":467710,"p999_us":36.0,"p99_us":13.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.130367285926543,"total_duration_us":5000001.0,"transferred_bytes":30651842560}]} +{"aggregate_operations":392736,"aggregate_throughput_gbps":10.295315948704912,"aggregate_transferred_bytes":51476692992,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.242410012792577,"avg_latency_us":12.731226574594638,"avg_transfer_us":11.725059582009289,"index":0,"operations":392736,"p999_us":20.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.295315948704912,"total_duration_us":5000011.0,"transferred_bytes":51476692992}]} +{"aggregate_operations":297589,"aggregate_throughput_gbps":15.602212320102751,"aggregate_transferred_bytes":78011170816,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":16.971876192736893,"avg_latency_us":16.801719821633192,"avg_transfer_us":15.470870899125975,"index":0,"operations":297589,"p999_us":20.0,"p99_us":17.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":15.602212320102751,"total_duration_us":5000007.0,"transferred_bytes":78011170816}]} +{"aggregate_operations":199615,"aggregate_throughput_gbps":20.93113726531764,"aggregate_transferred_bytes":104655749120,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":22.50889670427245,"avg_latency_us":25.04823284823285,"avg_transfer_us":23.467595120607168,"index":0,"operations":199615,"p999_us":37.0,"p99_us":32.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":20.93113726531764,"total_duration_us":5000003.0,"transferred_bytes":104655749120}]} +{"aggregate_operations":134976,"aggregate_throughput_gbps":28.306337674638883,"aggregate_transferred_bytes":141532594176,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":29.914929392219264,"avg_latency_us":37.04385964912281,"avg_transfer_us":35.131193693693696,"index":0,"operations":134976,"p999_us":54.0,"p99_us":51.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":28.306337674638883,"total_duration_us":5000032.0,"transferred_bytes":141532594176}]} +{"aggregate_operations":70278,"aggregate_throughput_gbps":29.476487943998862,"aggregate_transferred_bytes":147383648256,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":30.66079435041437,"avg_latency_us":71.14660348900082,"avg_transfer_us":68.47801587979168,"index":0,"operations":70278,"p999_us":94.0,"p99_us":87.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.476487943998862,"total_duration_us":5000041.0,"transferred_bytes":147383648256}]} +{"aggregate_operations":41700,"aggregate_throughput_gbps":34.9799286851553,"aggregate_transferred_bytes":174902476800,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.8416833318166,"avg_latency_us":119.90601918465228,"avg_transfer_us":117.17263788968825,"index":0,"operations":41700,"p999_us":164.0,"p99_us":156.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":34.9799286851553,"total_duration_us":5000081.0,"transferred_bytes":174902476800}]} +{"aggregate_operations":23066,"aggregate_throughput_gbps":38.6974209059508,"aggregate_transferred_bytes":193491632128,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.24844438472314,"avg_latency_us":216.77434318910952,"avg_transfer_us":214.00268793895776,"index":0,"operations":23066,"p999_us":303.0,"p99_us":261.49999999992724,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":38.6974209059508,"total_duration_us":5000117.0,"transferred_bytes":193491632128}]} +{"aggregate_operations":12195,"aggregate_throughput_gbps":40.91943341071963,"aggregate_transferred_bytes":204598149120,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.25388745928605,"avg_latency_us":410.0060680606806,"avg_transfer_us":407.2192701927019,"index":0,"operations":12195,"p999_us":584.0,"p99_us":462.1399999999394,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.91943341071963,"total_duration_us":5000024.0,"transferred_bytes":204598149120}]} +{"aggregate_operations":6141,"aggregate_throughput_gbps":41.20611417532886,"aggregate_transferred_bytes":206057766912,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.41085058447149,"avg_latency_us":814.3071161048689,"avg_transfer_us":811.4971503012539,"index":0,"operations":6141,"p999_us":1189.8600000000006,"p99_us":1051.0000000001128,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.20611417532886,"total_duration_us":5000660.0,"transferred_bytes":206057766912}]} +{"aggregate_operations":3112,"aggregate_throughput_gbps":41.75845975803051,"aggregate_transferred_bytes":208842784768,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.89283602191624,"avg_latency_us":1607.0723007712081,"avg_transfer_us":1604.3464010282776,"index":0,"operations":3112,"p999_us":2360.0,"p99_us":1615.3399999999992,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.75845975803051,"total_duration_us":5001209.0,"transferred_bytes":208842784768}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.log new file mode 100644 index 0000000000..b56e785b59 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-2.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:12:41.406334 2232757 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:12:41.406407 2232757 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:12:41.406791 2232757 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:12:41.411212 2232757 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:12:41.411218 2232757 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:12:41.414546 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:12:41.419085 2232757 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:12:41.419090 2232757 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:12:41.422403 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:12:41.426889 2232757 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:12:41.426895 2232757 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:12:41.430256 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:12:41.434765 2232757 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:12:41.434772 2232757 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:12:41.438097 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:12:41.442651 2232757 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:12:41.442656 2232757 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:12:41.446161 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:12:41.450724 2232757 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:12:41.450730 2232757 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:12:41.454244 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:12:41.458801 2232757 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:12:41.458806 2232757 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:12:41.462332 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:12:41.467347 2232757 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:12:41.467355 2232757 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:12:41.470856 2232757 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2232757-0Y5JRP.jsonl +I0924 19:12:41.479516 2232757 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:12:41.479524 2232757 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:12:41.479528 2232757 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:15085 started successfully +I0924 19:12:41.617487 2232757 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 134.883 ms, registered in 3.06041 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:12:41.617532 2232757 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:12:41.618394 2232757 tent_backend.cpp:356] Opened 1 target segments +I0924 19:12:41.619769 2232957 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.468504 0.522968 8.7 7.9 10.0 18.0 +8192 1 0.941346 1.036475 8.7 7.9 10.0 11.0 +16384 1 1.782174 2.028121 9.2 8.1 10.0 12.0 +32768 1 3.413125 3.828099 9.6 8.6 10.0 13.0 +65536 1 6.130367 6.891787 10.7 9.7 13.0 36.0 +I0924 19:13:11.858759 2232958 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.295316 11.242410 12.7 11.7 15.0 20.0 +I0924 19:13:17.895809 2232959 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:13:17.895812 2232960 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 15.602212 16.971876 16.8 15.5 17.0 20.0 +I0924 19:13:23.921084 2232962 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:13:23.921105 2232961 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 20.931137 22.508897 25.0 23.5 32.0 37.0 +1048576 1 28.306338 29.914929 37.0 35.1 51.0 54.0 +2097152 1 29.476488 30.660794 71.1 68.5 87.0 94.0 +4194304 1 34.979929 35.841683 119.9 117.2 156.0 164.0 +8388608 1 38.697421 39.248444 216.8 214.0 261.5 303.0 +16777216 1 40.919433 41.253887 410.0 407.2 462.1 584.0 +33554432 1 41.206114 41.410851 814.3 811.5 1051.0 1189.9 +67108864 1 41.758460 41.892836 1607.1 1604.3 1615.3 2360.0 +TENT trace incomplete: 417592 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2232757-0Y5JRP.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.command.txt new file mode 100644 index 0000000000..39d513eb0f --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=read --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/read-3.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.jsonl new file mode 100644 index 0000000000..6df0336b66 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":572950,"aggregate_throughput_gbps":0.4693601706398294,"aggregate_transferred_bytes":2346803200,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.525642316216526,"avg_latency_us":8.726773714983855,"avg_transfer_us":7.871545510079414,"index":0,"operations":572950,"p999_us":17.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.4693601706398294,"total_duration_us":5000005.0,"transferred_bytes":2346803200}]} +{"aggregate_operations":562859,"aggregate_throughput_gbps":0.9221881856,"aggregate_transferred_bytes":4610940928,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.022064025186536,"avg_latency_us":8.883219420849626,"avg_transfer_us":8.069663983342187,"index":0,"operations":562859,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.9221881856,"total_duration_us":5000000.0,"transferred_bytes":4610940928}]} +{"aggregate_operations":539756,"aggregate_throughput_gbps":1.7686706921293078,"aggregate_transferred_bytes":8843362304,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.0242422608388613,"avg_latency_us":9.263454227465743,"avg_transfer_us":8.143698263659877,"index":0,"operations":539756,"p999_us":13.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.7686706921293078,"total_duration_us":5000005.0,"transferred_bytes":8843362304}]} +{"aggregate_operations":512528,"aggregate_throughput_gbps":3.3589021572391373,"aggregate_transferred_bytes":16794517504,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.7798180506214227,"avg_latency_us":9.755568476258858,"avg_transfer_us":8.812121484094527,"index":0,"operations":512528,"p999_us":27.0,"p99_us":11.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.3589021572391373,"total_duration_us":5000002.0,"transferred_bytes":16794517504}]} +{"aggregate_operations":471285,"aggregate_throughput_gbps":6.177215633011861,"aggregate_transferred_bytes":30886133760,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.051228593223326,"avg_latency_us":10.609310714323605,"avg_transfer_us":9.473681530284223,"index":0,"operations":471285,"p999_us":42.0,"p99_us":12.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.177215633011861,"total_duration_us":5000009.0,"transferred_bytes":30886133760}]} +{"aggregate_operations":383193,"aggregate_throughput_gbps":10.045166543066765,"aggregate_transferred_bytes":50225872896,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":10.954977637763246,"avg_latency_us":13.048265495455293,"avg_transfer_us":12.020874598439951,"index":0,"operations":383193,"p999_us":20.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.045166543066765,"total_duration_us":5000004.0,"transferred_bytes":50225872896}]} +{"aggregate_operations":287421,"aggregate_throughput_gbps":15.069092917521248,"aggregate_transferred_bytes":75345690624,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":16.320715362667404,"avg_latency_us":17.396136677556616,"avg_transfer_us":16.18575191095988,"index":0,"operations":287421,"p999_us":26.0,"p99_us":19.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":15.069092917521248,"total_duration_us":5000015.0,"transferred_bytes":75345690624}]} +{"aggregate_operations":199385,"aggregate_throughput_gbps":20.906969855090434,"aggregate_transferred_bytes":104535162880,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":22.4463977972119,"avg_latency_us":25.077187351104648,"avg_transfer_us":23.5306116307646,"index":0,"operations":199385,"p999_us":36.0,"p99_us":32.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":20.906969855090434,"total_duration_us":5000015.0,"transferred_bytes":104535162880}]} +{"aggregate_operations":134668,"aggregate_throughput_gbps":28.241734509805333,"aggregate_transferred_bytes":141209632768,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":29.89766929328279,"avg_latency_us":37.128597736656076,"avg_transfer_us":35.172171562657795,"index":0,"operations":134668,"p999_us":59.0,"p99_us":51.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":28.241734509805333,"total_duration_us":5000034.0,"transferred_bytes":141209632768}]} +{"aggregate_operations":70284,"aggregate_throughput_gbps":29.47893375690218,"aggregate_transferred_bytes":147396231168,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":30.634495737132532,"avg_latency_us":71.14070058619316,"avg_transfer_us":68.5516333731717,"index":0,"operations":70284,"p999_us":98.0,"p99_us":88.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.47893375690218,"total_duration_us":5000053.0,"transferred_bytes":147396231168}]} +{"aggregate_operations":41732,"aggregate_throughput_gbps":35.00656876108726,"aggregate_transferred_bytes":175036694528,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.852584661733964,"avg_latency_us":119.81477043995015,"avg_transfer_us":117.14209719160357,"index":0,"operations":41732,"p999_us":163.0,"p99_us":157.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":35.00656876108726,"total_duration_us":5000110.0,"transferred_bytes":175036694528}]} +{"aggregate_operations":23082,"aggregate_throughput_gbps":38.724906641834835,"aggregate_transferred_bytes":193625849856,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.25637831672371,"avg_latency_us":216.6204834936314,"avg_transfer_us":213.95641625509055,"index":0,"operations":23082,"p999_us":303.0,"p99_us":251.0699999999306,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":38.724906641834835,"total_duration_us":5000034.0,"transferred_bytes":193625849856}]} +{"aggregate_operations":12203,"aggregate_throughput_gbps":40.943402614403915,"aggregate_transferred_bytes":204732366848,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.26519983953054,"avg_latency_us":409.76604113742525,"avg_transfer_us":407.11071048102923,"index":0,"operations":12203,"p999_us":585.0,"p99_us":485.75999999997293,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.943402614403915,"total_duration_us":5000375.0,"transferred_bytes":204732366848}]} +{"aggregate_operations":6145,"aggregate_throughput_gbps":41.2371598132056,"aggregate_transferred_bytes":206191984640,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.436511149952985,"avg_latency_us":813.6940602115541,"avg_transfer_us":810.9856794141579,"index":0,"operations":6145,"p999_us":1191.8560000000007,"p99_us":820.5599999999995,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.2371598132056,"total_duration_us":5000150.0,"transferred_bytes":206191984640}]} +{"aggregate_operations":3112,"aggregate_throughput_gbps":41.76582546861435,"aggregate_transferred_bytes":208842784768,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"read","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":41.90222059337872,"avg_latency_us":1606.788881748072,"avg_transfer_us":1604.089652956298,"index":0,"operations":3112,"p999_us":2364.889,"p99_us":2268.709999999906,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":41.76582546861435,"total_duration_us":5000327.0,"transferred_bytes":208842784768}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.log new file mode 100644 index 0000000000..699c195e8d --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/read-3.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:15:49.391861 2234022 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:15:49.391943 2234022 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:15:49.392759 2234022 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:15:49.397629 2234022 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:15:49.397634 2234022 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:15:49.401203 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:15:49.406325 2234022 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:15:49.406334 2234022 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:15:49.409726 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:15:49.414745 2234022 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:15:49.414752 2234022 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:15:49.418210 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:15:49.423285 2234022 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:15:49.423292 2234022 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:15:49.426842 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:15:49.431474 2234022 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:15:49.431480 2234022 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:15:49.434988 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:15:49.439616 2234022 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:15:49.439622 2234022 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:15:49.443126 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:15:49.447742 2234022 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:15:49.447748 2234022 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:15:49.451310 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:15:49.455940 2234022 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:15:49.455946 2234022 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:15:49.459488 2234022 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2234022-WL8pRq.jsonl +I0924 19:15:49.467162 2234022 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:15:49.467170 2234022 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:15:49.467173 2234022 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:15080 started successfully +I0924 19:15:49.614537 2234022 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 144.242 ms, registered in 3.10274 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:15:49.614588 2234022 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:15:49.615597 2234022 tent_backend.cpp:356] Opened 1 target segments +I0924 19:15:49.616968 2234231 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.469360 0.525642 8.7 7.9 10.0 17.0 +8192 1 0.922188 1.022064 8.9 8.1 10.0 13.0 +16384 1 1.768671 2.024242 9.3 8.1 10.0 13.0 +32768 1 3.358902 3.779818 9.8 8.8 11.0 27.0 +65536 1 6.177216 7.051229 10.6 9.5 12.0 42.0 +I0924 19:16:19.856909 2234232 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.045167 10.954978 13.0 12.0 15.0 20.0 +I0924 19:16:25.892422 2234233 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:16:25.892445 2234234 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 15.069093 16.320715 17.4 16.2 19.0 26.0 +I0924 19:16:31.915925 2234236 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:16:31.915939 2234235 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 20.906970 22.446398 25.1 23.5 32.0 36.0 +1048576 1 28.241735 29.897669 37.1 35.2 51.0 59.0 +2097152 1 29.478934 30.634496 71.1 68.6 88.0 98.0 +4194304 1 35.006569 35.852585 119.8 117.1 157.0 163.0 +8388608 1 38.724907 39.256378 216.6 214.0 251.1 303.0 +16777216 1 40.943403 41.265200 409.8 407.1 485.8 585.0 +33554432 1 41.237160 41.436511 813.7 811.0 820.6 1191.9 +67108864 1 41.765825 41.902221 1606.8 1604.1 2268.7 2364.9 +TENT trace incomplete: 276006 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2234022-WL8pRq.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/slurm-job.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/slurm-job.txt new file mode 100644 index 0000000000..c3328c5fab --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/slurm-job.txt @@ -0,0 +1,30 @@ +JobId=1695 JobName=tebench-rdma + UserId=kimbo(100001) GroupId=kimbo(100001) MCS_label=N/A + Priority=1 Nice=0 Account=cw-sup QOS=normal + JobState=RUNNING Reason=None Dependency=(null) + Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 + RunTime=00:00:02 TimeLimit=02:00:00 TimeMin=N/A + SubmitTime=2026-09-24T19:09:24 EligibleTime=2026-09-24T19:09:24 + AccrueTime=2026-09-24T19:09:24 + StartTime=2026-09-24T19:09:24 EndTime=2026-09-24T21:09:24 Deadline=N/A + SuspendTime=None SecsPreSuspend=0 LastSchedEval=2026-09-24T19:09:24 Scheduler=Main + Partition=h100 AllocNode:Sid=10-0-0-208:3612265 + ReqNodeList=slurm-h100-206-[025,035] ExcNodeList=(null) + NodeList=slurm-h100-206-[025,035] + BatchHost=slurm-h100-206-025 + NumNodes=2 NumCPUs=16 NumTasks=2 CPUs/Task=8 ReqB:S:C:T=0:0:*:* + ReqTRES=cpu=16,mem=128G,node=2,billing=16,gres/gpu=2 + AllocTRES=cpu=16,mem=128G,node=2,billing=16,gres/gpu=2 + Socks/Node=* NtasksPerN:B:S:C=1:0:*:* CoreSpec=* + MinCPUsNode=8 MinMemoryNode=64G MinTmpDiskNode=0 + Features=(null) DelayBoot=00:00:00 + OverSubscribe=OK Contiguous=0 Licenses=(null) LicensesAlloc=(null) Network=(null) + Command=/mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/benchmark/slurm/tebench.sbatch + WorkDir=/mnt/home/kimbo/networkx + StdErr=/mnt/home/kimbo/networkx/tebench-1695.out + StdIn=/dev/null + StdOut=/mnt/home/kimbo/networkx/tebench-1695.out + TresPerNode=gres/gpu:1 + TresPerTask=cpu=8 + + diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target-command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target-command.txt new file mode 100644 index 0000000000..a626346e04 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target-command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target.log new file mode 100644 index 0000000000..b004ff1c4c --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/target.log @@ -0,0 +1,41 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:09:31.005136 1960042 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:09:31.005209 1960042 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:09:31.005609 1960042 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:09:31.010011 1960042 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:09:31.010017 1960042 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:a3:a6 +I0924 19:09:31.013412 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:09:31.017927 1960042 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:09:31.017933 1960042 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:9f:56 +I0924 19:09:31.021441 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:09:31.025939 1960042 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:09:31.025945 1960042 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:f5:1e +I0924 19:09:31.029448 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:09:31.034026 1960042 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:09:31.034039 1960042 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:a8:a6 +I0924 19:09:31.037528 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:09:31.042197 1960042 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:09:31.042204 1960042 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:7e:69:42 +I0924 19:09:31.045702 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:09:31.050282 1960042 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:09:31.050288 1960042 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:f5:de +I0924 19:09:31.053797 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:09:31.058367 1960042 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:09:31.058374 1960042 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:aa:ce +I0924 19:09:31.061882 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:09:31.066494 1960042 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:09:31.066502 1960042 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:84:a4:3e +I0924 19:09:31.070029 1960042 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-035-1960042-VuDlgX.jsonl +I0924 19:09:31.082217 1960042 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:09:31.082224 1960042 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:09:31.082227 1960042 transfer_engine_impl.cpp:577] Transfer Engine 10.0.3.49:15309 started successfully +I0924 19:09:31.226130 1960042 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 140.843 ms, registered in 3.04691 ms +To start initiators, run + ./tebench --target_seg_name=10.0.3.49:15309 --seg_type=VRAM --backend=tent +Press Ctrl-C to terminate +srun: forcing job termination +srun: Job step aborted: Waiting up to 32 seconds for job step to finish. +[2026-09-24T19:18:55.319] error: *** STEP 1695.1 ON slurm-h100-206-035 CANCELLED AT 2026-09-24T19:18:55 DUE to SIGNAL Killed *** +srun: error: slurm-h100-206-035: task 0: Killed +srun: Terminating StepId=1695.1 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/topology.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/topology.txt new file mode 100644 index 0000000000..43dddc44bf --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/topology.txt @@ -0,0 +1,4 @@ +initiator=slurm-h100-206-025 +target=slurm-h100-206-035 +seg_type=VRAM +CUDA_VISIBLE_DEVICES is assigned by Slurm per node diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-analysis.json b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-analysis.json new file mode 100644 index 0000000000..d940bc7397 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-analysis.json @@ -0,0 +1,998 @@ +[ + { + "label": "read-1", + "file": "tent-events-slurm-h100-206-025-2231481-gziMb7.jsonl", + "file_bytes": 2951125300, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:16828", + "pid": 2231481, + "schema_version": 1 + }, + "footer": { + "dropped_records": 312506, + "event": "trace_end", + "schema_version": 1, + "written_records": 8853876 + }, + "counts": { + "submit": 4426938, + "complete": 4426938 + }, + "status": { + "completed": 4426938 + }, + "transports": { + "rdma": 4426938 + }, + "peers": [ + [ + "10.0.1.136:16828", + "10.0.3.49:15309", + "read", + "user" + ] + ], + "max_request_id": 4583191, + "matched_pairs": 4426893, + "submit_only": 45, + "complete_only": 45, + "unobserved_ids_before_last": 156208, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.367789774, + "sizes": { + "4096": { + "submit": 620338, + "complete": 620338, + "duration_ns_sum": 4874674410, + "duration_ns_max": 41302504, + "first_ns": 8056348470263156, + "last_ns": 8056354469454106 + }, + "8192": { + "submit": 631413, + "complete": 631413, + "duration_ns_sum": 4657636948, + "duration_ns_max": 877923, + "first_ns": 8056354524474027, + "last_ns": 8056360524477054 + }, + "16384": { + "submit": 597734, + "complete": 597733, + "duration_ns_sum": 4627158484, + "duration_ns_max": 922615, + "first_ns": 8056360574274815, + "last_ns": 8056366574276095 + }, + "32768": { + "submit": 583968, + "complete": 583968, + "duration_ns_sum": 4756718355, + "duration_ns_max": 1021983, + "first_ns": 8056366618445404, + "last_ns": 8056372618441966 + }, + "65536": { + "submit": 568641, + "complete": 568642, + "duration_ns_sum": 5083874129, + "duration_ns_max": 907026, + "first_ns": 8056372669587386, + "last_ns": 8056378669598113 + }, + "131072": { + "submit": 465667, + "complete": 465667, + "duration_ns_sum": 5140671812, + "duration_ns_max": 1168422, + "first_ns": 8056378710877663, + "last_ns": 8056384710884315 + }, + "262144": { + "submit": 367575, + "complete": 367575, + "duration_ns_sum": 5288523777, + "duration_ns_max": 1063249, + "first_ns": 8056384747712075, + "last_ns": 8056390747719641 + }, + "524288": { + "submit": 242181, + "complete": 242181, + "duration_ns_sum": 5039716562, + "duration_ns_max": 1070291, + "first_ns": 8056390771963675, + "last_ns": 8056396771962811 + }, + "1048576": { + "submit": 161704, + "complete": 161704, + "duration_ns_sum": 5338236137, + "duration_ns_max": 104667, + "first_ns": 8056396793252675, + "last_ns": 8056402793276252 + }, + "2097152": { + "submit": 84096, + "complete": 84096, + "duration_ns_sum": 5582428722, + "duration_ns_max": 920287, + "first_ns": 8056402806006851, + "last_ns": 8056408806010883 + }, + "4194304": { + "submit": 50063, + "complete": 50063, + "duration_ns_sum": 5746937372, + "duration_ns_max": 184378, + "first_ns": 8056408815893529, + "last_ns": 8056414815990182 + }, + "8388608": { + "submit": 27684, + "complete": 27684, + "duration_ns_sum": 5863271521, + "duration_ns_max": 386831, + "first_ns": 8056414822986469, + "last_ns": 8056420823065106 + }, + "16777216": { + "submit": 14639, + "complete": 14639, + "duration_ns_sum": 5926675744, + "duration_ns_max": 584541, + "first_ns": 8056420828582097, + "last_ns": 8056426828949498 + }, + "33554432": { + "submit": 7504, + "complete": 7504, + "duration_ns_sum": 5961587442, + "duration_ns_max": 1146232, + "first_ns": 8056426833425297, + "last_ns": 8056432834188856 + }, + "67108864": { + "submit": 3731, + "complete": 3731, + "duration_ns_sum": 5981972830, + "duration_ns_max": 2364511, + "first_ns": 8056432836975763, + "last_ns": 8056438838052930 + } + } + }, + { + "label": "read-2", + "file": "tent-events-slurm-h100-206-025-2232757-0Y5JRP.jsonl", + "file_bytes": 2926127648, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:15085", + "pid": 2232757, + "schema_version": 1 + }, + "footer": { + "dropped_records": 417592, + "event": "trace_end", + "schema_version": 1, + "written_records": 8778252 + }, + "counts": { + "submit": 4389126, + "complete": 4389126 + }, + "status": { + "completed": 4389126 + }, + "transports": { + "rdma": 4389126 + }, + "peers": [ + [ + "10.0.1.136:15085", + "10.0.3.49:15309", + "read", + "user" + ] + ], + "max_request_id": 4597922, + "matched_pairs": 4389071, + "submit_only": 55, + "complete_only": 55, + "unobserved_ids_before_last": 208741, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.378727415, + "sizes": { + "4096": { + "submit": 606475, + "complete": 606475, + "duration_ns_sum": 4735856159, + "duration_ns_max": 39263989, + "first_ns": 8056536486073727, + "last_ns": 8056542485489713 + }, + "8192": { + "submit": 614497, + "complete": 614497, + "duration_ns_sum": 4466379128, + "duration_ns_max": 664043, + "first_ns": 8056542542231311, + "last_ns": 8056548542226266 + }, + "16384": { + "submit": 601543, + "complete": 601543, + "duration_ns_sum": 4653948606, + "duration_ns_max": 811984, + "first_ns": 8056548595810095, + "last_ns": 8056554595805217 + }, + "32768": { + "submit": 583356, + "complete": 583356, + "duration_ns_sum": 4738428554, + "duration_ns_max": 749643, + "first_ns": 8056554637651688, + "last_ns": 8056560637653113 + }, + "65536": { + "submit": 563856, + "complete": 563856, + "duration_ns_sum": 5135051234, + "duration_ns_max": 901387, + "first_ns": 8056560682701776, + "last_ns": 8056566682695262 + }, + "131072": { + "submit": 473263, + "complete": 473263, + "duration_ns_sum": 5200500692, + "duration_ns_max": 1056803, + "first_ns": 8056566724704234, + "last_ns": 8056572724721270 + }, + "262144": { + "submit": 357203, + "complete": 357203, + "duration_ns_sum": 5095637138, + "duration_ns_max": 1049244, + "first_ns": 8056572761768125, + "last_ns": 8056578761777924 + }, + "524288": { + "submit": 239160, + "complete": 239160, + "duration_ns_sum": 5034344644, + "duration_ns_max": 1718686, + "first_ns": 8056578787079169, + "last_ns": 8056584787078945 + }, + "1048576": { + "submit": 161971, + "complete": 161971, + "duration_ns_sum": 5364998043, + "duration_ns_max": 151574, + "first_ns": 8056584809289105, + "last_ns": 8056590809330987 + }, + "2097152": { + "submit": 84347, + "complete": 84347, + "duration_ns_sum": 5608748103, + "duration_ns_max": 470327, + "first_ns": 8056590820746607, + "last_ns": 8056596820827902 + }, + "4194304": { + "submit": 50040, + "complete": 50040, + "duration_ns_sum": 5734883316, + "duration_ns_max": 195119, + "first_ns": 8056596831853265, + "last_ns": 8056602831958636 + }, + "8388608": { + "submit": 27676, + "complete": 27676, + "duration_ns_sum": 5859823175, + "duration_ns_max": 307806, + "first_ns": 8056602840311675, + "last_ns": 8056608840530569 + }, + "16777216": { + "submit": 14634, + "complete": 14634, + "duration_ns_sum": 5924930462, + "duration_ns_max": 585903, + "first_ns": 8056608845808941, + "last_ns": 8056614846010061 + }, + "33554432": { + "submit": 7370, + "complete": 7370, + "duration_ns_sum": 5961303610, + "duration_ns_max": 1189450, + "first_ns": 8056614849967129, + "last_ns": 8056620851080978 + }, + "67108864": { + "submit": 3735, + "complete": 3735, + "duration_ns_sum": 5982314920, + "duration_ns_max": 2360346, + "first_ns": 8056620862383461, + "last_ns": 8056626864801142 + } + } + }, + { + "label": "read-3", + "file": "tent-events-slurm-h100-206-025-2234022-WL8pRq.jsonl", + "file_bytes": 2938446459, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:15080", + "pid": 2234022, + "schema_version": 1 + }, + "footer": { + "dropped_records": 276006, + "event": "trace_end", + "schema_version": 1, + "written_records": 8815626 + }, + "counts": { + "submit": 4407813, + "complete": 4407813 + }, + "status": { + "completed": 4407813 + }, + "transports": { + "rdma": 4407813 + }, + "peers": [ + [ + "10.0.1.136:15080", + "10.0.3.49:15309", + "read", + "user" + ] + ], + "max_request_id": 4545816, + "matched_pairs": 4407773, + "submit_only": 40, + "complete_only": 40, + "unobserved_ids_before_last": 137963, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.361228293, + "sizes": { + "4096": { + "submit": 608671, + "complete": 608670, + "duration_ns_sum": 4734068544, + "duration_ns_max": 39621066, + "first_ns": 8056724483346769, + "last_ns": 8056730473689151 + }, + "8192": { + "submit": 623508, + "complete": 623509, + "duration_ns_sum": 4593350056, + "duration_ns_max": 721607, + "first_ns": 8056730542115145, + "last_ns": 8056736542110909 + }, + "16384": { + "submit": 609072, + "complete": 609072, + "duration_ns_sum": 4757037279, + "duration_ns_max": 920497, + "first_ns": 8056736586226559, + "last_ns": 8056742586224763 + }, + "32768": { + "submit": 603757, + "complete": 603757, + "duration_ns_sum": 4975888662, + "duration_ns_max": 1211186, + "first_ns": 8056742627784226, + "last_ns": 8056748627779392 + }, + "65536": { + "submit": 566854, + "complete": 566854, + "duration_ns_sum": 5117413982, + "duration_ns_max": 775480, + "first_ns": 8056748678830124, + "last_ns": 8056754678838370 + }, + "131072": { + "submit": 458861, + "complete": 458861, + "duration_ns_sum": 5165026439, + "duration_ns_max": 1074061, + "first_ns": 8056754722841481, + "last_ns": 8056760722841842 + }, + "262144": { + "submit": 348583, + "complete": 348583, + "duration_ns_sum": 5119236576, + "duration_ns_max": 1009307, + "first_ns": 8056760758433406, + "last_ns": 8056766758453193 + }, + "524288": { + "submit": 239004, + "complete": 239004, + "duration_ns_sum": 5021071271, + "duration_ns_max": 1644397, + "first_ns": 8056766781878128, + "last_ns": 8056772781902064 + }, + "1048576": { + "submit": 161657, + "complete": 161657, + "duration_ns_sum": 5338724209, + "duration_ns_max": 130777, + "first_ns": 8056772803514200, + "last_ns": 8056778803568064 + }, + "2097152": { + "submit": 84302, + "complete": 84302, + "duration_ns_sum": 5592452071, + "duration_ns_max": 155972, + "first_ns": 8056778815284975, + "last_ns": 8056784815371130 + }, + "4194304": { + "submit": 50091, + "complete": 50091, + "duration_ns_sum": 5748225059, + "duration_ns_max": 594739, + "first_ns": 8056784824877837, + "last_ns": 8056790824984913 + }, + "8388608": { + "submit": 27700, + "complete": 27700, + "duration_ns_sum": 5865441272, + "duration_ns_max": 309479, + "first_ns": 8056790831417435, + "last_ns": 8056796831452800 + }, + "16777216": { + "submit": 14644, + "complete": 14644, + "duration_ns_sum": 5928854405, + "duration_ns_max": 584670, + "first_ns": 8056796835938051, + "last_ns": 8056802836546496 + }, + "33554432": { + "submit": 7374, + "complete": 7374, + "duration_ns_sum": 5962548903, + "duration_ns_max": 1192358, + "first_ns": 8056802839845859, + "last_ns": 8056808840353310 + }, + "67108864": { + "submit": 3735, + "complete": 3735, + "duration_ns_sum": 5981758574, + "duration_ns_max": 2371414, + "first_ns": 8056808843277784, + "last_ns": 8056814844575062 + } + } + }, + { + "label": "write-1", + "file": "tent-events-slurm-h100-206-025-2232113-2zgoGf.jsonl", + "file_bytes": 2916445523, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:15344", + "pid": 2232113, + "schema_version": 1 + }, + "footer": { + "dropped_records": 505420, + "event": "trace_end", + "schema_version": 1, + "written_records": 8722968 + }, + "counts": { + "submit": 4361484, + "complete": 4361484 + }, + "status": { + "completed": 4361484 + }, + "transports": { + "rdma": 4361484 + }, + "peers": [ + [ + "10.0.1.136:15344", + "10.0.3.49:15309", + "write", + "user" + ] + ], + "max_request_id": 4614194, + "matched_pairs": 4361438, + "submit_only": 46, + "complete_only": 46, + "unobserved_ids_before_last": 252664, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.369534192, + "sizes": { + "4096": { + "submit": 610427, + "complete": 610426, + "duration_ns_sum": 4779846441, + "duration_ns_max": 43611012, + "first_ns": 8056442408020737, + "last_ns": 8056448399253879 + }, + "8192": { + "submit": 598750, + "complete": 598751, + "duration_ns_sum": 4368581870, + "duration_ns_max": 1176814, + "first_ns": 8056448467404399, + "last_ns": 8056454467402905 + }, + "16384": { + "submit": 588940, + "complete": 588940, + "duration_ns_sum": 4439362805, + "duration_ns_max": 700544, + "first_ns": 8056454514520220, + "last_ns": 8056460514527351 + }, + "32768": { + "submit": 587755, + "complete": 587755, + "duration_ns_sum": 4759998968, + "duration_ns_max": 955827, + "first_ns": 8056460551597015, + "last_ns": 8056466551604847 + }, + "65536": { + "submit": 557029, + "complete": 557029, + "duration_ns_sum": 4991356598, + "duration_ns_max": 807806, + "first_ns": 8056466592474749, + "last_ns": 8056472592475252 + }, + "131072": { + "submit": 466530, + "complete": 466530, + "duration_ns_sum": 5138890491, + "duration_ns_max": 1081677, + "first_ns": 8056472626851406, + "last_ns": 8056478626854125 + }, + "262144": { + "submit": 357914, + "complete": 357914, + "duration_ns_sum": 5108009365, + "duration_ns_max": 1171841, + "first_ns": 8056478658241700, + "last_ns": 8056484658255722 + }, + "524288": { + "submit": 245636, + "complete": 245636, + "duration_ns_sum": 5201989549, + "duration_ns_max": 1154523, + "first_ns": 8056484703074084, + "last_ns": 8056490703091457 + }, + "1048576": { + "submit": 161364, + "complete": 161364, + "duration_ns_sum": 5353752583, + "duration_ns_max": 187398, + "first_ns": 8056490734195170, + "last_ns": 8056496734231340 + }, + "2097152": { + "submit": 85707, + "complete": 85707, + "duration_ns_sum": 5590479062, + "duration_ns_max": 142718, + "first_ns": 8056496746764571, + "last_ns": 8056502746772876 + }, + "4194304": { + "submit": 49640, + "complete": 49640, + "duration_ns_sum": 5743284381, + "duration_ns_max": 423261, + "first_ns": 8056502755476886, + "last_ns": 8056508755581938 + }, + "8388608": { + "submit": 26958, + "complete": 26958, + "duration_ns_sum": 5849559396, + "duration_ns_max": 343352, + "first_ns": 8056508762525354, + "last_ns": 8056514762837690 + }, + "16777216": { + "submit": 14074, + "complete": 14074, + "duration_ns_sum": 5928722417, + "duration_ns_max": 2269196, + "first_ns": 8056514767099833, + "last_ns": 8056520767361156 + }, + "33554432": { + "submit": 7169, + "complete": 7169, + "duration_ns_sum": 5964092881, + "duration_ns_max": 1292101, + "first_ns": 8056520771126229, + "last_ns": 8056526772661758 + }, + "67108864": { + "submit": 3591, + "complete": 3591, + "duration_ns_sum": 5983589608, + "duration_ns_max": 2635058, + "first_ns": 8056526775392224, + "last_ns": 8056532777554929 + } + } + }, + { + "label": "write-2", + "file": "tent-events-slurm-h100-206-025-2233390-uky9P4.jsonl", + "file_bytes": 2906519707, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:15723", + "pid": 2233390, + "schema_version": 1 + }, + "footer": { + "dropped_records": 593004, + "event": "trace_end", + "schema_version": 1, + "written_records": 8693000 + }, + "counts": { + "submit": 4346500, + "complete": 4346500 + }, + "status": { + "completed": 4346500 + }, + "transports": { + "rdma": 4346500 + }, + "peers": [ + [ + "10.0.1.136:15723", + "10.0.3.49:15309", + "write", + "user" + ] + ], + "max_request_id": 4643002, + "matched_pairs": 4346452, + "submit_only": 48, + "complete_only": 48, + "unobserved_ids_before_last": 296454, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.44909977, + "sizes": { + "4096": { + "submit": 591729, + "complete": 591730, + "duration_ns_sum": 4592925574, + "duration_ns_max": 40460129, + "first_ns": 8056630461221807, + "last_ns": 8056636460596433 + }, + "8192": { + "submit": 572178, + "complete": 572177, + "duration_ns_sum": 4134912942, + "duration_ns_max": 800270, + "first_ns": 8056636519366467, + "last_ns": 8056642499847482 + }, + "16384": { + "submit": 619862, + "complete": 619862, + "duration_ns_sum": 4629186659, + "duration_ns_max": 950189, + "first_ns": 8056642573061377, + "last_ns": 8056648573061386 + }, + "32768": { + "submit": 579724, + "complete": 579724, + "duration_ns_sum": 4620717204, + "duration_ns_max": 1105226, + "first_ns": 8056648617460269, + "last_ns": 8056654617460369 + }, + "65536": { + "submit": 569369, + "complete": 569369, + "duration_ns_sum": 5027083859, + "duration_ns_max": 1075170, + "first_ns": 8056654687795385, + "last_ns": 8056660687804728 + }, + "131072": { + "submit": 466876, + "complete": 466876, + "duration_ns_sum": 5155125116, + "duration_ns_max": 1030421, + "first_ns": 8056660780450411, + "last_ns": 8056666780450828 + }, + "262144": { + "submit": 352472, + "complete": 352472, + "duration_ns_sum": 5160583695, + "duration_ns_max": 1293320, + "first_ns": 8056666815209001, + "last_ns": 8056672815204573 + }, + "524288": { + "submit": 244554, + "complete": 244554, + "duration_ns_sum": 5173402394, + "duration_ns_max": 1101953, + "first_ns": 8056672842487055, + "last_ns": 8056678842512738 + }, + "1048576": { + "submit": 162825, + "complete": 162825, + "duration_ns_sum": 5403758217, + "duration_ns_max": 507035, + "first_ns": 8056678863971209, + "last_ns": 8056684864030194 + }, + "2097152": { + "submit": 85442, + "complete": 85442, + "duration_ns_sum": 5609747806, + "duration_ns_max": 149786, + "first_ns": 8056684876387057, + "last_ns": 8056690876444305 + }, + "4194304": { + "submit": 49574, + "complete": 49574, + "duration_ns_sum": 5760600578, + "duration_ns_max": 239294, + "first_ns": 8056690885151453, + "last_ns": 8056696885333189 + }, + "8388608": { + "submit": 27011, + "complete": 27011, + "duration_ns_sum": 5871092281, + "duration_ns_max": 345055, + "first_ns": 8056696892611401, + "last_ns": 8056702892801812 + }, + "16777216": { + "submit": 14126, + "complete": 14126, + "duration_ns_sum": 5927223550, + "duration_ns_max": 680029, + "first_ns": 8056702896984837, + "last_ns": 8056708897429551 + }, + "33554432": { + "submit": 7167, + "complete": 7167, + "duration_ns_sum": 5965351515, + "duration_ns_max": 1394257, + "first_ns": 8056708900414231, + "last_ns": 8056714900688491 + }, + "67108864": { + "submit": 3591, + "complete": 3591, + "duration_ns_sum": 5983811475, + "duration_ns_max": 2782580, + "first_ns": 8056714908866691, + "last_ns": 8056720910321577 + } + } + }, + { + "label": "write-3", + "file": "tent-events-slurm-h100-206-025-2234663-oNUhoa.jsonl", + "file_bytes": 2958300924, + "header": { + "buffer_records": 65536, + "event": "trace_start", + "initiator": "10.0.1.136:16369", + "pid": 2234663, + "schema_version": 1 + }, + "footer": { + "dropped_records": 378892, + "event": "trace_end", + "schema_version": 1, + "written_records": 8848568 + }, + "counts": { + "submit": 4424285, + "complete": 4424283 + }, + "status": { + "completed": 4424283 + }, + "transports": { + "rdma": 4424283 + }, + "peers": [ + [ + "10.0.1.136:16369", + "10.0.3.49:15309", + "write", + "user" + ] + ], + "max_request_id": 4613730, + "matched_pairs": 4424234, + "submit_only": 51, + "complete_only": 49, + "unobserved_ids_before_last": 189396, + "duplicate_events": 0, + "invalid_duration": 0, + "span_seconds": 90.409687715, + "sizes": { + "4096": { + "submit": 607511, + "complete": 607511, + "duration_ns_sum": 4698905311, + "duration_ns_max": 42018134, + "first_ns": 8056818344890199, + "last_ns": 8056824344014006 + }, + "8192": { + "submit": 616705, + "complete": 616704, + "duration_ns_sum": 4479767325, + "duration_ns_max": 835063, + "first_ns": 8056824437225639, + "last_ns": 8056830395564814 + }, + "16384": { + "submit": 613728, + "complete": 613728, + "duration_ns_sum": 4621243043, + "duration_ns_max": 927462, + "first_ns": 8056830486665829, + "last_ns": 8056836486670584 + }, + "32768": { + "submit": 612189, + "complete": 612188, + "duration_ns_sum": 4898216759, + "duration_ns_max": 1188950, + "first_ns": 8056836530977064, + "last_ns": 8056842530977352 + }, + "65536": { + "submit": 574480, + "complete": 574480, + "duration_ns_sum": 5122379983, + "duration_ns_max": 720619, + "first_ns": 8056842578340443, + "last_ns": 8056848578344585 + }, + "131072": { + "submit": 466501, + "complete": 466501, + "duration_ns_sum": 5172184379, + "duration_ns_max": 1321588, + "first_ns": 8056848619246375, + "last_ns": 8056854619255523 + }, + "262144": { + "submit": 347135, + "complete": 347135, + "duration_ns_sum": 5049051294, + "duration_ns_max": 1087165, + "first_ns": 8056854651731709, + "last_ns": 8056860651734298 + }, + "524288": { + "submit": 240348, + "complete": 240348, + "duration_ns_sum": 5012208471, + "duration_ns_max": 1853511, + "first_ns": 8056860677918154, + "last_ns": 8056866677940966 + }, + "1048576": { + "submit": 159543, + "complete": 159543, + "duration_ns_sum": 5339006293, + "duration_ns_max": 264070, + "first_ns": 8056866701446699, + "last_ns": 8056872701501463 + }, + "2097152": { + "submit": 85025, + "complete": 85025, + "duration_ns_sum": 5585546304, + "duration_ns_max": 173186, + "first_ns": 8056872714885362, + "last_ns": 8056878714917639 + }, + "4194304": { + "submit": 49348, + "complete": 49348, + "duration_ns_sum": 5730324662, + "duration_ns_max": 244578, + "first_ns": 8056878726224294, + "last_ns": 8056884726312322 + }, + "8388608": { + "submit": 26895, + "complete": 26895, + "duration_ns_sum": 5847321551, + "duration_ns_max": 510324, + "first_ns": 8056884736253009, + "last_ns": 8056890736781025 + }, + "16777216": { + "submit": 14122, + "complete": 14122, + "duration_ns_sum": 5927144398, + "duration_ns_max": 662553, + "first_ns": 8056890742322789, + "last_ns": 8056896742969528 + }, + "33554432": { + "submit": 7164, + "complete": 7164, + "duration_ns_sum": 5962482513, + "duration_ns_max": 1290341, + "first_ns": 8056896747206230, + "last_ns": 8056902747395282 + }, + "67108864": { + "submit": 3591, + "complete": 3591, + "duration_ns_sum": 5983880782, + "duration_ns_max": 2641735, + "first_ns": 8056902751797946, + "last_ns": 8056908754577914 + } + } + } +] diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-loss.png b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-loss.png new file mode 100644 index 0000000000000000000000000000000000000000..d6aaf3ad23a593db74ff2825436ceab9815a1cdf GIT binary patch literal 52754 zcmeFa2UL~WwlzpuWz{uTl?jxfa!>>X5dlF$nZTo>5(LSG2uPMJSuI1U2zn$aD5xMg z2uMzr2uKpil2vk6qU4^7y7zytM|Y3W?|(hIU%&A)id#yZbJ*VwYt1$1T>IWUbwXjq z(hW=5*w|Juj~r5FV_Q7M#f0EyDeBu@Ft@TZ zH#xh>!O+Im#L7}sNMx6g|}?d&etN(l>F{Et@%S=ks1ZfGHu3Y3r`w($S$$XF%0IRo{!R6_9h>sw z)~x;chvz?-2;BOzR_TXBs;6QiHf=g^{>RYUTMj#i-e4U#65O>a=&{~wBbDC|YLB?< z)ahQ8G#sp#$qa0=?exswwfrCku<-S2d-@v3fBl8+W5d1fpZ@hqdZsVWY7!8%CPkUqWUy9Bq@Ne1jEbZVu#)#ORTek!TPi@j& z_0f5tG4-Uk++5lgUfzz54#S*q*|~29Ol4J7pY#=Mm*#&>{k#&RThnxpS78kYhg0)x zX`qDnRdF`9Zzm^bCt7Qs9bTrBVHcTp@H@7T*Lf>eEj-GjmOp>twd<+mOV7wKeSLL_ zX-$;x%5bFHA-j?Qj2HhxXU)OTzbwyYIUeUIVlr)C|zu~N$V^HoNNhPRxUuN~OQx9R@T*uWqJgX7+UkTD?nZ5{P%kubk(zwzfX1sOZb6 ziBJl85Q4!Jv>9bB4CYm&#{C)PcQ-LKpEdMr`8Q);cyH%#!JOgI&#+!z{Cd{Dr}8jH zb2l(B&@#QlkJG|os3n$pe`};cp;wf4dSX^qR{Ov}?3XVWWWK!merj%d$dFzB^0>$3 zuwmIFNtMS^_R80mu5It>*{-LjSA%(Xn;NZ6n}|Lc#RqII*Kmo8n( z^7r=_I2{$rE#rK?KGCpl>gAdSFRp!~qg5xPoyU5I$Hsi)tXgw2dcp$R`}zb=o;-Ov=8V#c4SU|VdCSEK=X^WhFRb@2dElhnZ0JusuBYzZ zyEi;O?w6XHDxe-0Q*dXa*{e(6X=Yta*exo`<@VOW(9qCiuqoZNKJodLKmO=wYYUOG z?>%4oK;-E0<2z3A++HtS`~52ECr5n+ib_h3?ccxO^uvp%x^82qA4}R6SZ4OOIh6&7 zISq!`_thqz*mDN!(UR+t8{{^waadly81MY+ufKYm(v#HUv^Aa>Xl6RZ zW;hPTB^s8{fBWT^UwRu;;~fT@G@gvBad2=oddVI(qb|fU5Mkc8T$AhECdr{r9&jDwp`} zT)Sa|z^YZNu3f(#Wzm=tAZF%QSy{O>YkphM#RFod)u(>@?UJ;#v}%s4lYRI5{Z`py zekUGF8Kw?knZ{l(6OO>&mj2@(cQJoB8a0`YRQZPR2S)f&p#^uYGF^^av>!+U+imhg}f+hNLlJ?oO|5aa+85VrI~U z(*6mh2WeK-jt|GgEIz!@_nMWa&G_jCkBx=x%r|Qsx8k!2-96#K?;Lr$-aWp5$cJqC%3sSK3Q2^%xa}YYIE4fkH0%PJL5#y(5lMLj;dr1CB*4u;`c^qCYz@F zZQHuFxVgF6(<1Nsiu78S?x#1-C7e->RDOP-a3vf6)8%YzUxI((=C(Yt+h+Qfx0jcA ziR@H2ZwR(_=X|U;*0f7gF1}D^qRBuz!)|+dnz;Fg-+hGjWLt9(4QwrV)~+>7OHc2p zAOGtoo;|Gl_v5G2`x;ZN;yN@mGzt)Qmfy}lEAiv{_3I6)Ba|E_L|ngpStJv#m1160 zA;im@*Il1ve7PY=#--LMw_fj-%C}dh+y>VmXPb)cJG^!ss zaHZnu!5h-v`Q_#1-*Dt@ErMqs^xxSStrn}v!Sclac4!98rcVxL4lT#A&lJfXdAHAL z(2Ra_qw~d!>?2*}60I(uK7DeSu1~cJR>B*lgM+cP4LXW_9j5oXjK@?w+5dg!fE5mV zTX#iB=X{|TS728{y~>C3b$PoydECGK#eX_VHK@>Wq)cL?f){5(X6v_a7u#A~hRP;n z^sP#Brw22yXJ)ztSs;*F^UBD`)Q#7Bj|}C`i$|$Ovbru>xA%#d)$Yh^Y}{A=K`YhL z;?X`wp}O(fi9^^12_{urc^Qm1VY7~{ZpM!Dt)H$@08iIg_1%(sG3i4#d0Nzph?w(a>UFr_(vY!dI8OQRxHy@l@AIo4{d}@Dm(6 zHDA6JT`4Rq+?G@>Tar9~#XZH7YoB;)`Q`rL`TD-bB>lYW+O6yBc@zWoh|uAya+n(J zF?x)MTkj*Rld(rZE~Ma#iD1Wm1SF!(YEvG&{D&W)5Z?6Emw-J$34?kapUWKYm)(o3~1?x;9oT z`CQSj*dnJOwcIMJm ztJLS`W@}#PAN1X>B9G8s+v+hNi*fZ#59OxTPsC^>y}a=008<$&G1_EX9_cX9fYhRl zJv%-B&@c6mL8qCV`omwdo2!YOn>0LkyPd(-)6_Sk0Ygv59>_)qllC4^GGaUw4 z@$pt(DL6wJ^K;Y5&SR!b<=Lq*v5Q}RuZdJqSh8dZMVV3Ty6xMyGZE~odTXNX3%MNE zOp45YE#%RBdH&vkg9pW|KOYjaXi$xdi=$A#^~9qHY}OSU#g7ALs1GzGTQub2^8a0eBWT&K^&zF z*U3bL0x`EKI|}y`EiUom77d{Yavj~gf`Z5B2M+xoHZwEEYPBeocZUU9Dr;*$lAWKi zbsH|))>{#B725GjTyCdw0ntE*L$EgIDjRqojt$_#b|xtzGOK@4*kPj72=pDr8hDxR4f z5j+2W6O$Dj9DIE_$FmfR#^S0fmBti{s*P4zPeT;EdqccsS+{TBZZ4OI9&FAO6c<;U zot~(3n`jQ-ZCKp>Pz2!&tRU8R;$4uRHM~)BDD$N9l;)R>FaiC zhGNMhaJ(^{zS=rEwgZB+P??VEHa3a!UYwEG6(!~6Ctb$-iyYTY*Kmr>apnIq#H)~@ z)M^;59*92#kX5QvbWdR|{!rU?LuL}KKq7wkS2$%r(sPc+3-dL?4*o86}eq%M5Cgdx>Ls|z>d*`k!6 z-vS6pwr-QBcq!#RlZJC2g3N+U{MTU4@RWH^OLivr#gE@T-fR83zb(BhNCk(XY+T)A zRfl1Dkc??-PBucnVZxc}R4YB1nJ;-uSFWr@<`y$3SRSjF+cLAVDbesYFk9)kV~bPx zc;FOrL+ zF0ZDh7H5%E?xUAvq!=X9c)!kbPR3=N9cSV?BgCeoxIk`x#z3VDV~_`AEH!A(_3-c- zM&R1WTF%46(*|7O(3XZ>UbgmEoZUSWkMG|7J=Mb1)io~3`}SG++_`!4uBPSO(jQ_* zFscten32XELmv`C?)mu{sTfZVcXZn7K0msa7x`{bMAB3~m(k{@Wm#n36t9s~5Ho@^ z@xiAsKcBH|Vt}2v#uV0Sr9l=Dy{Hm0er5-V?hhVERn9bF$*&{-oPF z+S;DjQAL)^y_{vRe~m99rLsJ~`~Jrd*$uxV;u^_Ot)rMeCZj)lydlLRu=&HX*Gg@z z_e7_-zRKEEd(Lef)A0)mvUK-P&&;H>ofvta#X9@0p>95Sd$7*gv(G-YwHbJwb{QXP zwHg;W_x9(&VH8Wq?FiQf*!Ah-zpP(xtZS{5Y+55N*R^ce<=A-L=J@iY8KY6-+@+tY zPRb>3FE1|MjU8-I%OPsyspxd!!i(miCPZF*43cyC-E$Y6oi`9VSL#Aa5}T$3+idV1 z+opZL&a$!1+*q@xf`^^`{F}l;y@|QXs;V+MBOpUl6GR9nZle_kh6;BLpBP=WVb5D>FU&=SC6AuRY;yCt7h`8O>^Xm``$Lp^JfK#F z!+>UY%5|kZH|O5JKgEj}UgsPc8QD3)!O3ZU|7!BqQ*N%VHag?4Oh3N5+&LE~%T!gp zImfln;ln^zt(H~t^oE?-@kS#a9@j6In`M#QlB8EZ#%7P!$U)wb_U_A`Xc6tx%|6Z= zlN=uzDY2K_7L9NxwWlJu)wl#3df5y|b%?^MgDj@fA8|J-^j_&JqYmYcIe`iW)8iir$-s>y5OjBFvq+N7!;$RIVYF^i8&vqW$9V%F| zvwq(vIiphc%-}!;Z*DPv{s1`-_aG5ty@F1EJ@+CdwGXXYocFq0v7t0-vt1lB-57j) z54XkHOb5Aqy!_Mhp5i4_{Xi4H2fb+mY;aQU=)HdvIS~tlipGgsl#Q+4T2z;f?dYaH z!r7yW^LKmr@R}%!THm*^-Gg;-@G|wYvPXz zZdZB6;NW1!KuRNnDIy|rt>NVUnVFGtR_Jg?NwjU(JBL2|fritDC4OZU1yQb(!>mv& z$gw~PE1~MSSc|$?t#E*yby2Z)9R+~}zDu|#Tlf>7aJzolpLuLE4pMi80*@~vKR>^N zeIwN&*gZ&@LpjrH)~z#g6wH^TtZ9#O+B1T|GS} z6Jb6qTGYiq{@AcHwS`S=$BrGX9k&wwjd>>2wIZV^i4L2$1cTCiumC4B*%GuXi zzNzo)EHdh?J9orV6%-T#CE^BJb91F-WR|TB8A~4i^5unwGH)i%Pb5kxR@{!*dv$l> zV%uaSyj*2sj=2yvv$S-Ie{JeECvMf^T9&ZhC!`=ZWbbFUc7BQB%pKER=X52iVoyE2 zl3~}gg-P|Zg{5l?N;synre;9S%y5bJ^b4V#s;QPbf#qd<$Ncztc@GK+3H2fd2Z^j7 zoy_%X)rvd3KlmU({fJY>l*@^!+ztnn=zeW3E?DT|;$me0$^dZ-Q=#w9bMdkO5^Ljh z0s&0fmR>m|$HvBY3g~H>sxTkl;i0S%uYf%!Tku%`^2}tjIyHPmS>e0K`}lMw4^Eh( zdhxe2)N$(5tXZ7wHtirPIR-!n!lr%lMO?nyaC^~#fo*aoCMFH2tV0F+OpWzQzL{dY z1aSD6w3Wb>Xw=5*$pN}P4-1QO8tJ6!40!zM<9&_-f`WqO?A$K>BFxHk+b%o6*gh0b z)sG}?UmBMO^~q$J)xI>TTI4^}kX$lTAMnK|6cN6oUI8(uEcEwr*;_|rHIs*T0Fu89 z6}*`lt>GOFxIR2GlGKO-^9BOV@2O20oijaE;UiolSRb)QL!OEHV1vGkFSq3mzb@`e z;u96EI({ACm%p_wDekFohqP>8*DVI3YSeoXBahfP-RXJLvOo!cyAH91@aXKsl^UY! zjo=rsq+X)sB5!Ng5Xx6SyAIL*rgYo%@t&@(@Tk+#og6Cr_nOY^yT#k0;AO*`TM`q0 z5K9B{Buu(bvoPh{^du64)<|fwJ=a{n`|Ozd$klMM`p1vQCK0v)&Gdo_17~F()M4Ex z{Q6Em7>dXW%In7gHyhUSNNLG^rWhzLZjUHum}ATw{hn>K_3Fn48r4w` z1%OcSVX>aY=S&r6MH-$zA3>?--CU*;@b}1$@;Pn8le>-P(|47JXCHdbJNfNfyT;&W zn-2A^rW2)S+De9;?#&H7u(6G8la-XL-|Bbe&3*M9bz9=g{V+a3Uf#BVsWNxh+VZtM z4YFD>I#+>?@n%yI!DpHgdAO>29{Db!w=y zED!_Kk}JcRTO?N!dut<_wdHXNNV@ZJyK8gSwLc`t|G7c%#)A zHd7RqmKd?J<82io3Vk=X1-Qcop5?o%O9%t7N}Dk@x1-`REP^7B3k0Ozm#D zYR{CecYk^7$G`p<1Ix8O%>v-hrdjuW+Opo(;q;H)6Zd$}-h0%S8FMmNHbBZgYI)v! z<6t+V9K9srE|5!s+J$~&4^ZAGW{MxWm(+u?=+n_gLf-#i`)&VSqMOBmcj8en9ASZ3eOh#XTVrgXTkTZM zfddEDY~1Lgcn7DYg@nPfHe4` z#%<+?yAAC=zqxsUOT`l0B`R21J_bX36^bTc@mO8JCJciy~&ISnry+gbE&e*>B5T}=$ z(_0%GO~YNbZpW_Gv&Z~}#BwL5M|?_;Zy#$F7jwO9K0A_l* z-{5yxoy2&h>9@SzA7cGcSG=1lR}7KA!(e7Q4hfA`3Z^#M+?L#Dhx~HQwC@NvH+QFI zA+r9vk>YDHF*`=&GqJ9qAY5{0TkWb8H%MQ60VHPh&Qts4d8M2IaH*=f?rx}7kLJ#f z2Ra!vfjZvOn&Nf0b;Y$g5QME+1S%|wA|GvCJ9b%~9@P@#ft1Df#y7pnVy;ovY4N>+Ikd^6p;`kQd~ zv}T44&l(qmnPP75G23pEy>lFxRa)w3;g%>CP^ zpGJBYn+0B3x_tQ_tKS2Cd{iUG9txsR4Dx(C5cc-)r|9?`5V?)_Y4rwrTBVEzKn(ZvF4UAl;205YZ>eMQ^nY&%gl%_b&?O3EF-ODyR2eU>_h<>-vw(~du!n&wz z`yJ1pKkq?|Owh6d<=D9-79@)HNabc64XV|jA6w5Qez`Z|E6OH%TPzT*2*S;+x711D)uN!h%bXErPy+@8m=?4T4qEQe&H6AXi;~b2$*=1}icQDSq8FPYlC_%t(>6ytL4yqcPtyXOuaI>f&1#G@CHt2b^mQ;j)OarYc} zX^!cMp>n0U$*$myOeWJ|!UG2;JCOBQ#Q2U~?ce_n6l0_trfISuIEyf`U$ zJe^B#Kj2S!zXIzkWS}n1q38a(yZb!ZMP7%wj`!CaIbr7SKuZdg!gksN)?qnQ9%yY#gSa8_02ic^+lS9-zg$O$%9xzD~AZ;vqOCO4E zE9{d5!zMjpcv4?h3nZrJt5?6F)c^VC-(yH5nZi_w=6w6SEO3BV!E4V1Hbl8}E04N| zVQLZ`mpSa@odYGh_F6 z?;eQcnWvu3za`x$Gu0hp)V%rN^>(SwSRtK^b6Mc6#O;@k)C0v?fBW)jky}v5n1&_$ zb%2d!Nv?=Z;tP6&fwzp&0xQt$rV)JO(j~T}IRGS^>Y~!pJy)(=S+ZQTJJz*b{)u}Rs+(g5kiQB5sqP@OzrNV{(AX1-1Fw`+Lzvt6Rb9|}M06M+R~KNu z?%dfZv4KL#S0ZVwOM_&lHw2|;t)4}_tfZu5#Hp{ZU(jfkYhm7$R`CI$`YQ+lj?J4l z8%4SknZ=T?gSTU#ZH>*#w z^SF)POK_ic2FuS?Ae=jM_IQwVz4ZP~Mc=UREyi`$+??C07yz0FkedTkz~cmqM)9^n z9*;eP*%P`+Ck`FDdJkE#>ZFlTSVfrP#w1+`O?OVpPQLk&VIQ3U=DIC^z22Vt4^dL5 zZ;f8gYkXw@1n_P;dFG8NWoHonzc!e~a~Kx;Y=6&gf9P53E>BM83rtblSe>54*|TRo zfeSgTa%WS1`SsVcNbnDsM_rZ_W567k3nRVMR`<-WV8b_{DqomEY}}2aMy0!m10}Wr zI`WGiBHz+bpq0s5W)6tA-4@Q-MR}M?_yXD%Sg;V2sxXN?6;|xmzXD)B-1ZKE_l^R$ z)0;D3(-WLVj20FWg6_HF0eco$j};KNMv;rdya6^stlJ7Y-7)1R#{AB@t)@1}L@*P`Z1f@ZJD9?dexUf`Dx;ak4#H+BfGA*EY(^ z%3f#WfeN|!`Q|oHcof*dUW?L68kseav9^WApE(_{&(YFr^?vo>x2cIi9}{JPdedFK z*Rx%UCQHIcxi?BT-67Z!mx=iL?&}TXafKsC{!Gx%`*CEIt_xE&FKzd9E~s&b=|owZ zW1}0d>mPqCRfDKioSL0~w#a_mYJacK@cIIdJgc zK|1E0`{b8RHMRS5?)H3CloibbdZ_>h%Y5sfBS#atWnK3wR7%8C)o(k{hSij0IjhSN ze+XVU^tYP{S!>3V@^{5CcuOF3J!$J}&Gm?Lqww90<<1!x$c_Vh6z6oHsHF{xQ2E%c z)p`8dI9;66&aTLaou#ca+*VL{6sMmZ!Q;0T%tI`D)YbJG@!KP>MFFDgTWh}8HQuOL z*b)5IFJn7=&Me)vybUQ?B>u&V3HnWO+s?-O`S_fxsyil}t@>myRYmC; zpt{DOuk|6=|mO2tyHDvDUXNPYu8mmb(JjS*6!f!GPZAn?#PK8En^uE~C7G;pyr$Ky;Opc&L1R=WT!pGkRJ|r9j zyC=vpD8BPwD^Y~-oo~Qcv*LM^TC(4T_;eh#PF|d zjRVuST}n!mGC63B4$M2kN2GeZE~fYX)cAnb6L&Z!fK0Z|;(47%D-=#Ao1UJWoGdCU zQ+$5RUk$?_>rX20#p>08S+gC0s(lF2t`-A~#ThDs_64~kN-H%EvGApRpW3w>H}-w$ zUKWR7PXaDQMSB=g$OHs#0d(YP*fy}9N3ry($B%oWSk2a+{>Sp=)xaOgXY-d5D!Aq8 z=>v;MQ*-laI7vv|0x1~K;@lUj-MET#?uBu~56v^FLZSO;ztI1rSh0PbMx2#3DYi{o48 z%H#R$JvPHEc-erx*3V$g2_t%;xmgD%<}_ddXckwChL=oaTc`2W92^hZ+$@HE%&bq< z+Oe*->nAqt)wlK8<*k1L@P`j=sIM+Q7?~j+-YhOes zF}@Rfga>zB+9(ih7r`j%n0ff`rmQZmne_nkiDA83QSjexx}IFsg3^C|6;B+>1Hs+9 zm2+}(VE9u*q^^MyU{Rl#uRl#zhrg|$ID!6Vk3kR}R#50GlW48}W0hvpA|Zzx=Zm2&8UY3S`P%!}GT*l>;iPS0cEUprZsY0({mt=yY_*|h(N%td+BW)Z8iB6^+KWR-eF+fG$H!-a>;g)s8Wl8=je+7I-=mOa!Xfy+ zExY4%3gftNf^T<)M>bhepo8>5T|14+uH*A(vYAFiM0{Sk0L4-Gw!SkE$F z-MzR54hW*9CWrT;D!a$v0v0VC2Vva7?;i>=3w4rWz;44^peppviO`9bScG5j1!crr zkTEm!Aj;J|EAVqRFGnygpWcB*wyn zO9O1ka!P`sfR~wp%ASu&+&{d8s+kTJmamp!xbs*0bVKB}Y&ifxL%=J+ppfjkg3{93 zB+`N+DRYK2stlV7&W@Oqi;Ex*1U5j`{M>{bsJ4!V2`tAR*KZfZtXi~z+y;YX0+q$$btX9M ztQg8A1&a1#Fgv#&adLivofWqyN&9tntPty~ULV^W? z2mqHDPI)h+6JIdL_X-!|m#q=I(d_AnvtauE$z>x<6;Whqj0u2tRczidlx}?xGi>bb zPs3san~wy!>1h6ZsXjNxbP$@cXi8JUMnh<-1r|*l_;w>AZw2g zHVfwM^6~Kr1%fz)Vuk!}0}*+&J?BJ$#@4P|7e+WcO?y)1DPRTd2PwPmQ{8jAP91AW zMZhOz>zA~F!O$YtFx-tWN|58NPS-_|0tCZ^=*y3PtQrN}jzkzLe*5q&k|yvdNWn&o$*}wOMgI=Bj3%g&-OCSt|KoD9H#>dbLa~^n zMObpisGLCY2axX%wzg{HI=C6wE@X51b;BA~hZ7fr&~$`_5FDiz8wE~`#B&Hw6%RxV zR8!0~lH5@|(=sEjo5DH;SN|a|+4TsnIBNV*W17@5>`cfK2zH4TN?iN-`wt#Oz}lj~ z8ZFpp)g2igM03H2#JR*HKWu(}<(3v$t7ZjDb!iNS?Qnix-oeynhd}qqj`htBjTV}@ z?%CMGr=O64(RsYym_#PT<{2};!iRDmIhd(E@Ou!Fh1IcAfya&X)$@pnM(M!%nBy== z7@t>i;KTWUzQH#|3J>pQVJ0fGQ-J=k2AaL=&DA`2E|kz}Y3cD;oy?S1(|y&E_ZbwN zBvazgBpfIzDk6K}n2|ngiq79Y-B^}Ir}hcM*4DO4v|O6E=jgMnO$1dD2YYg+2UYM^ zU+np|dk-IOXJR+*N?n_bBb*J3LX<{Qn3LF7AY(ozfodqH3+}(u*u(ZyqbB%>*tc!8 zQ#t?IIWMrdDn>&z&|;&6r4Qq=oQE#_#Yv6gFmAIT=mi>N>a%M?G*~C39YKx&A)$9h zYGu&u3)kX#aG{-Rf2`dLKFqcn zK`THbsIgTNpY|&EPTsGx>gApObHYo4{rtk9vvf5oAeKzpf}cfsVfrrd9>dDY3Tfsj zl;q;lQdQVEk<)!bywlS5GK)uy^u~+ftvG9&xSxfRKJEs$Gk@zhB#2A%&ye&V$W2d9 zuGQOAcOF%5fTV30*??^=c%|k*zsu7)epG4oqi^Mn1R1 z%&D+kHQ2<9##Y^fFuqobij`L!=5QQSnV2)xEBF}Z_og6BI6TeCCRGL6+_q@@kZh1Y zd>E#aHOvFj*fx~a^Q3O_lT~bB+!b)H0R$s0>aDeco%~5H6Mb0sI^YpQ_Uq)}|7XEQ zT_SnjNSSQ>OIK=TY?G)R+hJf}knA#HiBuQ<%EFsRPKarWm=xCmsV3dh3fYBW0%t9? z@8B>hK1Q}$zn8j>KIU3Nc+`X)FjVl^2uySWva&6Y28O^6pycdE0?O4FLC$g`uW{Kp z=Go-^ll`6Zt9WG1Ck`D1Tq1ak6Ke{8hIj`)R~;H5RhMEWm50DhcGhG;gA=J6rVgyR zfm?R+9?*yEkU5XCbe((n42_H$VqLIF+xQ$q_ikKox0j}K!vM&gIgJY&Llj8;&M9t1lLaOv{TtG6FGlX(0Q+WcUHUW$^%% z@U7yOO%ap{K-bp+gz^;`DVAfjCQ#a8%6gBaP!oQV2U4yXKA-w_Kr`#iMKxZNmWeZ=EZ(pv#80j7;)@U)H2 zD-Vt~9zKq?`G#bzUsakmf0V+99?KY>_~8_5*SHc(WRAYMLf)>PWxa%Hf2_Y=1?iN< z051i^$q)LraL0WA(?>EcYS6Wn&CO#NMer_IT3W&Y##;3K_hK+oQrnjJK&YU^@;+)u zLd}vgHabd}8ahA-LSY|_9yy(dJV4*=A)iNI$JnDa>u$RHs4|>S{c{lUTsPgj3VAa5 z|9vU06nbppOtWpd4EeLjPk~yC*bJEF1z?#POlw^B;khUj=lCMd!_&9vL02Mt>C1fP^Mk?P$${G0hCwDhl58d)&g<0}Edg#rEah zg?B&&_^6h4A7U1KQ^0qwLut38d#UL`X8s5IJAFHPZrn}gw7e9tsy&bU*$Ijl`xBTOEU@m_K42tV)8L#dXF3bg;^-EU_{uA`jf0R!8kG{S%HtqZ0 z{xS^Ht}S*6SntUGYK8A=q%J(@4|t@Fr9@#be7*Xg z>m&WI{oap1N;wUO;xNPY50?BTEQ$vv^eqDG3~(?O55jfSBHDQP(4h~p>$$jM(Gc;r zu~7>uvonZndkEs0V=!<d|HCq$>o@B&;1tLp0ZO>&!-@F||~lfkucas*gqQ`3dp^f;i~T{Ly#wWDa{*$(&E zKF2{-yh$+-8iNMvAF@UD7c7Cy%Og6qY`ct%&Vti#Ir|f!Pk~LFeuI&RB0fb=$Sdm{ z|D+*Y@`%J=D5;n>yo@|J)rhVJ_}_f{LXg`R_ZlFtVTYGXgn>F101gMFMmuCBqIV7S zY%V!>ZPW|Nk-6Z_-5><^R)u>5-nlClVyi5@e#m1pk;MyE(RYs`VfW@26FUu#nk+3a zd}~o}5Nuuk`t12!0yLhFdJMdR3I`Ct#BzULsla*(&6Uy!HQu0+@O)|u2zg>oF!U2J zaQXpQVcp7p@UXM9^O^!g->~a<$_G>lt?LoN=l(i-4L1EaKnp?!l&zGNm3>eZj9Gzk+H zdk?P>P@^MLInIA4rC_$BVX#GL#pljtP@C=wj@_W0?=iB!7HotkD43GK+qZ9}`+i)! z7?7$PH6|>>yHk0j?2fQh!;W*6NhA{uKncYgLH{K4tt+0e?V%M4gR9Xc@SD5#mi z;V_T+?I;qBgMoy)(ZGWEp`M`TSGY45Mn&}u1Q}Gi3p*}QG zK`;|7p*p%IPDcx@Ud~ExX?}Heb-r!e`k=*EEfdaqvTrDhFoC$61Dbis1&W5}wLCn+ zs2%`!)nJHF0Xs>3xt$x1;VH0BBXDNOnukglR-LK#`zNc@tPMz&@RBm$SZ^$H&d6w8 z{oh>wJNx2s`BK-;RVQ_DWfkFPEtWil&J6oBJ9 z{lrc2(OxAK-M4PtqGm8+Z=jt;Lp3MjaRL1j;Ycl2U|E|t7yg5dZMf^d5dV;pfCdt# z@|iOa;J3VOP81(vQ!dtV1-JAW05IpS$Bvw}RwR8w?h-@+ftW|=i`*c(8uKuRoS*sj z6+Rp+!ArD=lGKJKv+ZAGhcfzvcIxG5Q$fEx?=I$tVjvtTry-W6w_KV)1XaQ=r}le5 zCUm?X!FtuX&9Jd;+J;&k&BkN+ozZ_c3mB+C@4|UJgdR{5xZq(50h$UJ17;<7Od>6| zbsf62ZAR07|7+1$cf~>cqRtdl+GfD4v_^IvKg!jEhY84)rXx+HYH7(9)YdWhgnAu; zZf9yt^9-jVZfXHaU1FI4y-|iDHVVpNz5eXGK#7we%2YsIgbI56(P~q&a|)fpB-jGo zN_?$3R=joVR?{yZ{~8A5!(LgIjh2%}0b~y#>5PkCpMf(svvr%Nf&fWwgM{Uy*RNl% zL3#sGZi;k9?r;#{d(ia;Go4s@G$`iZK1uGVXhnwNT;jx}&O4%sz{}XG&DW6T44MFC zO+eD8WzUcsSfQZ7IzHq;&cty-8Lw9^I~}8#2`z2JpId<%9fD=uC=yd_e)H!q*9Lhv zZF)7xoBI3?Z34D=U9>OMp*A4e({|_dMtz)pR%m{7S-W@drhU_gM&T-)TuNTEup>>jIO5iZHC|eCrwQdeAdnvZch2+Di;?e;yv*_02^bUileNvL#5b zEJR}vlJPFNxMN}Ik|h;bAA2xsw$s$2i*i&Q8uDBFC0uVg2YqzypFckXH3c6F-hNDS>d)J^?=yUb^;!nq`+ZUI zzUe&o2SYJfR%eyW=-uYIt;Zic0e4lDpMRJR1earzeyXMJzh`~K#;A^GOZ)1@EhE5$ zmZT6g7NZ_Bfkp&$3t8XGr#lC*nR*e{VaZR{f}z#`446LyB2zSKK&SCc2<>GLMHOJI zivX9my=UI5IU~j!;(d*c&-ZL>g%?0?q>9b}6tF@8+S;9~nzCF$ol#FI76yr94?d_& zyH$^cuz!B;%13_C_GM#N-*l9WukI(4*&gT7^UO-(X2?7BbU`0rJH?K)LMy|^M4@~* zLl_DL$QG#ACaCxR$^Qh`wblXDG~mPp2O_pG)s4PiQ%ej>PjQC_@^b2SIXGtH1`zf5 z>Vm;wu4S2i7~`SdZX9Q70Z^5FQy|QryJt+dh9DMS$#VI0*x3 zM_*qw@(UP?k}zqsIfIiE_#)D71cx(m(>R@Du+q^5Aayl=SGJ4oJ4-nfAMDx>NXUZ2 z1_R|KH%z>qUaRZK9YzoJ5v!`I-YXyr8#vGtFzw;Omw_wa0WsVIN`+5M(Bp4(g~q_s zNE!CK8Z$N?bCqKd(|%t01| zQa;p3iMr|5|8cdzOYMoSp&=`wa8dIQyUHH#t9$h61>Q*lCRC(4S814*PE8Cp`vMDZ z3wBiojLq!)8-MY~P|$=sLls)&+FyTt78=T6s$1G2`sz{ZJgC+^{w3R&QZ$CCwVHaZ zVYo}LH33hIC=mtL9_<^~u3kM4a6)-#Slbx^W=&B3TeHhKi~a=4#Q`7DR=H5b;_HQ> z6W2TN>1b?wUs$MsTrO!eX}qcf0w{Vk?xF38kBK?~$gB!jA4ZV~2phfdpT02J#su-X zaO2DWImIGWo$-en$U+hS6^NZhBkRNIegb&-uJyJCUAn|ZW$)*4Mb1i|k2?~N-Z;wA ze%Ix$xAJ4I=T~pm-{12?wU*4|);GQc6U1Wz1@GuQa%rRY!V8Aj56gz=v2fV|H`)sI zc;!8*6|ZA(Fb=!P7k9NdHd)`mLPvv8>pZ_2Gl?VFlTgU50(67hS=Ovs6N=&r(`G@B z+C1yz?2Jot3`yz64>mKw%Nj zQ?2Q{Cm(mK2$^-rl-(xQsrYYl(gpH$}$@@Hx=q6Y@tFT=^Y_43Qen;u zX=XgA{|_Q)2w5RP(Xa@v!nDu>Awmg020kWGsPKhg?2ZcbFKuE@!Cyhj2I1#acs9wm zv0zp3wbCQ)96eZT*RFjI*p@j)(gl)jv{_wT28^eCOx*DG9@nSvF`+R@_Ra%SZ;*4( zqPv37O1kdb2aJeq?|RTqYdAS$XcmF(rEpOnXs0ziJn?A4C)7GOH;2dP@i+o-w|5El z4be3$J9K0(N}c|ETbRKFwW5T5YhH|NL3S{SxPlAGA~hJ;kz7`)@FfiE9Gr*$l@$s` zh_f+TH4>E4?b{^@ZrFiD5&*bvyPeM}$Mtp)+SNcvUP^+MCkzSuTv9VkqbH@mUWMEq zbn){NGH(H6M}avIIfN~L=?Z-UszFBm*7qZcMr9kA%1B(cJFS}15v zzcjy`1WFe75GwWDg&Ar(11Af*Ui<6-&AP$X!hlkPx>4&%%60woNX)EIEJK%OAuc3? zCx8`-yI0yZjUnPut;g-u{)=r-aMm8P+T*&qx`cyXqV{E~$Nl303?&T0=g_Z(q~T^F ztOR(uSZXWnmUkukqEo3S?DcssgQ2F}|sun9fB>be!UAj9G;6WyfH;sKHcnB;$8u~?%;+k(%d zm?P{Fva(SgR)k4gG@jiXpmTVQtT^o%r6<5MlLBkIBzT6wVDLkoX3>>%nPV}AXw*&I zejCOUM7s`He^OhVqko%Tv2`iNMe8>j|=O9rv46+$_CQyY)wq>{^-mJKY^}qD% zD}A%kRlxoTxP*D(?}7l91Ca79V8I}$Es1u_(Y08m61 zi*)2J%E}pbMOohsVGXG?6NJX$Ej|1vaXW~WiJ>$rUwB}v^1y+yVYKrR(|+sb&4nq@ zm{q^+=}Bz14+vtIlnUDm$6=VGeT{&e9+$9zn>+3Xk9*py>7rw31;>RY zeg;KP;ReuyIU_~mdc-D(3;nVvV7PuI_fp&!$k%jvp`QEXVWJtSF#^i=+K5;nlgSZ0 z`ao@x&`?`fm$79@ad9QWNi>WgWe-FSQeAfOhZGnpLy+wb5IeR(+&lvI6RI>p^t=Zh zJO;m^8S3+;%a&CE7pz&ozUze>%`m_0%44GvX0@e8ji9yon7H>y9eo<$D7q z_DaD8cQeAHNailOdJK9x6fXQzO;vn~L{@XUZ8#$mWDU6XVzF9;4&Ay0O}F{k39=2~ zgxI0-C1QUH%DyUmoNkcjSJqRIxaEf%O@o{|Ihd6QTFa`pg1tLFt~Wb<-k~;mGAX_d zR$EtWKSkVc7L4A7sL;?*PH)giOl5ib>*=wZuzY0gT>R5dfk|WPaoUl%A&(SzF#oK$ zChsTc(IB@iI{B2Jct(J7K@LWM(`&y9+m{G`=53Vd1na~=4c{NlT{IIeRP|{O7CCr3%nfRUmfY{FX?Z84kXPw=cqOV zf|#1{nU#+b{&c=}bm3|PP}a&M1Ho(3=o20q3Q|SiJ(oP3UxuI}cvylpW-wv46|`%F zWTnX(MbwM%n4jxZbd80X>%LhgE?;9h=^bzzQ}iH=%<5b6C%K0N9Qr(8AjYnZ;=-qK z3d6)zK8(xn=*n9b?aq;~l*xpai5~a@$y;k)9IhC+m5HXYAlw;eVd;JAR#-Z} z*RXX#(>nrDdr#3w z!P)c#-CA?pzPgFonGse`d@W2&Wy!&aBqfk1;2S}D@l$ldqk|Gv{b-P;=`Q*je>|rv za$$NPRd0k;sgHXJt&xo%5+B_jvv=(S*!^DT(8ukmXd<3AVWP^tB`u}TEEFLgHsnYk@ z13;i?$dt&I8iTmf4(K3$@*?Fyx}pW8Px-x3ITTxjj)g|5a6cxLq{)B@infL4LDWqyDntYZTHU2aEcRy zi0eI^4kkv!;C%=%l;8}5Zc|3VOR~L4iJS|u*{AUVaA^uM0U~@5uHkS*`_FG62-R0f zf4QW%ucufl8gNOx?kFrvK6omGe3FrgU`2cIOOVC3O&4P~Y2tQQUB3a`FM_wDRD>g| zD2C>UZer-vob~z{SN@jzlxQ~P+{~N$;n9Azo>NR#$z;<|$#*mUC^@ z-GL#BXo01ch(d0c<1jJ)^WrRO2g6L)$H)ad#?ufKMvYEaxUv}7Wwd-~_dN|f6+d|f zHiMnypMo=wuGpq)#ZkT!V@S7^QQjo?;7Y||+5+_Td-=b!MV*)b+X4lT)VcrViTeMq zC+gp`O#iq2Q64_3p0gX{|JeujanpZCi_~v96!A26_(}7iUnl9{_!t^hAG*mOI@GQ) z0Vuvf$}R#Si&HT(yGj)>SQQf|K;sg*zzX0O7hc*Ku;XHqQdhYf?YQ<=%ofZz6zU7fg555RFg= zMTq&OnrHX9KN0n7&@3$7N0;1C`x9ar zwJU(X5yX{zaEg$177W~WCWR_QD4!6z@0>_55?=5E-0*%1suXJ72WCRO;2E4}$LfIn z1Z8l6L7kf?+sE_s4To}N&`1MKNi`ve?Fg=)26OcUi7Fb*8z@b2;cd9U{YXMSVKHRv zQnh&9Sn7kt5>cb;NX#_IM7qS6wa2POx*5_R*bH9=`~w=nuERGM^oE5S%G{wCi`n<8 zVhQQud@4|hKLgbl=$8HM^X;Q2Pm(*m>k`e>be&MM3J3t$NAo&a@x=4NRmD;@FPeh} z9|@8yKhbJoM6k&doF8h{0~tb{%XGtn@hqJ9fpQY(-@giLs;ME)whpWr0`GH9L!@Vw z3l}bU@m)lXMdTR)cYA+Ca5!^x4_;rcgsLsB<2~Dxx3gxeX8|~Bfl$GUXHv5h3^qV; zXQ({_S1sJy$5sq= zGZ6%IJ^bQwGDfomTc5{A5r^GdDw0Xbj+=#IbTX+1m*)_chl(GGyT-(4jSLN` zkERL)%@``eHT+K8?CcNPh9h6Rc)_^}*I;mtp&$t(^oM3Q>e{0wc?f~%-%&$BQuF;P z>EDBT{yv_U+=%Nji_Z=tZW|UH@|l1{fM~7R+837y6@(acIb zfiG@vZn@-$s+8Kf&{%L9XlksdYFi07EBGXjOt4i2#4ClXX!E7KnRbIGWGUbr)^Dmk^sfE@r`bXH*a4Tr?PT>QcnI zQeT=6+8A&zR}UhR8Xjy6HHiu2fk~OaXNQeX7b6pc*MbV*N1KOW$0B0@y4^^;LE8=( z4)u?+t5>h4TjR0mOmQ*gHLo5pjG)q*L+RRE^oc28#uY%+h4H_2_ar1CC8JTah@zwtQc8;@ zm7=;|XZ)A@exCa{p8GhS=Qy7Ke~#Z`X1Kb(-|y%9`JCr_J0%@Lpc#ao*_T&Sze7Bn z=WB5Ll{&M7@Ar?)>7|^oa9ld! zQomK0VwLO<0{3zpx$!E36Y`8U<$l`uC%eY=gN=TDLA6k#S%OdvwjU1&r1N&eUAoH$ z@A4CdfUTp8x?=WeBncBe9E*N|xSESfC`Q11ncvc+NfSwxs{|T7$Jr|*a1f8qc;=j* zmi7SLx=fO!ZS&Ox{aFboH z1{wE39bL)$sx@TDqkapq$|XAhn&4(Xg4bCSoEt5^XgES_|rlJF+O%U7q+&37% z4=MV?i;b>pI(kN;!2`Xl&~(C*B-E1EQG)uTcp-q^1I3Z45_jY6_;~5Sfw}EMonVFL zU;;TX?Axx5zBuXRBv>OifsH_+q%@h(Zqg#|)?=r2%fF0ddb@DL|D{hB!QO&#aYYVjm9kbI7WDzz%GSd7@6bIU`OInfj*=W>waSER-H!-Pew2fij2?`)&G{p)y%s;Z&wKg{Oc;T@OP96GRD z$8;Pzy#c3$DV7t4dKd0YJ8r~);DylKRjcp!QOQ(2O6jrA-IF00-G z@9CkSAiGe5vDeCXg!}vy;jv?FMzE)!j)$xI%R;>`E8tEx&P@USf_NiDnF1Gti}w+t zFr4G#0cmRy*;{}vc5OGEiU#(k4_DP4<0ew&gB&t~jv%O#ARl*y(Yt@zJL2`LC@rCs zdB92^f`wZGzK=`??xsRGZnQ0e%1`(uQ7?kBJYY7*ECtN)t{ErC$9`}LhsX83fU>?M zB705J`a#xqHM|;{N`$*2?)}xqUBPoaX1p+Z%fQUxvS-UIEQ+}O-3;|p>miVz-Ff@v z>gw%5E(a6hW%TjcAAvK@{Do*;=vk@RFdtAYrcn_*I+C5l@S zolGJTK`wa)c=GCG%OT| ztHANs-=4)Dc=tx5`H8Rd>$r$|ClkWHzx?&=pZ`w3&e=Oct%G-vN92?Kd$bzEb5Oac z(D2Pkc2hIx(P+gKJ7!N!bsGXH?GpZrMCbfJO9g6F;eB;)HqUI75vqOG`L7~1`#+Mw z-E?_MRrdj7OCXz+WCL+H2zg0pgQ%aHm-hjA5DH#K?NHTEk?<9OTp45V+JLpn&HnF$oBr2Flsx2$7ouWzyyRxvX~YDifk0QL5uhE2O5yToCnqu= zUgutqtZ~5;4Dv*=d^aZ}daJ!0q$H??**2hOC?~|24p$*_WuLw;T zAS!a1-~{@vP>PDEGJU$SOkp_Z%Q-@s^BNbJzzCt2K6hnS%M@`I?=x)os_KHzgFGUa zkO#mIFoN^zw&DGdGc^P>NaRi&F|%+bu*aX|5RjZ5`6q;n{{Cqn(p#`69F1Lkkf5PL zm<>@Nv%xrfJ4>v3rkt{wgma5WL77HS2{ydf-O5M-ImeEF z7ymK6$V?zMaQGk|h8;bf?$IRX}&xidlIMy(9P>pGL&~7qC63T%y zLW0me7C0(FTBcvR{pD~{d$u=LY;dF@LF0-yTkJrM5k34A7A8dWEWT97)o@J=F zkigHLJ!_mnBgsFuKx4PtSwelG|F{5hA(hZ0YCviJs7gfJ<_PH~KKwd5vfJ&xDp7an zaHO3T@CiV0Wo}eu>#XH`ElnkhFKXTMo!5*Q=ODQ*3M1y_fQ3Zbie_xG3iAIn`Y^9- ztT27VCzm`UmRqIr6DTbkB~~CS1Uu`tp;Ky~#TN6ueW9P^F#^PN+}zhtwzGDX&F4Po6%tH!5v?`~s#(nMHCkJysl`4ABQ(Yrcj~&yg{^>2{BE`Tp zBb4w&m#F-_)3Z z`pIQ@1rCEyQLc|*O4(SI@Kg2+5s(m4{%y2q=V7C;y{yYkX#UFWe~Z06Eik+erON9p zA!<1tzl_9$VaBx}i7hjM9SVT5a7qHLPEM>_7LCF@Y5mr{7%1M|S)E%?gDkUJ#kmQ+ zMUL1*nbJVT1^<=!=Iv_2U1cuXrX+G6zEnuWdRC3C{(>c_J*y>L^ycTSv`kSf10M8n z5u@U~MU|6I>Zu=^9Gzc&d49Z#aY_^TqU22pdnOu3Q3^7B6{dCIq+FO>5)w>Q049g+ zTu2}o(!LHK=HSQ{&70fe<`UQ#oTtCBv9V+g2xW9a`<&ckZIj zzm`t;@uJ6$pMT!l_hi%48|VCd`OsLCU+&h_x#jt!XjY$fd%2;Ya{I{gxFv<&o~g;+ zonI`!XVm9WZH?r!f%FNaSj250QX^g9OM)0sBCcc|7Cf2sMcrnj`Bu}G0B}NR%-sfl zjERW}IZ-Y9Avo>6Rc~;dggZ~0K7I3?>f}MGMW}NmsMP)N;WLO4ZHfI5WNXz>FOjoh z;(dcm?GOCmy_L1Ej2q4asEH#_?iI$1BnR)SAKi}?F{LNS@ARH}niXyIX{Sz|nwFXE z@`fm)bVWU#i0J18Vnu>1;!t)p^={ot^^XD+jo3coXHLfFgBUt#my2Ry@TSr^nBMJJ zAEG2e(RIYl?G&}X=&eNt%vmunO{?$WoE)o??6%DBId^~c*Rr^;&#g0C$0IG{U|~~r zQ)uC0r-sjfbkbBxe0Cc)3iA#@SOc9hdZL6c>k5mZeko@Eq;`{&aa8 zNpsXB5=W8?2eUM=YsZd-@9pcmHilKP(S@q_sFQ)Ma@@P=ixtjq=k^x>nf7BoUB`np z1eF1M*AyrNIWPHE`DSOyMil=l9j_Dyg0b`RNPMjX(+ERO0zCFU+ae#Dx+0E?bQGPk z&-jeoYL1_21_1mO3I@5^B#xM)9c66kX%M9a_b~>0QUT7AC=$g}7lozWH6u4; z=2aq{HUTcP*t99)iZidgs@~v3YK`!LJiY~HZ$pEb?1J2xaJ(|B8!Mqx!NSlX3v;nNd3x~=YJ@a&HUM%~$#s+J09@~1NanbJA&s6mxuc1D2Y}F@^ z*v}K7G~}2NEy$8OI5)X~T!2PNB9;OKaDKJ~49j>`WM%VI!E1XSQShW2s$Ciapk{(1 zovEERY}l|7J)5Lrn*ZIc2~6T_t_Eaz?k@>@_wAl;+fp*i}lTIg)igIjOBBKk9daW(tcuSv7Tz}a4&t1E;^n!}6T7Rt*_tNHivet%NTdnkN z?LM;ycY*isxNHh2@`v|Nhf+xKHIBXWK011&_+nKr^)EQyfK#0-cCXm_N0jyp8c)=2xqrfl1Gz!FJHb~AtTxW=;dXjMZvk)urvQF?Y{20bZNr-QL71T z3i#h&nRxGB28gI^hRQVA*(Uz1G%;a+vOoSKaB3z2;S#eQ6n9A>hjshjruQMjw3I?6 zN?q<(kIXul%{rsYQhARSI#s%dGIUw6J-)nv+EYeGK*bdc9w=fnxbk7aCHMnBfBt+K z;>+&D4#a;}yiA9Ylp)lb@uKSa+{x}m)#O63l(@ke3z@mV=%a33%*bJ5yFMjs4IHdY zr=Jxt_M58Og?B>Rz2)6Gq>y}X&LWFl2|ug5dN`0PfPn9Y+)#;tuwH*#-979p_^Gz^ z72qZNsR>MvDDkLaex(Hii#3&lNKOK%C-_@i|EGnV88Ub)2p>6v-OVUearMf8W#P{g zQ&R!GV?Myp8ip97_q1kL7hVg@ls3-+sJevrZ3eGWCF3Oku_GUA01k@x7*mEF3X2ID ze1f&mGGXE24*B~HgmmRZn=x(Lf!@U&WF&kN4stCZD1Ko}o^4 zh>mrXQOFgOMt(cX_VMz+kNUBv1}qir8#koyh!OMo@bQ0|1=%Em= zEL=__he#DOS&EC2RAkWXE9SiX4v?>?$zMm6gr=iAU$noQxk?UD_CV2F1b@FJ)r0q0 z3|nC0i76?w_%ys0{ZOi1dzjEN_4&*VZcVG+Rh8ur%Sw0^8Q!P1tYlwT--fZc8bo0( zL6s83OJk<4g&sSAs>Ke&syMZ9^h>iV`KjiAD>}#kvHA+%h84iHPng{9xp)h)@PzF5 zLJV2LvhJ5YXa3h)5;&X3yYy0vY;jscr=<3g6*>WFolst%|D_5L&A>%0= z8B{yDp8>TIy0faCRn1e)UiYZu@@M|`+sVpLHTiECk7dC*x$fGHUUEFC{Af0HP)j0B zz}rI17wI#1#!!Egn>@*E&<@vEah=p;t#Vv$#m3tTaa-;PvafRsKDq{!L7zX0`o4fvbQ2 zcl!0Jc#Zt>pUuJ~SU?7kdi~lYa5jXHLTlv6oD?^;SqG~QPU&3M(?g&8HtVue<9$2w zpkC)&Bc>pJ`Rna(GM?=JufHBu-bT&e!^8djKVBoQjrif@ zUB3CmPYM3P`cGU~1s0DZxQ=!2PE9iYdO>PE{*$Wjzw)n7j+f4v#Z48*e=v*8JX?!M zMWQQZI-*EoMH7ET#9tlf8?cpQMe&z<99ZEz`=0-Se2NiiIuIlGM@9#xV!t6CzFS{vk;6*|UiP^(SH zigiSReWNiei5}sIw|6i@(N3HAV8tkgWH+P!aiFs~sR>%bpR`1(oT_%efw|OD z6;!gPuYeg6kpxGqxRu0JhBwKAf12BlKdYu@;a6I^N-~a_A18LDhv;rLtnP!#iaYfF zeeFV*aA4yo3kUoL=P|Kmt%}eus{lSas;H~H;386Ng;F#oB7~We=G>%cBYd5S-7finZ7y08kZW1F~|G-1TXR6ap!zPh}II^XN#v?S(7SX+S| zK~_Z;4vrzM#E zZyQdEZi5yWXu-;14DFr8j(tNPxD4z%YWr1*TnY+}nk){XmW}~X0HX;(j}5sH2Ks$} z{|spcZ|>@BUVt75z9S$>K{LzBX0;O?SzcnE&2PyvQ~)~lNesNN{(r1*THIomjW~KuYjtK zN$EyeA{Z()4*xJ?iOId*`Q7L^77m82Gwf&m~^3u&2O zoSp~8ut}q?Ln~Vjv>k~NydpyPQnCmiaXn7UgSEAQ;X5LsgarN-WV8s8tNEHWp`v_a zV2d*C*MlQZfBw9^_cOrS9zp+%cfp*XP{OY`%sJuV5dgnxq9SV}#J5ah*2RNJVil0H zgIihQt5-otBV^~03AX@&9vZ9IiQfg%7^+rSeHsyqb-&A_Tt3B@7J>#q8QhSD1ry%2in-V`2|Bj62JReeFC@QZHrKz&nJAtCV%wVAOWz(H=$>)5sH z`~&0qYU}Au7ikR(nEb77i@*EsyZeMi1p-?H$!jf<*4s)iZP;&cjaP=He(uhDZlYAV zWIayzjOYXuwZR%E&d%*qqOIhat{a19D45nL5Cvr$pY!#u5}Yl35XpCvb18E25UXqE zf-i}!7%Ma5wiCD$`-J79XtVUIy|$%5p-e8eG6tF)c#Yx#a$5Wo2d5 z(mX0_^X?nJq9;H8SDWsRWB$G#Mr5R{L~3gLn$s-8Hlm?LL6%sTaGbt%$f`pJr;lm( z6!epKR>-)d?u}K$)g(FbcqhQFWJIN`uS!Gg)U$Ni)G+(4|JX2kdKg1G#Jp+b_~Aa! zOt&%JPoZSXRe4zc-JrXW=8o6rirfiubRut)b7kh;u6>>PgY~uh2TBW$B!U#)DZ+Dd z6;EVD?M~xkoHq(Nob6MFOpD9{IMOj(@St?utIsr_B05P*MN_G%gu~I!bOsU#Ui1f% zG9OU)$vdA0Sow&H)n9ASzkfZ4fVJk^v zx~B8667j^2IX)$2ZjE%Q)L{91RvU-I6+pKRZ^E zo$1QcS3}&?-Rh+m58y(hF@W)%$C6T61d6+_-NS%JJ@lGSeu1Ks6UkggCCHZn1IX=C zEk;DDHkr=YXmjPvA&b=3t&+&DHnD(4vuCZ~cNCbP9p8YOb*by6b~WS$wJAxR{^5F; zh)JsS?;Uj}J3HS{*nUN9IE#vCmhkMeOWw8J0KZJF=s2C;{C74u-pIqW1<=|c)Bf1G^L>D<)t$qvP)h8O&vY2 zE0?_`kS>vv1(lwmwJ$nD(9?N6xAl==`(f4#9%HcUo_O`iGNlxGay&^Oh5#KI^6sk! z17pZP<=Y8yRZk@#?mPe52m{WF5l6Hiyn$o9=QCo8c2P(yEDvXcOvjn+$ssIpu0rn# z%tMN0(6n9OmR80dvetYyVq+2M7`PfXf;zN3cuK)p9O*o&q3tC3&Q~p@=;N+Mj;dPw z-N?^x5xz`c z=1`#dk~&KJ*$;fU#Q}58TK|U1doIX=B=ZjN;K3q&>rb^ZfyMmqo{QYwUbbn~3erQV zTtDwDWw8_}a?UgRv%~cmHYE?+25!jhs<#I5E2xxJe!#XUf&EMyk5GODl)eSDcboQa zJ1z^)qNndEd1NC0avex@>?7c>12ya3GaQ$FW6Z%2^2^pTFD$Jg{(pfK=8VhlH|OF_ z+s+VNax|z&_TDb8P#KVRWfb|HaF&^D`bnQd{$Ll91cZ!MZLbT?O2r9xd_jJi5*_qm zZ3o@PgW}v|y8Mi{XSr26VWYT1@fy3=8g`1nmT#VNrf^k}?~B4VeakKe2{;4xL%&`a zW(v{De3>(B=34wzgr zaLT`7D?9eJ{f~ieBQ^dT{rcF!>)(tn-rAyBU}m4LIo-cC^T#W#sYH{!=0LM^j$eTq zZWwFOZ1#u$h!6imCYpZv>*jx}UuU@vjL#i&atYP0eth4-gCDYlzWxzkPTu5JD`kM6)6qqh`{uS zGiH$8!1Sct?!C-x1qlI(h2~r%cnO=CI@y9UtB2SC83lTMT(iJrikR;8_2BsXua9f0 zodS*>5>fs-otcuGcYkjYh_z5=gIH3u%Ae+Vkv31rWO$z3?mf&-G2Cq73;Wo&E&N~Q zmy`b~1A)YXM5RY7NR;U#t_hrZq@<2RBUzxk$nDF|16|aLG5_9psOUHsj#?;cf(X!ji-L|H0ZJb`i*6&Hd4uT`7LVt!&(SAsTq)aV? zJKxW%k|G5f&%SrE?~9M!Ll7miq zO>SLC9(D$i1)8X~EP0+P_lW{Ojo_x@jdfI_jUkjWT?u|~_O1s_%0<5bZ)z$jO2i?E zY&kCXmnQDL!2NIn_zSTsoEgQ2yMmbHc^r%OYuAm|oO^ZMDbdd`|K#xf1kI(eWl`@F zb>#CnGxxq4`L;on9g^S6$uIR636#0Hzij=%@x#JDv4LDcf~QP_IPb3d{6!Oo5=SMW zsC}QH$KWm~^T!1@eTC1Fd9#u+ z0C~B7^Dwm?lLa$bx_o&FWJ0gIZ99YkC@9ka8%L{RT}}83ax~ zMr&#BYpwQd6s`DSh27_A&&=J(uH!W|QQ1Wy61WuS%3VFJTRl`JY1 z@X^nZ3sxgV07q$oRhFnBgGu>YW!6JzWx$B)0km*|bR04PjwxgXg(klTE`5{I zvvcS3toYk054QAZSJ&y6PnKz9sh}czb2%A~dRd+Kks}tG>-pNtDe8GgW+PRT!SS1S ze`9f6f)1C3-(422g${-0N?;MdMhPCV`>yA+)Fr$$!gk{=m0?5tnX9)!GsF%-s&z5Q zvV>5H%tT;ik@e@uHpq*=x-MEW(5d`Eo7|yerBsJSvL*No!G#yvkQ@yyvn#SyDVyhQ zJe5B#%dIH(J9aS>(Z53?&=DtW=+$^=m8y5{N_=<#)%O-B0*)sP+{~Sn2z$L$XJLPC_6 zdg3&jXT1B*KP?n-UmP}Jg4=2{5 zK<&Xd`}2%M{HttU-7InU_=X90M%CBLbMWygyKMS=eNC%_nQhfR+7BbQMLGdgF`~?^ zMNPZ)(!ME20er+8sZ7IXSoGkfqR`sz>-Y(BT zS*mOvXO-1QulY!8w#Q&!9(L(vQ&34#Fngar`OS}Smsim}YO!Gt^_q%=noX?7KEB^M zY6q2~aj%eFYWr9~O~6=-xvPM1Jvh+>vy~iqN(c$NrkZDLgaF`sTXgHvCGe#IbyDv2&#NmDF}N2yu}9lC00e)0wGwi5COG6LTAGzvZX z%?Xr-)5UToE@P%2Q*h`H?yB2al*>?8vWR7}!@Lu~+CV5JqI2LmFoDsy?y}eEW@;9f zDUr_6#mjjgr|}YhgcL%o(qLXDZK#UWkrl38e{HV)^qaOL8)y4)4Dn|kvOj6oUHBso z(nM2rOhEF4l{}eSEr86hb+&w9f1bkCD zR;0$;0^VbZAY*dBP8>O`F$?F>qt)kdvf-=NRQ~?^?-~EhzxP9z!FOIi|K@XN#0c@H z0_Jg^aBuN2uB@!8uBb>COBu?>t2(A9C#%d&Jmp8e_`w~^DJFS-F$quPeZ2022P9~&i5GlNJW|y zw%x0_3xDGB1Im4oxXMgcWTj**%;M6-0xl zvw^=iQ8T+Ox{e+C9D~Q)&xN@l9kYmWbdae)qBf#FaX=%udH3EaCb*Shoyj72R&KVg zaqKWJn^YsRrsizsw~6kFchgc}smr!!5NxVnkD*^nV&0b^XUP|>y8tSmaIUp2WAF#~ zrDRN+nwt8T+oQFkhw3}fZB^`YniuUMC-!}x+H#ovqgj^4(yBnR-= zQ}xN+jG=24MiGEbaaZFES{AC{`2}2uOmitAgUTJ4{rauCnU|`YKBWP2uew;(?CX1~ zP`^CM!j<>*94`!EI3cRFDsRNAv|y5$Ea(}CoSikK65WIZV@ailn3~N+h2?>jzs3VGbNlw~myCS$ZZA)~{WiFW zhjpW-uxtN07e`ai*iZSvr6V||42Tg^2lb2f{Rp1Zad`zv0+PPkh$U;E| z=4HutrKv(5G?VWti2;&M%@*u~;8rp|go21sa%-+ zSG+Zid6iX9>{Nc7J{^@$3JVKYg7J!>7oIMtRHNl5*eG}USB~CUb5%V@*I?M#97)1?J;(ccQ+8AJPR| z^E8fN6_K(-lKghJ?U}(yljmm6)EZABR(pv@N`AES+i8zu*Z{f5I$KtKRb=dWj`@}?Tbhp_|L3?Y1Z(45 zYLXU8)Eh;y%t&Fp_Pnqqdy?upow!;hB0AWrP-Iy+u3wYU_>jGuuW9+*p^PBmP|BmZ zkh;7H>GTX|#ahe3r5EuOb2>e5AE6Iu4MeU`NlV>fe!yUU@0E4W$I6f+lCb%^nZ(`< zUUKJ0!Ez=fW|C)3X(S%Nq}5$)ETBC(bVNQjAp@@8lC9L7)gmwo(1ioDwQ&E0rLWyu znire5=(NP?&6Pk^h{xya$}1`=y1(WbC3o37&fLvVF?mGC>YDbcczY6LWCtC~((o*a zTi*S?ASPbMjLHOZo|+&*!f2y|rUYH48J-%+#5##XLJ516V@tFiARIF9d|HyPi>@@} zV#q;73zjEoiLLW@g$ z*8wTIz8gA;g3XU(#?@-9oQ4Dj_B5TWL2J51j<4$~r+vH`?>XI!&MqQ6jKX zf@t{=*93vU3log4__bNa)1#&1_H@I;b870*P3`LL4ZUMb5&dm8Y{j^y~+hWJF}BNd&Y^(p`==a-=6w0v<*ub-yo z?Tt{8i?Zn_GTsp8S!mo)b8wswlA|0GeS@5Je>{976 zS*v>Y>UC?&)~$21w;-F?_fTuQF0Kgc{IasuxyAr2g@j<}$7>vRcNZP=BKOLQitr~B zso#>3#(Kw7UD$}+eD~0$#?z~o9yF?&bN=4-T!5PWib13f2b8>FdhV}xx8v_^lf;s- zQ+Sf~k}_{7TbNCFRGFa8)F#bZw3r{uEav0jYjX=?$qgct$cF0* zshD|n>J9o>|0fbs-92yF6MFTQi9Y*So9z0eh4tnyqdPpUl2mFx zr|e9^Ez(XcnCq)9I-0neHmzGbd~~8=>1|*TX|mndsme1hwiDFMY%9VSF0x1sRLLX} zPkz*{R0y27_D4Mo(h?;_)3Cuh=sNu)gYs+O1!;$X0nZTKk+~s z+V#}W!;BJW^qE@xkBL5@eE-TTP1J7YeyQekrku{xu#uc!g(oK5?BY#O-=^HsxyQLbaUDob%QEC)84abz#B@xknm_FCv{U0+gma+tZ>e_*Y2G4K zJeqw;kJaWaM`HOZ^zlAJ?A%(RX&uLbr71d_+{aq9OT0uPBAszXtJbFpFUk} zTx$qzwiAl`P&Zw>(+x~97^pG@uTAC%no8L+F#Z^l4{^Zhtgqot&3A}cle<#UnV>Fv zw)=qW?u6UAEfE34?gEKwqU0KgVybX0qCP`X0|%b{+0gsgu@&f@7Arh)6^wYO;~kBO zdjLM8q5LFS@3dJg&E=ZgBk(BFHAtUOVs4T3iR^)7%?_HYjljX|np%Cm)M>pBJCdB}L@ z%)qvcq}x1;Im$%Nb5C#YZo3_KE$rc*fb?Y*SNoFpr1FSd@OER_lSj4H{n$krnxIkaqW5i*R}nEp1BJG!L0&-zz~SPBcCV{VqDu_L z(^QC?kdNRz%VU1Hlaw@w{NSNt=c{2C4;jvFKTEwgQH7l1_qnv=vz#$xpQR$3r~(a} z?q+kTPc5k_rgTb5Q`5`ki#{1fdWi&Y5NyoAcnw*DgK)d&mz3Dm?Kd0tzy-jm zWI|5+=_40i8S$x-cgxl-UoW{nEdO&G#@kpWe|;J4PUoE%#wdfs>18JRjkwZgI5WNs z);V`?-O>v#bwTbO=+@WGvAml3lrOaJA+=zjyCSU5O1;bF#Qv?JWjjy7MdZo`N7Jg8 zkEGkbm~-NTc8*TEE`l9MtU#+h^*BQkP4wAs2aDip-m;f>+@@`debRmpwyAk;!_jU? z;@(8nT#%8tC`31%&p+$>)4fu}qwLLHIri+KF6_2GZ)Sy7$tUBA?{4PK$1mDCp?9sO z=eF`|3AUcCuEk@Z7@$ffXKyuT+9~RX>%25TJ;GGK?Gp1eSBB&^uJ+ceGd$mEC1?Hp z=PLELwQaa1NEy(yPw&98)yu%+{D&V%yILBS%Ng^eU=>XTJNb*gTy?r&g_ z$z>&upD^Lt_A#?02I^APHQJ$GB`)YGgA(4=C<|Im{m^MS3o$-3GqYKgYdM z_*1n#waLDR=y47?JcW=+fnVHr2K+T{N(q6SPVw{Gk)QSf)s%f)_I%@bPQ+i zK&K4(_Kld{u+((Y8U!*eLF1Gu1#=1@Li*_!#wsM#0Sr*WE~yoC{K_dWm$*J4he-@3 z0olh9`2G6@`D~_6fYvHMt(j;0{`~XFe>B9L$983+0``45;A~WGqbshKUCsKy9%Dl0 z0~{FqG(}vH9EH$CGzu~$b1KhZqQP8_BE+Idrzl~{<3-lmF zj$1uE;a5>zojD}aDu1Id=g9h~XhqdAH1ehTXPIkcq5|1_X;$4hq!CCZ!sJ%miO)Tm zNFFyu<5N9K1nCB^qpCKa0-Bt48 z_R&FA5t5_Esxs^^VA5Q@bgL^imet2%axqLiE!fw>vaI~Cu4@h{WQIi-UQQXCg=Aoa z@gehJPPLaS&Hr&8d5_CoLFBQRb%Efoh$;JhvLvYq?ag8Mfx}H3`Ita1tc*tZ9p&yL z6vshqS;hRnj>9TWrK2xv4K23#1lg_oMG6p!Pm=-iB2DalP!dfvXDt{TcDno(pMsE4 z+WZi`a_7yJ-emus0NB$xUz&N_IY(k?lWam!w#v-yVwVITAD`7%`djLKM^PGbH8JUt zk<%}kVw6_-fu~-2xZdKQnCGwO2v-ZQrwmP9%JTBQp@x3rlwh>9A%WD&qFtluc~v|5 zo{h`5>L;FcRok)8`hkngz|t03rKXa|+i962O5FPz*0+r?x>h0`iijPcmxoy|M5yWE zP*$QH7;2PtlwvE!ukV4FfKzpINYwac?bLCOsjYLk~z zvS%A5sIRX$Iv-s7sa_o8C|lcA7Y9cA2jySs<3~7)<%@`9W{b2d8j@zeQ6 zGz7(wGrhHPbiFI*MC>E?k#B1qk*_Mhlyae6q&`#trI*S^-r}g z%BxjR-UwW}-MTcv$J2A}%WFlhbyuhRjtT+JG*6hdAhf)=)wKmx%PrzoC4FyAedEglODnI92}5q&uUdF~TkuHW;t$mEB2p2QiIZPm99UMD znXKy%z&jp}^LH(5cJx(t7Xaw{98ZI{e_U5Gf1t#vDvBHG-!ueo0Ah_4Z6x?sTJ5)E zY-d&47tte%KtMM0?A4LQG}IC!zz*R~0u@a}$s8Xh?mbUkXY8@N?kZ)gRbYOPp{wS& z$G>`gVpC>i)S+!>!cI)jw6fnX8XF)Ykw8m^gT&oYzs{t$04zC@6jjxxCC5V05eUF@ zKZxaTw7nAFp^8( zU%EPH@64yiO3_OxU^J=do(|8f@@oK8T(IQvpRIE9ZihxEEWfasm2TeWcIj#fExcXT zuYYlmb^e3%%{(5K%R7o~qZnc@jfGvwRuRM@B{KVRGA7LXq}SEeg}h5HQ%0n5?xs^E zpeK<@BlDBWUirVBG7)rxBW60Co_~*{an1|$UK(2T5FS~`Z@yWPKpJpna}`=E-|VuU zJMy{njYG?$bqb8ifpJQ*M|aw=&^>dPmEJb%ZS!^aCq^5zOntQ8`JmHv5TeCz@`1`F zURz!)LC%TVSxNn=^22C0OH6g?j#bMK1=Wm239ZqnV{}hEv9Z;6-(8IVT`z?GN(U>L zD)@V4DtBRE8IpO`MgpEk^bg-uHZLHFg3fO0$vHC|%4}M@;6$2Ma&571@BoB6BH2yX zue$4)6c{})OC(0GyOX`WSV{z-<4b<0WR?D_mX3mZA@_+A}N+Tvx z1WH|d!}T9(O3TJ>aB90MH|o*k$=6mUgg#DcfJ4w-IO>RhZcT0NYmSMFfBj>KH5~40 z=IdHUvA8MFtHeHZP&DGqp!Aic;i+iJfIVuR3b1H|$K$T(B*rc+P7{i6%7z3&!3bjAuTLqAU;x3}wuU?$>p8ZR!02%r1HL2>K4z&Xw-0azJeaD%Ls+3#vzyc+8Nmj3J?B@*u?XL_)C)8!y68NOs zRYkgXsbw!7w;yX>Fzw;|r*_@CbO}W5HbrVei@-~=@Rv_BsjElWiMxv(sD`G4(Boer#*Ky!wo_xWw(}#$SaZ5BJ--XWtZg&jcNocow^E zYesx}{H)4jgZY^3$C$J!d#B{i&m5KY2E@Rp_uxoo=74D7hA;%D?8&*#UL3r;v-IAE+$OPy#l-eJ+ z-O9Rjb-2fEBVWG;qcc`UzN}|Ou}i8#O>Lxv?DiECSr=vRQKN3$iK1BXqJ+H(nGo)en2eX(T2g-qRP)6+J7Zfa2cv2{bnpl6T2ML%;DV5AC zzRY!CZ;Cn~{DhAmKWZptAgTJiYJpHxuAD+-T?~-4kqr74FMzpYh!2i~AR7uKbh&B- zc&9z!IIs~6&rlHK!9Z*oj-R&n4dF?5RrVe>>~S;|S#Lwbta%;`1e8E)WEyYY&+EpP zx@Ft8uF67?x1p+JRB2Y_YpHb5(WIK_b1V%;Z!#Da4_FK3L$WMS%vb}#cAuM({^gW0 zFe(gjl}FhHAd!ixt-MA147-NXCOk3GrK?d!@Fr*aL{E+!J7c~<_`OUHyfQHa5zf)+oC6cG(T zFdAJ6JPz5kx!$ldNvfUkW!7*ZrU;s^Y-cMv3-vTnh2J6KZc0l3O%1z5>N-(%4JpZF z>*a*82)HH!j5|?)HJz1bVN429TJ#35|6HKwGEkMwn>;0^t&M9jr>@%)4)0{amMDq= zPDFI@&+Xg%IZls^nsA-fUK^H?AD=OV`x*^OJt)3IPB`bYKI$Z3Ms~Q+rR#4i_ zFX1lb_R8*`622G}LdMYY=!$9K1Yy`&Qv}YX@$AfDYYWRXE{26VKhqyba~Vw~Pf#>Q z_3sY4XaLp|I=t6tjb$V5N4qvaW?(#rZrsP(+Ahk%10%{Wp18si*OwarL7kEJ z-LBNTWOq#k$WDnR&nL)QW9I##Vk-h?2Dl(|q(jzn^1t1dP)>@t*NOpSH*fzn9$Q@B zLECmd1)89JU|NX=Wl!t&KmPce<^)A&@M(cM1C;f*vbA&{tTp|+;bQjTjw(|aN%s{V zOC+vXGq+{<1sB+x;PIp79}hmXglxZ#_H17*L$*;%+VeO@c!O|$r*p}s!P~dl(D-A> z3+I=_L`R##9HAtL&(6-Cyz^+E2V`Zg5rJjU4Wls{(564mMBbgW`f>ZwRjHT-_vXHW z%vgoxBji1P(B7cdj*Fi4YqI6kp88vNV*sVwckOD&olha=&Vtcz{!+H^QA#*$pO}W?k@!iB?n~zUz2?=9BR*)SGep47pyN8=zF40C9%<#Th%{iKKR1 z>DRaKonWhLVUL}B^?>~T{PRx@!J)v#+axgf__zgnO3X(_IQ&gUiU%YxBwwK4fJpuvqP(b7Au>gVq2C zI2dWs1*@d^U^En>Yomb`DJJOpcS`+Fqu=)2NzzZ^-=baLY;OY8^ zHz3!!R`h-uwc-F?J_hrp_fz_f7W5aJYd&vdGx9lcn1L4~x-G{RwC7*{S~sT(v!P*b z-iSBRHhw!j*x$FbYdpx4OTVwZ1wP)hy}J5Rq%*4o93O-f*#5;od(2k{_>GSp*gWRX zH|Qe)?z_tvP1u&H2&$5yr>rxyygM{dNT$FKuB66Z1ED*WaL;5O@yoClCA%J8nU9wL z8mi*pW8XAwYKkT#nVAsFWbE|r1^0I$V*Cd__wUJl$vw+vYdLFTRd5L&IX%|J#hz}x zGki?@4jq;VBrg$9;nZbNP6c71M$dcS9lYxCMT<`(fbO@#L{H^Th#N~_eF{|r>Z2jj zv1KtZWn2wN->bsH<3f|*XPu|FVsdu8tJXI2i0I6Z2-ucUI1~a%4B#N^jelhD>L(9> z=s7AUD&ti)7Ey_B7yaRWVr2ImI50FaXTv^_-i`BaT;Ahs)6%hKn%B^j!M17teEjsu z3trM9$YlzZu#iVwgeoBrYPa=Zu-nrA)ybp)@$eNdjc91$Q4^c7~t~_3wd5c zRLP?HT#TXG%5+t{#S7D=5X@_q_%7MNx!slreE%H}OJ4_V|;B^F0h5g|; zv)^o@R{xSh{~U@OY?{+hs%${!uXd!fjmWi(`^Oix*kRm<|HT13{&dgA58{8?#%uh) e<6^ql&|>DDQ3F~oGMyuzt(-7zeCXKUcK#3Dgi>Sx literal 0 HcmV?d00001 diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-windows.json b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-windows.json new file mode 100644 index 0000000000..dbffad0093 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/trace-windows.json @@ -0,0 +1 @@ +[{"run":"read-1","file":"tent-events-slurm-h100-206-025-2231481-gziMb7.jsonl","op":"read","loss":3.409,"phases":[{"bytes":4096,"start":0.0,"end":5.999191,"completed":620338,"mean_us":7.858,"observed_GBps":0.424,"samples":[[301200,0.0,7.148],[301201,8.499,15.645],[301202,17.008,23.86],[301203,25.291,32.361],[301204,33.882,41.006],[301205,42.442,49.682],[301206,51.124,58.208],[301207,59.6,66.554],[301208,67.976,75.237],[301209,76.687,83.838],[301210,85.204,92.56],[301211,94.004,101.331],[301212,102.757,109.661],[301213,110.998,118.013],[301214,119.419,126.542],[301215,127.923,135.069]],"window_offset_ms":0.003},{"bytes":8192,"start":6.054211,"end":12.054214,"completed":631413,"mean_us":7.377,"observed_GBps":0.862,"samples":[[971904,0.0,7.377],[971905,8.829,16.243],[971906,17.7,24.876],[971907,26.276,33.479],[971908,34.938,42.169],[971909,43.588,50.652],[971910,53.228,60.813],[971911,62.427,69.7],[971912,71.144,78.566],[971913,79.961,87.615],[971914,88.993,96.346],[971915,97.782,104.997],[971916,106.347,113.502],[971917,115.951,123.918],[971918,125.512,132.727],[971919,134.152,141.446]],"window_offset_ms":0.003},{"bytes":16384,"start":12.104012,"end":18.104013,"completed":597733,"mean_us":7.741,"observed_GBps":1.632,"samples":[[1640752,0.0,7.455],[1640753,8.92,149.777],[1640754,152.933,160.979],[1640755,162.378,170.005],[1640756,171.573,179.319],[1640757,180.709,188.497],[1640758,189.97,197.65],[1640759,199.18,206.882],[1640760,208.35,215.936],[1640761,217.339,225.844],[1640762,227.228,234.876],[1640763,236.28,244.158],[1640764,245.936,253.942],[1640765,255.267,263.133],[1640766,264.584,272.496],[1640767,274.074,282.03]],"window_offset_ms":32.335},{"bytes":32768,"start":18.148182,"end":24.148179,"completed":583968,"mean_us":8.146,"observed_GBps":3.189,"samples":[[2277493,0.0,7.947],[2277494,9.352,17.39],[2277495,20.216,28.198],[2277496,29.705,37.965],[2277497,39.379,47.125],[2277498,48.563,56.509],[2277499,57.953,65.805],[2277500,67.199,75.384],[2277501,77.917,86.47],[2277502,88.133,96.44],[2277503,97.925,106.152],[2277504,107.63,115.783],[2277505,117.195,125.491],[2277506,126.833,134.82],[2277507,136.177,144.347],[2277508,146.903,155.41]],"window_offset_ms":0.002},{"bytes":65536,"start":24.199324,"end":30.199335,"completed":568642,"mean_us":8.94,"observed_GBps":6.211,"samples":[[2871855,0.0,8.922],[2871856,10.404,19.084],[2871857,20.486,29.278],[2871858,30.699,39.715],[2871859,41.116,50.031],[2871860,51.368,60.036],[2871861,61.489,70.147],[2871862,71.506,80.083],[2871863,81.492,90.135],[2871864,91.6,100.195],[2871865,101.56,110.474],[2871866,111.914,120.887],[2871867,122.229,131.105],[2871868,132.512,141.212],[2871869,142.59,151.398],[2871870,152.733,161.335]],"window_offset_ms":0.001},{"bytes":131072,"start":30.240615,"end":36.240621,"completed":465667,"mean_us":11.039,"observed_GBps":10.173,"samples":[[3389281,0.0,11.062],[3389282,12.725,23.781],[3389283,25.498,36.341],[3389284,37.986,49.092],[3389285,50.78,62.145],[3389286,63.781,74.768],[3389287,76.322,87.453],[3389288,89.044,100.533],[3389289,102.663,115.515],[3389290,118.146,130.668],[3389291,135.623,146.334],[3389292,148.072,159.08],[3389293,160.705,171.475],[3389294,173.065,183.957],[3389295,185.491,196.906],[3389296,198.487,209.388]],"window_offset_ms":0.008},{"bytes":262144,"start":36.277449,"end":42.277456,"completed":367575,"mean_us":14.388,"observed_GBps":16.06,"samples":[[3807395,0.0,13.949],[3807396,15.574,30.069],[3807397,32.002,46.27],[3807398,47.816,61.943],[3807399,64.533,78.556],[3807400,80.222,94.662],[3807401,96.19,110.559],[3807402,112.261,126.723],[3807403,130.032,144.548],[3807404,146.106,160.421],[3807405,162.05,176.301],[3807406,177.869,192.145],[3807407,194.635,208.718],[3807408,210.38,224.618],[3807409,226.263,240.918],[3807410,242.533,256.897]],"window_offset_ms":0.011},{"bytes":524288,"start":42.301701,"end":48.3017,"completed":242181,"mean_us":20.81,"observed_GBps":21.162,"samples":[[4113429,0.0,20.983],[4113430,29.709,51.072],[4113431,54.02,75.027],[4113432,86.304,106.811],[4113433,110.466,131.264],[4113434,135.311,155.966],[4113435,158.939,179.831],[4113436,187.158,207.616],[4113437,212.637,233.306],[4113438,236.948,257.52],[4113439,260.441,280.693],[4113440,285.873,306.189],[4113441,310.067,330.892],[4113442,333.634,354.261],[4113443,356.996,377.754],[4113444,380.64,401.128]],"window_offset_ms":0.01},{"bytes":1048576,"start":48.32299,"end":54.323013,"completed":161704,"mean_us":33.012,"observed_GBps":28.26,"samples":[[4314694,0.0,32.621],[4314695,36.568,69.358],[4314696,73.261,106.295],[4314697,110.354,143.259],[4314698,147.239,180.287],[4314699,184.231,217.266],[4314700,221.258,253.741],[4314701,257.987,291.363],[4314702,295.279,328.062],[4314703,332.154,364.681],[4314704,368.576,401.086],[4314705,405.241,437.698],[4314706,441.605,474.197],[4314707,478.23,510.824],[4314708,514.769,547.655],[4314709,552.008,585.003]],"window_offset_ms":0.013},{"bytes":2097152,"start":54.335744,"end":60.335748,"completed":84096,"mean_us":66.382,"observed_GBps":29.394,"samples":[[4437543,0.0,66.794],[4437544,71.222,138.304],[4437545,143.064,210.577],[4437546,215.521,280.715],[4437547,285.587,352.665],[4437548,357.288,422.553],[4437549,427.257,494.456],[4437550,499.12,565.095],[4437551,570.073,635.854],[4437552,640.514,706.191],[4437553,710.698,775.985],[4437554,780.446,846.589],[4437555,851.195,916.667],[4437556,921.774,993.258],[4437557,997.678,1064.262],[4437558,1068.822,1134.298]],"window_offset_ms":0.067},{"bytes":4194304,"start":60.34563,"end":66.345727,"completed":50063,"mean_us":114.794,"observed_GBps":34.996,"samples":[[4504580,0.0,114.662],[4504581,119.301,241.012],[4504582,245.831,360.44],[4504583,365.346,480.072],[4504584,484.88,598.421],[4504585,603.231,718.253],[4504586,723.287,836.82],[4504587,842.133,956.709],[4504588,961.422,1075.499],[4504589,1080.505,1194.446],[4504590,1199.425,1313.746],[4504591,1326.393,1441.258],[4504592,1446.213,1561.902],[4504593,1566.402,1680.832],[4504594,1685.615,1800.295],[4504595,1805.35,1919.6]],"window_offset_ms":0.007},{"bytes":8388608,"start":66.352723,"end":72.352802,"completed":27684,"mean_us":211.793,"observed_GBps":38.705,"samples":[[4543477,0.0,211.97],[4543478,216.688,425.678],[4543479,430.451,642.149],[4543480,647.373,856.913],[4543481,861.726,1072.774],[4543482,1077.373,1288.404],[4543483,1293.332,1503.752],[4543484,1508.718,1721.396],[4543485,1726.181,1934.514],[4543486,1939.334,2150.989],[4543487,2155.966,2366.687],[4543488,2371.598,2582.524],[4543489,2587.342,2797.175],[4543490,2801.867,3011.936],[4543491,3017.036,3229.296],[4543492,3234.339,3445.393]],"window_offset_ms":0.072},{"bytes":16777216,"start":72.358319,"end":78.358686,"completed":14639,"mean_us":404.855,"observed_GBps":40.931,"samples":[[4564637,0.0,403.954],[4564638,408.755,811.778],[4564639,816.506,1219.459],[4564640,1224.76,1628.347],[4564641,1633.194,2035.153],[4564642,2040.1,2443.022],[4564643,2448.14,2850.364],[4564644,2855.289,3260.675],[4564645,3265.925,3668.829],[4564646,3673.859,4077.6],[4564647,4082.23,4488.189],[4564648,4492.907,4892.57],[4564649,4897.429,5302.097],[4564650,5307.512,5708.522],[4564651,5713.526,6117.641],[4564652,6122.495,6526.424]],"window_offset_ms":0.293},{"bytes":33554432,"start":78.363162,"end":84.363926,"completed":7504,"mean_us":794.455,"observed_GBps":41.96,"samples":[[4575710,0.0,790.556],[4575711,795.568,1588.245],[4575712,1593.323,2382.3],[4575713,2387.127,3173.625],[4575714,3178.289,3967.61],[4575715,3972.384,4762.964],[4575716,4768.02,5557.3],[4575717,5562.346,6354.164],[4575718,6359.176,7149.682],[4575719,7154.729,7946.251],[4575720,7951.114,8738.726],[4575721,8743.811,9534.593],[4575722,9540.065,10332.327],[4575723,10337.632,11129.766],[4575724,11134.882,11921.647],[4575725,11926.308,12718.218]],"window_offset_ms":0.393},{"bytes":67108864,"start":84.366713,"end":90.36779,"completed":3731,"mean_us":1603.316,"observed_GBps":41.723,"samples":[[4581327,0.0,1596.366],[4581328,1601.547,3194.369],[4581329,3206.971,4803.836],[4581330,4808.237,6401.425],[4581331,6406.172,8003.459],[4581332,8008.446,9604.042],[4581333,9609.134,11202.933],[4581334,11207.468,12798.806],[4581335,12803.461,14397.785],[4581336,14402.771,16001.727],[4581337,16007.568,17609.744],[4581338,17614.618,19206.828],[4581339,19211.504,20808.885],[4581340,20814.742,22405.908],[4581341,22410.839,24006.956],[4581342,24012.026,25604.773]],"window_offset_ms":1.005}]},{"run":"read-2","file":"tent-events-slurm-h100-206-025-2232757-0Y5JRP.jsonl","op":"read","loss":4.541,"phases":[{"bytes":4096,"start":0.0,"end":5.999416,"completed":606475,"mean_us":7.809,"observed_GBps":0.414,"samples":[[301238,0.0,7.107],[301239,8.458,15.579],[301240,16.954,24.014],[301241,25.292,32.415],[301242,33.789,40.972],[301243,42.365,49.341],[301244,50.686,57.849],[301245,59.188,66.195],[301246,67.515,74.744],[301247,76.093,83.076],[301248,84.348,91.398],[301249,92.814,99.909],[301250,101.277,108.257],[301251,109.544,116.517],[301252,117.792,125.104],[301253,126.37,133.61]],"window_offset_ms":0.008},{"bytes":8192,"start":6.056158,"end":12.056153,"completed":614497,"mean_us":7.268,"observed_GBps":0.839,"samples":[[993328,0.0,7.438],[993329,8.837,16.611],[993330,18.329,25.872],[993331,27.275,34.648],[993332,36.063,43.351],[993333,44.725,52.025],[993334,53.469,60.859],[993335,62.307,69.638],[993336,71.29,78.65],[993337,79.994,87.798],[993338,89.183,96.71],[993339,98.131,105.393],[993340,107.036,114.577],[993341,115.907,123.272],[993342,124.579,131.96],[993343,133.426,140.789]],"window_offset_ms":12.895},{"bytes":16384,"start":12.109736,"end":18.109731,"completed":601543,"mean_us":7.737,"observed_GBps":1.643,"samples":[[1685228,0.0,8.659],[1685229,10.118,17.803],[1685230,19.36,27.052],[1685231,28.598,36.146],[1685232,37.462,45.75],[1685233,47.141,54.921],[1685234,56.396,64.692],[1685235,67.183,74.847],[1685236,76.542,84.304],[1685237,85.726,93.655],[1685238,95.044,102.627],[1685239,104.16,111.847],[1685240,113.756,121.987],[1685241,124.415,132.87],[1685242,134.525,142.377],[1685243,143.938,151.532]],"window_offset_ms":189.514},{"bytes":32768,"start":18.151578,"end":24.151579,"completed":583356,"mean_us":8.123,"observed_GBps":3.186,"samples":[[2309728,0.0,8.551],[2309729,10.589,19.339],[2309730,21.955,30.448],[2309731,32.825,41.317],[2309732,43.702,52.506],[2309733,54.79,63.315],[2309734,65.65,74.069],[2309735,76.571,85.191],[2309736,87.858,96.718],[2309737,99.192,107.78],[2309738,110.099,118.744],[2309739,121.118,129.593],[2309740,132.169,140.802],[2309741,143.19,151.713],[2309742,154.117,162.724],[2309743,165.507,173.93]],"window_offset_ms":69.941},{"bytes":65536,"start":24.196628,"end":30.196622,"completed":563856,"mean_us":9.107,"observed_GBps":6.159,"samples":[[2897048,0.0,8.825],[2897049,10.21,19.147],[2897050,20.819,29.379],[2897051,30.816,39.921],[2897052,41.358,50.528],[2897053,52.003,60.788],[2897054,62.15,71.072],[2897055,72.641,81.677],[2897056,83.039,92.103],[2897057,93.523,102.366],[2897058,103.757,112.456],[2897059,113.944,122.788],[2897060,124.263,132.99],[2897061,134.364,143.079],[2897062,144.391,153.11],[2897063,154.493,163.275]],"window_offset_ms":0.009},{"bytes":131072,"start":30.238631,"end":36.238648,"completed":473263,"mean_us":10.989,"observed_GBps":10.339,"samples":[[3417770,0.0,10.849],[3417771,12.455,23.47],[3417772,25.04,36.289],[3417773,37.838,49.172],[3417774,50.829,61.546],[3417775,63.215,74.289],[3417776,76.334,87.385],[3417777,88.964,99.712],[3417778,101.238,112.176],[3417779,113.803,124.368],[3417780,125.922,136.739],[3417781,138.267,149.199],[3417782,150.729,161.887],[3417783,164.263,175.253],[3417784,177.287,188.581],[3417785,190.271,201.428]],"window_offset_ms":0.01},{"bytes":262144,"start":36.275694,"end":42.275704,"completed":357203,"mean_us":14.265,"observed_GBps":15.606,"samples":[[3830363,0.0,14.304],[3830364,16.786,31.026],[3830365,33.541,47.478],[3830366,50.053,63.922],[3830367,66.369,80.742],[3830368,83.244,97.676],[3830369,100.133,114.642],[3830370,117.064,130.974],[3830371,133.716,148.352],[3830372,150.9,165.276],[3830373,167.748,181.91],[3830374,184.452,198.776],[3830375,201.265,215.758],[3830376,218.169,235.514],[3830377,238.088,252.048],[3830378,254.646,268.611]],"window_offset_ms":0.007},{"bytes":524288,"start":42.301005,"end":48.301005,"completed":239160,"mean_us":21.05,"observed_GBps":20.898,"samples":[[4129404,0.0,20.903],[4129405,24.17,44.903],[4129406,54.604,75.446],[4129407,79.217,100.287],[4129408,104.201,125.311],[4129409,128.557,149.0],[4129410,159.095,180.474],[4129411,184.238,204.916],[4129412,208.753,229.149],[4129413,238.727,261.022],[4129414,264.822,285.375],[4129415,290.781,311.603],[4129416,314.476,335.006],[4129417,339.015,359.882],[4129418,373.126,393.626],[4129419,397.444,418.894]],"window_offset_ms":0.023},{"bytes":1048576,"start":48.323215,"end":54.323257,"completed":161971,"mean_us":33.123,"observed_GBps":28.306,"samples":[[4329146,0.0,33.039],[4329147,36.875,69.617],[4329148,73.456,106.086],[4329149,110.031,161.048],[4329150,164.874,198.717],[4329151,205.15,238.575],[4329152,242.587,276.156],[4329153,279.943,312.57],[4329154,316.445,349.503],[4329155,353.369,386.225],[4329156,390.082,422.739],[4329157,426.518,459.671],[4329158,463.717,496.323],[4329159,500.203,532.955],[4329160,536.808,570.149],[4329161,574.004,607.019]],"window_offset_ms":0.026},{"bytes":2097152,"start":54.334673,"end":60.334754,"completed":84347,"mean_us":66.496,"observed_GBps":29.481,"samples":[[4452298,0.0,65.771],[4452299,70.33,136.863],[4452300,141.436,207.877],[4452301,212.314,278.795],[4452302,283.419,349.313],[4452303,354.052,420.993],[4452304,425.712,492.582],[4452305,497.273,562.679],[4452306,567.199,633.342],[4452307,637.859,703.117],[4452308,707.524,772.973],[4452309,777.466,843.942],[4452310,848.31,913.529],[4452311,918.013,984.73],[4452312,989.137,1054.135],[4452313,1058.595,1124.588]],"window_offset_ms":0.025},{"bytes":4194304,"start":60.34578,"end":66.345885,"completed":50040,"mean_us":114.606,"observed_GBps":34.98,"samples":[[4519485,0.0,111.841],[4519486,116.3,228.65],[4519487,233.767,348.347],[4519488,353.239,467.674],[4519489,472.739,587.755],[4519490,592.49,706.267],[4519491,711.388,825.274],[4519492,830.109,943.172],[4519493,947.889,1061.676],[4519494,1066.674,1181.723],[4519495,1186.764,1301.304],[4519496,1306.635,1421.119],[4519497,1430.55,1545.251],[4519498,1550.417,1665.113],[4519499,1670.099,1783.643],[4519500,1788.423,1902.601]],"window_offset_ms":0.002},{"bytes":8388608,"start":66.354238,"end":72.354457,"completed":27676,"mean_us":211.729,"observed_GBps":38.692,"samples":[[4558339,0.0,210.094],[4558340,215.222,424.929],[4558341,429.753,641.933],[4558342,646.563,858.637],[4558343,863.188,1072.793],[4558344,1077.767,1285.944],[4558345,1290.945,1502.204],[4558346,1508.278,1717.674],[4558347,1722.951,1934.347],[4558348,1939.687,2150.614],[4558349,2155.623,2455.236],[4558350,2460.339,2668.458],[4558351,2673.777,2884.58],[4558352,2889.749,3100.143],[4558353,3105.193,3315.671],[4558354,3321.053,3531.287]],"window_offset_ms":0.214},{"bytes":16777216,"start":72.359735,"end":78.359936,"completed":14634,"mean_us":404.874,"observed_GBps":40.918,"samples":[[4579501,0.0,402.078],[4579502,407.275,810.641],[4579503,815.775,1216.504],[4579504,1221.363,1624.118],[4579505,1628.853,2033.416],[4579506,2038.333,2445.836],[4579507,2450.884,2853.794],[4579508,2858.801,3265.619],[4579509,3270.67,3674.079],[4579510,3679.073,4081.328],[4579511,4086.446,4490.641],[4579512,4495.795,4899.523],[4579513,4911.116,5315.373],[4579514,5320.46,5723.468],[4579515,5728.638,6132.327],[4579516,6137.448,6538.564]],"window_offset_ms":0.391},{"bytes":33554432,"start":78.363893,"end":84.365007,"completed":7370,"mean_us":808.861,"observed_GBps":41.208,"samples":[[4590504,0.0,805.395],[4590505,810.583,1611.001],[4590506,1615.95,2419.088],[4590507,2423.926,3226.484],[4590508,3231.593,4036.807],[4590509,4041.597,4846.743],[4590510,4852.029,5654.693],[4590511,5659.57,6466.863],[4590512,6472.179,7275.193],[4590513,7280.39,8086.582],[4590514,8091.584,8895.913],[4590515,8901.321,9707.36],[4590516,9712.451,10521.165],[4590517,10526.074,11328.516],[4590518,11333.228,12141.506],[4590519,12146.622,12952.355]],"window_offset_ms":0.563},{"bytes":67108864,"start":84.37631,"end":90.378727,"completed":3735,"mean_us":1601.691,"observed_GBps":41.758,"samples":[[4596056,0.0,1592.562],[4596057,1597.756,3195.552],[4596058,3200.451,4790.427],[4596059,4795.56,6389.669],[4596060,6394.874,7987.835],[4596061,7992.932,9586.446],[4596062,9590.977,11187.598],[4596063,11192.394,12787.13],[4596064,12792.226,14386.649],[4596065,14391.663,15984.173],[4596066,15989.145,17575.822],[4596067,17580.873,19177.916],[4596068,19183.322,20779.468],[4596069,20784.336,22378.409],[4596070,22383.196,23979.413],[4596071,23984.624,25577.637]],"window_offset_ms":1.241}]},{"run":"read-3","file":"tent-events-slurm-h100-206-025-2234022-WL8pRq.jsonl","op":"read","loss":3.036,"phases":[{"bytes":4096,"start":0.0,"end":5.990342,"completed":608670,"mean_us":7.778,"observed_GBps":0.416,"samples":[[304669,0.0,7.21],[304670,8.518,15.821],[304671,17.187,24.381],[304672,25.694,32.842],[304673,34.151,41.257],[304674,42.835,49.717],[304675,51.112,58.159],[304676,59.576,66.903],[304677,68.211,75.28],[304678,76.647,83.849],[304679,85.229,92.321],[304680,93.71,100.806],[304681,102.238,109.617],[304682,110.997,118.16],[304683,119.489,126.596],[304684,128.118,135.398]],"window_offset_ms":0.001},{"bytes":8192,"start":6.058768,"end":12.058764,"completed":623509,"mean_us":7.367,"observed_GBps":0.851,"samples":[[988459,0.0,7.452],[988460,8.853,16.372],[988461,17.7,25.077],[988462,26.557,33.931],[988463,35.239,42.592],[988464,43.972,51.038],[988465,52.561,59.776],[988466,61.262,68.661],[988467,70.194,77.417],[988468,78.745,86.312],[988469,87.627,94.765],[988470,96.17,103.293],[988471,104.589,111.668],[988472,113.162,120.239],[988473,121.662,128.93],[988474,130.402,137.587]],"window_offset_ms":0.006},{"bytes":16384,"start":12.10288,"end":18.102878,"completed":609072,"mean_us":7.81,"observed_GBps":1.663,"samples":[[1646795,0.0,7.862],[1646796,9.557,17.924],[1646797,19.332,27.522],[1646798,29.086,37.032],[1646799,38.562,46.386],[1646800,47.89,55.776],[1646801,57.291,65.224],[1646802,66.645,74.455],[1646803,75.92,83.695],[1646804,85.242,93.591],[1646805,95.015,102.769],[1646806,104.24,111.938],[1646807,113.465,121.637],[1646808,123.005,130.933],[1646809,132.395,140.152],[1646810,141.533,149.424]],"window_offset_ms":5.965},{"bytes":32768,"start":18.144437,"end":24.144433,"completed":603757,"mean_us":8.242,"observed_GBps":3.297,"samples":[[2273146,0.0,7.886],[2273147,9.268,17.267],[2273148,18.66,26.552],[2273149,27.936,36.003],[2273150,37.458,45.272],[2273151,46.62,54.788],[2273152,56.222,64.657],[2273153,65.976,73.93],[2273154,75.302,83.372],[2273155,84.756,92.525],[2273156,93.97,102.314],[2273157,103.82,112.049],[2273158,113.657,121.459],[2273159,122.892,130.962],[2273160,132.493,140.723],[2273161,142.27,150.734]],"window_offset_ms":0.004},{"bytes":65536,"start":24.195483,"end":30.195492,"completed":566854,"mean_us":9.028,"observed_GBps":6.192,"samples":[[2866562,0.0,8.793],[2866563,10.234,19.256],[2866564,22.183,30.949],[2866565,32.428,41.103],[2866566,42.594,51.404],[2866567,52.808,61.58],[2866568,63.037,71.802],[2866569,73.206,82.14],[2866570,84.93,94.913],[2866571,96.429,105.351],[2866572,106.893,115.93],[2866573,117.373,126.202],[2866574,127.969,137.012],[2866575,138.547,147.334],[2866576,149.64,159.749],[2866577,161.25,170.571]],"window_offset_ms":0.001},{"bytes":131072,"start":30.239495,"end":36.239495,"completed":458861,"mean_us":11.256,"observed_GBps":10.024,"samples":[[3377433,0.0,10.985],[3377434,12.557,23.589],[3377435,25.179,36.778],[3377436,38.265,49.597],[3377437,51.652,63.129],[3377438,65.318,76.582],[3377439,78.304,89.792],[3377440,91.359,102.189],[3377441,103.847,115.019],[3377442,117.069,128.822],[3377443,130.858,142.075],[3377444,144.107,157.83],[3377445,159.564,170.965],[3377446,172.487,183.648],[3377447,185.315,196.9],[3377448,198.554,209.263]],"window_offset_ms":0.003},{"bytes":262144,"start":36.275087,"end":42.275106,"completed":348583,"mean_us":14.686,"observed_GBps":15.23,"samples":[[3791861,0.0,15.0],[3791862,17.624,32.576],[3791863,35.761,50.626],[3791864,53.839,68.37],[3791865,71.68,86.542],[3791866,89.158,103.847],[3791867,107.628,122.309],[3791868,125.716,140.373],[3791869,145.43,160.343],[3791870,163.543,178.065],[3791871,181.425,196.91],[3791872,200.087,214.867],[3791873,217.404,231.94],[3791874,235.215,249.693],[3791875,253.293,268.151],[3791876,271.535,286.099]],"window_offset_ms":0.003},{"bytes":524288,"start":42.298531,"end":48.298555,"completed":239004,"mean_us":21.008,"observed_GBps":20.884,"samples":[[4076707,0.0,21.115],[4076708,25.139,45.203],[4076709,47.828,69.043],[4076710,72.512,94.587],[4076711,97.356,118.974],[4076712,125.888,147.071],[4076713,150.633,171.674],[4076714,174.413,195.156],[4076715,198.754,219.744],[4076716,223.933,244.317],[4076717,247.001,267.643],[4076718,271.22,292.118],[4076719,294.84,315.097],[4076720,325.312,347.43],[4076721,350.769,371.628],[4076722,375.162,395.738]],"window_offset_ms":0.021},{"bytes":1048576,"start":48.320167,"end":54.320221,"completed":161657,"mean_us":33.025,"observed_GBps":28.251,"samples":[[4277118,0.0,31.566],[4277119,34.356,66.399],[4277120,69.106,100.794],[4277121,103.536,136.018],[4277122,138.785,170.719],[4277123,173.49,206.212],[4277124,209.006,240.513],[4277125,243.341,274.66],[4277126,277.39,308.601],[4277127,311.529,343.022],[4277128,345.877,377.667],[4277129,380.565,412.611],[4277130,415.382,446.669],[4277131,449.453,481.472],[4277132,484.477,516.671],[4277133,519.726,552.242]],"window_offset_ms":0.0},{"bytes":2097152,"start":54.331938,"end":60.332024,"completed":84302,"mean_us":66.338,"observed_GBps":29.465,"samples":[[4400095,0.0,66.362],[4400096,70.876,137.099],[4400097,141.903,208.864],[4400098,213.635,279.498],[4400099,283.973,350.568],[4400100,355.276,421.934],[4400101,426.61,492.67],[4400102,497.12,563.67],[4400103,568.386,634.554],[4400104,639.331,704.525],[4400105,709.332,776.187],[4400106,780.762,846.529],[4400107,851.226,917.46],[4400108,921.96,988.661],[4400109,993.324,1059.467],[4400110,1064.089,1130.423]],"window_offset_ms":0.07},{"bytes":4194304,"start":60.341531,"end":66.341638,"completed":50091,"mean_us":114.756,"observed_GBps":35.016,"samples":[[4467326,0.0,114.611],[4467327,123.079,236.136],[4467328,240.979,356.743],[4467329,361.686,475.098],[4467330,480.099,593.467],[4467331,598.214,710.76],[4467332,715.559,829.746],[4467333,834.35,949.344],[4467334,954.112,1068.651],[4467335,1073.182,1187.357],[4467336,1192.261,1307.612],[4467337,1317.283,1431.939],[4467338,1436.967,1551.002],[4467339,1555.627,1669.59],[4467340,1674.624,1788.896],[4467341,1793.958,1908.177]],"window_offset_ms":0.023},{"bytes":8388608,"start":66.348071,"end":72.348106,"completed":27700,"mean_us":211.749,"observed_GBps":38.727,"samples":[[4506216,0.0,209.234],[4506217,213.917,426.569],[4506218,431.129,639.478],[4506219,644.229,855.384],[4506220,860.166,1071.038],[4506221,1075.474,1286.711],[4506222,1291.439,1503.041],[4506223,1507.702,1718.901],[4506224,1723.839,1935.777],[4506225,1940.37,2150.119],[4506226,2155.026,2364.494],[4506227,2369.256,2577.861],[4506228,2582.632,2792.702],[4506229,2797.459,3006.864],[4506230,3011.836,3220.661],[4506231,3225.366,3434.636]],"window_offset_ms":0.185},{"bytes":16777216,"start":72.352591,"end":78.3532,"completed":14644,"mean_us":404.866,"observed_GBps":40.943,"samples":[[4527386,0.0,402.134],[4527387,406.767,808.984],[4527388,813.897,1215.604],[4527389,1220.379,1623.32],[4527390,1628.196,2029.486],[4527391,2034.1,2438.037],[4527392,2442.935,2846.828],[4527393,2851.307,3251.934],[4527394,3256.833,3661.825],[4527395,3666.863,4068.864],[4527396,4073.613,4475.918],[4527397,4480.427,4887.548],[4527398,4892.465,5293.686],[4527399,5298.507,5703.296],[4527400,5708.037,6112.827],[4527401,6117.672,6522.536]],"window_offset_ms":0.218},{"bytes":33554432,"start":78.356499,"end":84.357007,"completed":7374,"mean_us":808.591,"observed_GBps":41.235,"samples":[[4538395,0.0,804.209],[4538396,808.997,1612.346],[4538397,1617.577,2419.57],[4538398,2424.486,3236.222],[4538399,3241.221,4044.367],[4538400,4050.417,4857.235],[4538401,4862.186,5665.932],[4538402,5671.152,6477.106],[4538403,6482.038,7290.117],[4538404,7295.344,8100.717],[4538405,8105.804,8908.9],[4538406,8914.082,9716.252],[4538407,9721.07,10527.494],[4538408,10532.442,11337.783],[4538409,11342.649,12147.146],[4538410,12152.353,12958.216]],"window_offset_ms":0.108},{"bytes":67108864,"start":84.359931,"end":90.361228,"completed":3735,"mean_us":1601.542,"observed_GBps":41.766,"samples":[[4543950,0.0,1593.136],[4543951,1598.208,3192.996],[4543952,3203.496,4796.092],[4543953,4800.777,6395.174],[4543954,6400.087,7992.942],[4543955,7997.677,9593.357],[4543956,9598.521,11187.566],[4543957,11192.443,12786.648],[4543958,12791.305,14388.801],[4543959,14401.24,15993.626],[4543960,15998.593,17593.64],[4543961,17598.122,19192.646],[4543962,19197.507,20793.318],[4543963,20798.041,22395.271],[4543964,22400.184,23994.47],[4543965,23999.32,25595.399]],"window_offset_ms":1.012}]},{"run":"write-1","file":"tent-events-slurm-h100-206-025-2232113-2zgoGf.jsonl","op":"write","loss":5.477,"phases":[{"bytes":4096,"start":0.0,"end":5.991233,"completed":610426,"mean_us":7.83,"observed_GBps":0.417,"samples":[[294482,0.0,7.28],[294483,8.731,15.68],[294484,18.141,26.003],[294485,27.62,34.771],[294486,36.26,43.463],[294487,44.921,52.222],[294488,53.617,60.606],[294489,62.064,69.273],[294490,70.647,77.728],[294491,79.111,86.132],[294492,89.48,96.796],[294493,98.279,105.156],[294494,106.535,113.64],[294495,115.051,122.006],[294496,123.344,130.268],[294497,131.66,138.495]],"window_offset_ms":0.001},{"bytes":8192,"start":6.059384,"end":12.059382,"completed":598751,"mean_us":7.296,"observed_GBps":0.817,"samples":[[989767,0.0,7.167],[989768,8.578,16.434],[989769,18.2,25.429],[989770,26.921,34.318],[989771,35.74,43.024],[989772,44.426,51.672],[989773,53.027,60.296],[989774,61.715,68.873],[989775,70.565,77.791],[989776,79.165,87.341],[989777,88.856,96.228],[989778,97.595,104.752],[989779,106.756,114.149],[989780,115.629,123.002],[989781,124.308,131.465],[989782,133.018,140.23]],"window_offset_ms":60.643},{"bytes":16384,"start":12.106499,"end":18.106507,"completed":588940,"mean_us":7.538,"observed_GBps":1.608,"samples":[[1659502,0.0,7.438],[1659503,8.82,16.311],[1659504,17.712,25.151],[1659505,26.553,33.824],[1659506,35.211,42.677],[1659507,44.034,51.519],[1659508,52.836,60.198],[1659509,61.551,68.857],[1659510,70.232,77.737],[1659511,79.12,86.534],[1659512,87.916,95.175],[1659513,96.637,104.155],[1659514,105.504,112.787],[1659515,114.2,121.657],[1659516,123.148,130.64],[1659517,132.016,139.384]],"window_offset_ms":0.001},{"bytes":32768,"start":18.143576,"end":24.143584,"completed":587755,"mean_us":8.099,"observed_GBps":3.21,"samples":[[2307724,0.0,7.891],[2307725,9.252,17.237],[2307726,18.557,26.789],[2307727,28.171,36.057],[2307728,37.412,45.473],[2307729,46.846,54.73],[2307730,56.036,64.222],[2307731,65.602,73.613],[2307732,75.064,82.993],[2307733,84.405,92.041],[2307734,93.48,101.519],[2307735,102.884,110.516],[2307736,111.838,119.825],[2307737,121.18,128.984],[2307738,130.354,138.207],[2307739,139.573,147.699]],"window_offset_ms":0.001},{"bytes":65536,"start":24.184454,"end":30.184455,"completed":557029,"mean_us":8.961,"observed_GBps":6.084,"samples":[[2909042,0.0,8.697],[2909043,10.182,19.025],[2909044,20.406,28.992],[2909045,30.338,38.932],[2909046,40.327,49.253],[2909047,50.599,59.316],[2909048,60.661,69.464],[2909049,70.882,79.619],[2909050,80.956,89.592],[2909051,90.983,99.693],[2909052,101.097,109.686],[2909053,111.029,119.641],[2909054,121.021,129.638],[2909055,131.018,139.816],[2909056,141.182,149.694],[2909057,151.06,159.794]],"window_offset_ms":0.007},{"bytes":131072,"start":30.218831,"end":36.218833,"completed":466530,"mean_us":11.015,"observed_GBps":10.191,"samples":[[3427726,0.0,10.899],[3427727,12.756,23.966],[3427728,25.563,37.216],[3427729,38.821,49.978],[3427730,52.457,63.504],[3427731,65.217,76.675],[3427732,78.492,89.69],[3427733,91.343,102.136],[3427734,103.676,115.062],[3427735,116.581,127.352],[3427736,128.946,139.656],[3427737,144.786,156.68],[3427738,158.942,170.321],[3427739,172.252,183.619],[3427740,185.321,196.101],[3427741,197.701,208.745]],"window_offset_ms":0.007},{"bytes":262144,"start":36.250221,"end":42.250235,"completed":357914,"mean_us":14.272,"observed_GBps":15.637,"samples":[[3842014,0.0,15.028],[3842015,17.526,31.577],[3842016,34.134,48.918],[3842017,51.403,65.572],[3842018,68.127,83.023],[3842019,85.516,99.671],[3842020,102.139,116.648],[3842021,119.12,133.651],[3842022,136.169,151.078],[3842023,153.664,167.72],[3842024,170.201,184.44],[3842025,186.982,201.261],[3842026,203.765,218.03],[3842027,220.532,234.689],[3842028,237.31,251.474],[3842029,253.992,267.983]],"window_offset_ms":0.012},{"bytes":524288,"start":42.295053,"end":48.295071,"completed":245636,"mean_us":21.178,"observed_GBps":21.464,"samples":[[4142077,0.0,20.756],[4142078,23.297,45.636],[4142079,48.272,69.351],[4142080,74.777,95.412],[4142081,98.732,119.624],[4142082,123.9,145.343],[4142083,148.038,169.266],[4142084,171.897,193.247],[4142085,196.487,217.885],[4142086,220.692,242.811],[4142087,246.063,266.628],[4142088,269.333,290.648],[4142089,293.31,314.019],[4142090,316.747,337.516],[4142091,340.215,363.88],[4142092,367.125,387.617]],"window_offset_ms":0.013},{"bytes":1048576,"start":48.326174,"end":54.326211,"completed":161364,"mean_us":33.178,"observed_GBps":28.2,"samples":[[4346381,0.0,32.679],[4346382,36.692,69.457],[4346383,73.331,106.393],[4346384,110.363,142.726],[4346385,146.54,179.805],[4346386,183.567,216.422],[4346387,220.408,252.898],[4346388,256.859,290.701],[4346389,295.031,328.318],[4346390,332.206,365.155],[4346391,369.052,425.269],[4346392,429.094,462.15],[4346393,466.016,498.58],[4346394,502.409,534.844],[4346395,538.818,571.642],[4346396,575.646,608.168]],"window_offset_ms":0.022},{"bytes":2097152,"start":54.338744,"end":60.338752,"completed":85707,"mean_us":65.228,"observed_GBps":29.957,"samples":[[4469911,0.0,65.666],[4469912,70.652,135.486],[4469913,140.174,206.012],[4469914,214.139,278.647],[4469915,291.206,356.241],[4469916,361.185,425.179],[4469917,429.718,494.285],[4469918,499.085,564.132],[4469919,568.792,633.704],[4469920,638.207,703.195],[4469921,708.222,773.004],[4469922,777.599,842.846],[4469923,847.437,911.708],[4469924,916.643,981.412],[4469925,986.188,1051.236],[4469926,1055.944,1120.448]],"window_offset_ms":0.024},{"bytes":4194304,"start":60.347456,"end":66.347561,"completed":49640,"mean_us":115.699,"observed_GBps":34.7,"samples":[[4537584,0.0,114.941],[4537585,119.563,234.51],[4537586,239.055,353.924],[4537587,362.027,478.017],[4537588,482.864,598.371],[4537589,602.928,716.715],[4537590,721.654,836.586],[4537591,841.049,1020.282],[4537592,1025.129,1129.299],[4537593,1134.204,1249.423],[4537594,1254.069,1369.505],[4537595,1374.149,1488.852],[4537596,1493.804,1608.058],[4537597,1612.641,1726.029],[4537598,1730.623,1845.259],[4537599,1850.041,1964.643]],"window_offset_ms":0.113},{"bytes":8388608,"start":66.354505,"end":72.354817,"completed":26958,"mean_us":216.988,"observed_GBps":37.688,"samples":[[4575881,0.0,214.495],[4575882,219.113,435.714],[4575883,440.51,657.115],[4575884,666.24,881.229],[4575885,885.879,1102.749],[4575886,1107.399,1324.221],[4575887,1329.176,1545.778],[4575888,1550.598,1767.245],[4575889,1772.098,1988.248],[4575890,1993.369,2209.036],[4575891,2213.777,2549.972],[4575892,2554.804,2749.683],[4575893,2754.479,2970.721],[4575894,2979.297,3194.177],[4575895,3198.807,3415.312],[4575896,3420.366,3636.789]],"window_offset_ms":0.008},{"bytes":16777216,"start":72.359079,"end":78.35934,"completed":14074,"mean_us":421.254,"observed_GBps":39.352,"samples":[[4596367,0.0,415.898],[4596368,420.458,836.212],[4596369,840.779,1257.1],[4596370,1261.748,1679.758],[4596371,1684.38,2102.61],[4596372,2107.151,2524.764],[4596373,2536.824,2955.739],[4596374,2960.696,3376.517],[4596375,3381.757,3801.351],[4596376,3805.619,4220.955],[4596377,4225.938,4641.344],[4596378,4646.101,5063.106],[4596379,5068.053,5484.173],[4596380,5488.806,5904.041],[4596381,5908.804,6326.687],[4596382,6331.396,6748.196]],"window_offset_ms":0.315},{"bytes":33554432,"start":78.363105,"end":84.364641,"completed":7169,"mean_us":831.928,"observed_GBps":40.082,"samples":[[4607021,0.0,827.285],[4607022,831.925,1659.918],[4607023,1664.901,2490.614],[4607024,2495.229,3321.13],[4607025,3326.199,4151.2],[4607026,4156.239,4983.024],[4607027,4987.698,5818.12],[4607028,5823.115,6654.353],[4607029,6659.255,7488.921],[4607030,7493.736,8322.397],[4607031,8327.321,9155.13],[4607032,9160.255,9989.248],[4607033,9994.338,10823.271],[4607034,10828.165,11655.243],[4607035,11660.06,12486.974],[4607036,12491.531,13320.689]],"window_offset_ms":0.239},{"bytes":67108864,"start":84.367371,"end":90.369534,"completed":3591,"mean_us":1666.274,"observed_GBps":40.15,"samples":[[4612400,0.0,1659.026],[4612401,1663.698,3316.047],[4612402,3320.863,4982.105],[4612403,4986.804,6641.441],[4612404,6646.135,8303.628],[4612405,8308.205,9970.321],[4612406,9975.464,11634.177],[4612407,11638.704,13298.616],[4612408,13303.315,14964.324],[4612409,14969.229,16626.402],[4612410,16631.142,18288.531],[4612411,18293.157,19952.153],[4612412,19957.169,21610.257],[4612413,21614.792,23278.97],[4612414,23283.585,24942.875],[4612415,24947.625,26607.089]],"window_offset_ms":1.138}]},{"run":"write-2","file":"tent-events-slurm-h100-206-025-2233390-uky9P4.jsonl","op":"write","loss":6.386,"phases":[{"bytes":4096,"start":0.0,"end":5.999375,"completed":591730,"mean_us":7.762,"observed_GBps":0.404,"samples":[[303203,0.0,7.241],[303204,8.684,15.808],[303205,17.164,24.061],[303206,25.362,32.437],[303207,33.906,41.169],[303208,42.605,49.88],[303209,51.287,59.396],[303210,60.82,68.095],[303211,69.441,76.342],[303212,77.816,87.094],[303213,88.654,97.221],[303214,98.731,106.058],[303215,107.438,114.492],[303216,115.848,123.06],[303217,124.466,131.723],[303218,133.036,140.131]],"window_offset_ms":0.008},{"bytes":8192,"start":6.058145,"end":12.038626,"completed":572177,"mean_us":7.227,"observed_GBps":0.784,"samples":[[994684,0.0,7.554],[994685,8.996,16.734],[994686,18.363,25.685],[994687,27.25,34.564],[994688,35.903,43.309],[994689,44.654,51.97],[994690,53.382,60.638],[994691,62.073,69.497],[994692,71.09,78.488],[994693,80.009,87.63],[994694,89.107,96.352],[994695,97.779,104.993],[994696,106.659,114.104],[994697,115.441,122.962],[994698,124.387,131.767],[994699,133.411,140.661]],"window_offset_ms":23.976},{"bytes":16384,"start":12.11184,"end":18.11184,"completed":619862,"mean_us":7.468,"observed_GBps":1.693,"samples":[[1676464,0.0,7.31],[1676465,8.615,16.226],[1676466,17.581,25.086],[1676467,26.43,33.722],[1676468,35.175,42.502],[1676469,43.926,51.279],[1676470,52.728,59.935],[1676471,61.273,68.68],[1676472,70.035,77.346],[1676473,78.624,85.871],[1676474,87.273,94.578],[1676475,95.919,103.222],[1676476,104.543,112.01],[1676477,113.335,120.668],[1676478,122.005,129.568],[1676479,130.927,138.509]],"window_offset_ms":0.003},{"bytes":32768,"start":18.156238,"end":24.156239,"completed":579724,"mean_us":7.971,"observed_GBps":3.166,"samples":[[2328246,0.0,8.022],[2328247,9.578,17.494],[2328248,18.869,26.62],[2328249,28.053,35.862],[2328250,37.259,45.23],[2328251,46.536,54.433],[2328252,55.841,63.847],[2328253,66.959,74.951],[2328254,76.387,84.338],[2328255,85.719,93.71],[2328256,95.124,102.807],[2328257,104.102,111.992],[2328258,113.322,121.051],[2328259,122.8,131.41],[2328260,133.355,141.337],[2328261,142.681,150.441]],"window_offset_ms":0.004},{"bytes":65536,"start":24.226574,"end":30.226583,"completed":569369,"mean_us":8.829,"observed_GBps":6.219,"samples":[[2937735,0.0,8.716],[2937736,10.174,18.656],[2937737,20.021,28.742],[2937738,30.138,38.739],[2937739,40.069,48.726],[2937740,50.078,58.753],[2937741,60.117,68.688],[2937742,70.072,78.956],[2937743,80.373,88.98],[2937744,90.435,99.37],[2937745,100.791,109.352],[2937746,110.687,119.297],[2937747,120.686,129.195],[2937748,130.537,139.278],[2937749,140.582,149.393],[2937750,150.823,159.442]],"window_offset_ms":0.008},{"bytes":131072,"start":30.319229,"end":36.319229,"completed":466876,"mean_us":11.042,"observed_GBps":10.199,"samples":[[3463763,0.0,11.071],[3463764,13.23,24.226],[3463765,25.716,36.886],[3463766,38.469,49.768],[3463767,51.454,62.918],[3463768,64.583,75.526],[3463769,77.192,88.364],[3463770,89.924,100.583],[3463771,102.054,112.615],[3463772,117.019,128.693],[3463773,130.275,141.159],[3463774,142.666,153.373],[3463775,154.898,165.996],[3463776,167.475,178.727],[3463777,180.225,191.309],[3463778,192.851,204.057]],"window_offset_ms":0.011},{"bytes":262144,"start":36.353987,"end":42.353983,"completed":352472,"mean_us":14.641,"observed_GBps":15.4,"samples":[[3874276,0.0,14.325],[3874277,16.804,31.548],[3874278,33.973,48.329],[3874279,50.89,65.418],[3874280,67.905,82.541],[3874281,84.986,99.443],[3874282,101.939,117.224],[3874283,119.69,134.636],[3874284,137.226,151.745],[3874285,154.243,168.619],[3874286,171.059,186.033],[3874287,188.649,203.245],[3874288,205.731,220.097],[3874289,222.585,237.582],[3874290,240.038,254.522],[3874291,257.042,271.451]],"window_offset_ms":0.012},{"bytes":524288,"start":42.381265,"end":48.381291,"completed":244554,"mean_us":21.154,"observed_GBps":21.369,"samples":[[4171230,0.0,21.11],[4171231,24.36,48.555],[4171232,51.663,73.026],[4171233,75.804,98.1],[4171234,103.358,124.773],[4171235,129.02,149.122],[4171236,152.359,172.93],[4171237,175.576,196.202],[4171238,198.837,219.995],[4171239,224.605,245.159],[4171240,247.949,268.918],[4171241,273.098,295.383],[4171242,298.143,318.99],[4171243,322.19,342.319],[4171244,347.2,369.441],[4171245,372.336,393.91]],"window_offset_ms":0.024},{"bytes":1048576,"start":48.402749,"end":54.402808,"completed":162825,"mean_us":33.188,"observed_GBps":28.455,"samples":[[4374253,0.0,31.719],[4374254,33.574,65.049],[4374255,67.048,97.913],[4374256,99.901,131.423],[4374257,133.448,164.762],[4374258,166.603,197.58],[4374259,199.59,231.031],[4374260,232.933,264.77],[4374261,267.25,298.842],[4374262,300.937,332.508],[4374263,334.61,366.761],[4374264,368.549,400.516],[4374265,402.769,433.776],[4374266,436.323,493.651],[4374267,495.784,527.771],[4374268,529.905,560.741]],"window_offset_ms":0.023},{"bytes":2097152,"start":54.415165,"end":60.415222,"completed":85442,"mean_us":65.656,"observed_GBps":29.864,"samples":[[4498822,0.0,66.696],[4498823,71.201,135.333],[4498824,139.718,205.802],[4498825,210.445,275.284],[4498826,279.817,344.875],[4498827,349.126,413.626],[4498828,417.896,482.454],[4498829,486.947,551.495],[4498830,555.998,621.87],[4498831,626.27,692.565],[4498832,697.459,762.226],[4498833,766.656,831.417],[4498834,836.132,900.764],[4498835,904.971,969.548],[4498836,973.763,1037.976],[4498837,1042.59,1107.9]],"window_offset_ms":0.004},{"bytes":4194304,"start":60.42393,"end":66.424111,"completed":49574,"mean_us":116.202,"observed_GBps":34.654,"samples":[[4566320,0.0,115.91],[4566321,123.668,239.546],[4566322,244.177,359.366],[4566323,363.9,478.624],[4566324,483.161,597.629],[4566325,602.612,717.617],[4566326,725.648,841.432],[4566327,846.5,961.828],[4566328,966.647,1082.92],[4566329,1087.642,1203.366],[4566330,1208.979,1324.471],[4566331,1329.18,1460.064],[4566332,1464.677,1579.24],[4566333,1584.01,1699.782],[4566334,1704.47,1820.365],[4566335,1825.17,1940.062]],"window_offset_ms":0.039},{"bytes":8388608,"start":66.43139,"end":72.43158,"completed":27011,"mean_us":217.359,"observed_GBps":37.763,"samples":[[4604612,0.0,215.41],[4604613,219.898,436.109],[4604614,440.754,658.464],[4604615,665.097,880.926],[4604616,885.203,1100.251],[4604617,1105.082,1321.067],[4604618,1325.96,1541.575],[4604619,1546.37,1762.987],[4604620,1767.474,1982.482],[4604621,1987.448,2203.607],[4604622,2208.392,2423.214],[4604623,2427.754,2645.202],[4604624,2652.203,2868.914],[4604625,2873.577,3092.038],[4604626,3100.225,3317.586],[4604627,3322.534,3538.847]],"window_offset_ms":0.207},{"bytes":16777216,"start":72.435763,"end":78.436208,"completed":14126,"mean_us":419.597,"observed_GBps":39.496,"samples":[[4625181,0.0,417.933],[4625182,422.786,839.898],[4625183,848.677,1265.083],[4625184,1269.835,1688.491],[4625185,1693.368,2110.355],[4625186,2115.154,2531.763],[4625187,2536.2,2951.679],[4625188,2956.709,3374.607],[4625189,3381.254,3799.718],[4625190,3804.529,4221.007],[4625191,4225.725,4639.911],[4625192,4645.65,5063.748],[4625193,5068.718,5486.762],[4625194,5493.855,5913.11],[4625195,5918.05,6335.469],[4625196,6340.468,6757.882]],"window_offset_ms":0.078},{"bytes":33554432,"start":78.439192,"end":84.439467,"completed":7167,"mean_us":832.336,"observed_GBps":40.079,"samples":[[4635829,0.0,828.15],[4635830,833.146,1661.145],[4635831,1665.791,2493.273],[4635832,2497.85,3323.837],[4635833,3328.698,4155.321],[4635834,4160.197,4990.91],[4635835,4995.632,5822.898],[4635836,5827.712,6656.504],[4635837,6660.901,7488.523],[4635838,7493.183,8323.263],[4635839,8328.229,9153.526],[4635840,9158.465,9987.568],[4635841,9992.015,10821.758],[4635842,10826.776,11656.125],[4635843,11660.465,12488.156],[4635844,12493.12,13320.938]],"window_offset_ms":0.814},{"bytes":67108864,"start":84.447645,"end":90.4491,"completed":3591,"mean_us":1666.336,"observed_GBps":40.155,"samples":[[4641208,0.0,1661.624],[4641209,1665.941,3323.489],[4641210,3328.553,4983.125],[4641211,4987.892,6641.655],[4641212,6646.36,8305.04],[4641213,8310.057,9969.828],[4641214,9974.485,11633.823],[4641215,11638.826,13298.371],[4641216,13303.203,14964.99],[4641217,14969.812,16628.476],[4641218,16635.416,18300.71],[4641219,18305.205,19961.509],[4641220,19966.32,21620.971],[4641221,21625.834,23285.745],[4641222,23290.625,24947.214],[4641223,24951.874,26611.952]],"window_offset_ms":1.041}]},{"run":"write-3","file":"tent-events-slurm-h100-206-025-2234663-oNUhoa.jsonl","op":"write","loss":4.106,"phases":[{"bytes":4096,"start":0.0,"end":5.999124,"completed":607511,"mean_us":7.735,"observed_GBps":0.415,"samples":[[306378,0.0,7.106],[306379,8.418,15.268],[306380,16.543,23.496],[306381,24.922,31.74],[306382,33.23,40.248],[306383,41.556,48.563],[306384,49.897,56.817],[306385,58.134,65.074],[306386,66.412,73.627],[306387,74.948,81.853],[306388,83.194,90.362],[306389,91.707,98.694],[306390,100.011,107.123],[306391,108.67,115.693],[306392,117.006,124.068],[306393,125.466,132.814]],"window_offset_ms":0.006},{"bytes":8192,"start":6.092335,"end":12.050675,"completed":616704,"mean_us":7.264,"observed_GBps":0.848,"samples":[[994998,0.0,7.425],[994999,8.807,15.91],[995000,17.306,24.531],[995001,25.903,33.07],[995002,34.378,41.648],[995003,42.959,50.164],[995004,51.516,58.64],[995005,60.085,67.383],[995006,68.822,75.99],[995007,77.335,84.453],[995008,85.878,93.048],[995009,94.363,101.433],[995010,102.807,109.888],[995011,111.278,118.331],[995012,119.665,126.627],[995013,127.966,135.133]],"window_offset_ms":0.003},{"bytes":16384,"start":12.141776,"end":18.14178,"completed":613728,"mean_us":7.53,"observed_GBps":1.676,"samples":[[1675058,0.0,7.617],[1675059,8.926,16.4],[1675060,17.726,25.137],[1675061,26.528,33.83],[1675062,35.106,42.633],[1675063,43.942,51.692],[1675064,52.976,60.559],[1675065,61.932,69.408],[1675066,70.817,78.125],[1675067,79.545,86.98],[1675068,88.379,95.894],[1675069,97.247,104.633],[1675070,105.95,113.445],[1675071,114.82,122.119],[1675072,123.477,130.963],[1675073,132.387,139.728]],"window_offset_ms":0.001},{"bytes":32768,"start":18.186087,"end":24.186087,"completed":612188,"mean_us":8.001,"observed_GBps":3.343,"samples":[[2322741,0.0,8.102],[2322742,9.442,17.273],[2322743,22.991,32.416],[2322744,34.762,43.415],[2322745,45.896,54.323],[2322746,56.892,65.235],[2322747,67.699,76.179],[2322748,77.778,85.83],[2322749,87.285,96.272],[2322750,97.654,105.864],[2322751,107.254,115.045],[2322752,116.427,124.592],[2322753,125.985,134.125],[2322754,135.569,143.578],[2322755,145.058,153.149],[2322756,154.592,163.254]],"window_offset_ms":0.005},{"bytes":65536,"start":24.23345,"end":30.233454,"completed":574480,"mean_us":8.917,"observed_GBps":6.275,"samples":[[2925623,0.0,8.63],[2925624,9.969,18.755],[2925625,20.025,28.646],[2925626,30.058,38.556],[2925627,40.043,48.793],[2925628,50.075,59.884],[2925629,61.154,70.132],[2925630,71.397,80.368],[2925631,81.64,90.425],[2925632,91.766,100.443],[2925633,101.724,110.392],[2925634,111.647,121.313],[2925635,122.566,131.253],[2925636,132.526,141.402],[2925637,142.843,151.37],[2925638,152.759,161.274]],"window_offset_ms":0.001},{"bytes":131072,"start":30.274356,"end":36.274365,"completed":466501,"mean_us":11.087,"observed_GBps":10.191,"samples":[[3447295,0.0,11.656],[3447296,14.53,26.068],[3447297,29.9,40.963],[3447298,43.085,53.584],[3447299,55.3,66.003],[3447300,67.529,79.392],[3447301,82.177,93.693],[3447302,96.426,107.142],[3447303,108.677,119.374],[3447304,121.086,132.223],[3447305,133.868,144.804],[3447306,148.165,160.116],[3447307,162.701,174.164],[3447308,175.696,186.989],[3447309,188.752,200.915],[3447310,202.905,213.721]],"window_offset_ms":0.003},{"bytes":262144,"start":36.306842,"end":42.306844,"completed":347135,"mean_us":14.545,"observed_GBps":15.167,"samples":[[3854509,0.0,14.215],[3854510,18.346,32.762],[3854511,36.204,50.684],[3854512,55.839,70.1],[3854513,73.459,88.606],[3854514,91.997,106.964],[3854515,110.184,124.383],[3854516,126.767,141.745],[3854517,145.1,160.402],[3854518,164.407,178.199],[3854519,180.613,195.201],[3854520,198.647,212.785],[3854521,216.118,231.217],[3854522,234.134,248.394],[3854523,250.935,269.942],[3854524,272.551,286.861]],"window_offset_ms":0.004},{"bytes":524288,"start":42.333028,"end":48.333051,"completed":240348,"mean_us":20.854,"observed_GBps":21.002,"samples":[[4147892,0.0,20.74],[4147893,23.484,43.474],[4147894,46.873,70.856],[4147895,74.331,95.715],[4147896,99.199,119.439],[4147897,122.377,142.726],[4147898,145.667,165.859],[4147899,169.499,189.606],[4147900,192.249,212.463],[4147901,218.63,239.244],[4147902,243.092,263.532],[4147903,267.098,287.109],[4147904,292.775,314.093],[4147905,316.842,337.517],[4147906,349.935,371.799],[4147907,375.172,395.927]],"window_offset_ms":0.019},{"bytes":1048576,"start":48.356557,"end":54.356611,"completed":159543,"mean_us":33.464,"observed_GBps":27.882,"samples":[[4347935,0.0,33.689],[4347936,38.03,70.922],[4347937,74.922,107.787],[4347938,111.729,145.178],[4347939,149.501,182.721],[4347940,186.72,219.34],[4347941,223.379,257.401],[4347942,261.648,318.101],[4347943,322.074,356.857],[4347944,366.413,399.183],[4347945,403.505,436.942],[4347946,440.9,473.61],[4347947,477.581,511.282],[4347948,515.52,553.049],[4347949,557.165,589.813],[4347950,594.103,626.595]],"window_offset_ms":0.008},{"bytes":2097152,"start":54.369995,"end":60.370027,"completed":85025,"mean_us":65.693,"observed_GBps":29.718,"samples":[[4470151,0.0,65.309],[4470152,70.239,136.64],[4470153,141.591,207.254],[4470154,212.54,278.706],[4470155,283.509,349.613],[4470156,354.466,418.825],[4470157,423.802,489.189],[4470158,494.343,558.886],[4470159,563.757,629.604],[4470160,634.265,698.916],[4470161,703.607,768.1],[4470162,773.283,838.776],[4470163,843.625,907.873],[4470164,912.757,978.881],[4470165,983.526,1049.74],[4470166,1062.608,1128.412]],"window_offset_ms":0.049},{"bytes":4194304,"start":60.381334,"end":66.381422,"completed":49348,"mean_us":116.121,"observed_GBps":34.496,"samples":[[4537285,0.0,114.541],[4537286,119.255,233.956],[4537287,238.715,353.819],[4537288,358.863,474.942],[4537289,483.936,600.192],[4537290,604.821,723.462],[4537291,728.366,843.91],[4537292,848.509,964.616],[4537293,969.255,1085.345],[4537294,1097.288,1213.378],[4537295,1222.118,1336.522],[4537296,1341.518,1457.854],[4537297,1463.058,1577.623],[4537298,1582.364,1697.548],[4537299,1702.366,1818.341],[4537300,1824.593,1940.278]],"window_offset_ms":0.118},{"bytes":8388608,"start":66.391363,"end":72.391891,"completed":26895,"mean_us":217.413,"observed_GBps":37.599,"samples":[[4575410,0.0,216.528],[4575411,221.865,438.755],[4575412,444.867,660.249],[4575413,665.233,880.794],[4575414,885.806,1101.139],[4575415,1105.927,1320.428],[4575416,1325.435,1542.036],[4575417,1547.497,1763.776],[4575418,1768.57,1986.05],[4575419,1990.973,2208.423],[4575420,2213.733,2431.072],[4575421,2435.831,2653.073],[4575422,2658.072,2873.982],[4575423,2879.132,3094.988],[4575424,3099.83,3314.171],[4575425,3319.064,3536.433]],"window_offset_ms":0.179},{"bytes":16777216,"start":72.397433,"end":78.398079,"completed":14122,"mean_us":419.71,"observed_GBps":39.484,"samples":[[4595916,0.0,417.138],[4595917,422.105,840.712],[4595918,845.533,1262.375],[4595919,1267.183,1686.239],[4595920,1690.736,2109.471],[4595921,2119.603,2539.13],[4595922,2543.698,2961.383],[4595923,2966.631,3383.724],[4595924,3388.821,3807.255],[4595925,3812.203,4231.296],[4595926,4236.164,4657.043],[4595927,4662.175,5078.677],[4595928,5083.479,5497.868],[4595929,5502.64,5918.715],[4595930,5931.731,6348.223],[4595931,6352.821,6771.429]],"window_offset_ms":0.021},{"bytes":33554432,"start":78.402316,"end":84.402505,"completed":7164,"mean_us":832.284,"observed_GBps":40.063,"samples":[[4606558,0.0,829.809],[4606559,834.956,1662.889],[4606560,1667.876,2496.151],[4606561,2501.385,3327.097],[4606562,3331.874,4163.492],[4606563,4168.312,5001.804],[4606564,5007.213,5837.439],[4606565,5842.291,6670.394],[4606566,6675.372,7506.976],[4606567,7512.164,8340.665],[4606568,8345.427,9173.854],[4606569,9178.947,10007.386],[4606570,10012.045,10840.0],[4606571,10845.143,11672.532],[4606572,11677.827,12504.49],[4606573,12509.759,13338.211]],"window_offset_ms":0.248},{"bytes":67108864,"start":84.406908,"end":90.409688,"completed":3591,"mean_us":1666.355,"observed_GBps":40.146,"samples":[[4611936,0.0,1658.217],[4611937,1662.947,3321.403],[4611938,3326.476,4991.777],[4611939,4996.517,6651.342],[4611940,6656.306,8316.856],[4611941,8322.042,9976.873],[4611942,9981.757,12611.118],[4611943,12615.772,14080.52],[4611944,14093.519,15747.43],[4611945,15752.416,17408.966],[4611946,17421.455,19079.815],[4611947,19087.883,20745.951],[4611948,20750.778,22408.043],[4611949,22420.15,24077.595],[4611950,24085.382,25741.838],[4611951,25746.518,27403.808]],"window_offset_ms":0.731}]}] diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.command.txt new file mode 100644 index 0000000000..4831df9970 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=write --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/write-1.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.jsonl new file mode 100644 index 0000000000..a9a763b9b2 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":567460,"aggregate_throughput_gbps":0.4648625811923864,"aggregate_transferred_bytes":2324316160,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5260555524390206,"avg_latency_us":8.811206076199204,"avg_transfer_us":7.889770204067247,"index":0,"operations":567460,"p999_us":19.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.4648625811923864,"total_duration_us":5000007.0,"transferred_bytes":2324316160}]} +{"aggregate_operations":570844,"aggregate_throughput_gbps":0.9352708096000001,"aggregate_transferred_bytes":4676354048,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.0298261104542912,"avg_latency_us":8.758960416506087,"avg_transfer_us":8.005153772309072,"index":0,"operations":570844,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.9352708096000001,"total_duration_us":5000000.0,"transferred_bytes":4676354048}]} +{"aggregate_operations":556147,"aggregate_throughput_gbps":1.8223810316951747,"aggregate_transferred_bytes":9111912448,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.028475210987776,"avg_latency_us":8.990435981853718,"avg_transfer_us":8.118330225641781,"index":0,"operations":556147,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.8223810316951747,"total_duration_us":5000004.0,"transferred_bytes":9111912448}]} +{"aggregate_operations":524262,"aggregate_throughput_gbps":3.435800007399992,"aggregate_transferred_bytes":17179017216,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.9211486978991177,"avg_latency_us":9.537225661978171,"avg_transfer_us":8.457549851028684,"index":0,"operations":524262,"p999_us":22.0,"p99_us":11.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.435800007399992,"total_duration_us":5000005.0,"transferred_bytes":17179017216}]} +{"aggregate_operations":478456,"aggregate_throughput_gbps":6.271210957746851,"aggregate_transferred_bytes":31356092416,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.1181172805762305,"avg_latency_us":10.450294279933788,"avg_transfer_us":9.364821425585633,"index":0,"operations":478456,"p999_us":41.0,"p99_us":12.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.271210957746851,"total_duration_us":5000006.0,"transferred_bytes":31356092416}]} +{"aggregate_operations":387960,"aggregate_throughput_gbps":10.170130487895609,"aggregate_transferred_bytes":50850693120,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.151412618403493,"avg_latency_us":12.887936900711413,"avg_transfer_us":11.834805134549953,"index":0,"operations":387960,"p999_us":21.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.170130487895609,"total_duration_us":5000004.0,"transferred_bytes":50850693120}]} +{"aggregate_operations":297028,"aggregate_throughput_gbps":15.57279046081908,"aggregate_transferred_bytes":77864108032,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":17.02388273218726,"avg_latency_us":16.833463511857467,"avg_transfer_us":15.43161924128365,"index":0,"operations":297028,"p999_us":21.0,"p99_us":17.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":15.57279046081908,"total_duration_us":5000010.0,"transferred_bytes":77864108032}]} +{"aggregate_operations":204925,"aggregate_throughput_gbps":21.487849133463815,"aggregate_transferred_bytes":107439718400,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":23.00729256968598,"avg_latency_us":24.399277784555323,"avg_transfer_us":22.893585458094424,"index":0,"operations":204925,"p999_us":32.0,"p99_us":27.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":21.487849133463815,"total_duration_us":5000022.0,"transferred_bytes":107439718400}]} +{"aggregate_operations":134409,"aggregate_throughput_gbps":28.187441192152846,"aggregate_transferred_bytes":140938051584,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":29.771722064342786,"avg_latency_us":37.20011308766526,"avg_transfer_us":35.3444709803659,"index":0,"operations":134409,"p999_us":60.0,"p99_us":58.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":28.187441192152846,"total_duration_us":5000030.0,"transferred_bytes":140938051584}]} +{"aggregate_operations":71409,"aggregate_throughput_gbps":29.951099443380112,"aggregate_transferred_bytes":149755527168,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.177730077481755,"avg_latency_us":70.0191992605974,"avg_transfer_us":67.39885728689661,"index":0,"operations":71409,"p999_us":103.0,"p99_us":99.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.951099443380112,"total_duration_us":5000001.0,"transferred_bytes":149755527168}]} +{"aggregate_operations":41370,"aggregate_throughput_gbps":34.703608829504105,"aggregate_transferred_bytes":173518356480,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.54273117423244,"avg_latency_us":120.86074450084602,"avg_transfer_us":118.27810007251631,"index":0,"operations":41370,"p999_us":183.0,"p99_us":178.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":34.703608829504105,"total_duration_us":5000009.0,"transferred_bytes":173518356480}]} +{"aggregate_operations":22465,"aggregate_throughput_gbps":37.688530815885855,"aggregate_transferred_bytes":188450078720,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.22826352241568,"avg_latency_us":222.57720899176496,"avg_transfer_us":219.91457823280658,"index":0,"operations":22465,"p999_us":342.5360000000037,"p99_us":257.5200000000186,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":37.688530815885855,"total_duration_us":5000197.0,"transferred_bytes":188450078720}]} +{"aggregate_operations":11718,"aggregate_throughput_gbps":39.317007479605074,"aggregate_transferred_bytes":196595417088,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.75149678957684,"avg_latency_us":426.71650452295614,"avg_transfer_us":424.14285714285717,"index":0,"operations":11718,"p999_us":911.1130000000139,"p99_us":658.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":39.317007479605074,"total_duration_us":5000264.0,"transferred_bytes":196595417088}]} +{"aggregate_operations":5973,"aggregate_throughput_gbps":40.07759983394703,"aggregate_transferred_bytes":200420622336,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.28990827150249,"avg_latency_us":837.2365645404319,"avg_transfer_us":834.5655449522853,"index":0,"operations":5973,"p999_us":1290.0,"p99_us":1010.3599999999078,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.07759983394703,"total_duration_us":5000814.0,"transferred_bytes":200420622336}]} +{"aggregate_operations":2992,"aggregate_throughput_gbps":40.15367989679496,"aggregate_transferred_bytes":200789721088,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.30865104129517,"avg_latency_us":1671.3004679144385,"avg_transfer_us":1668.6818181818182,"index":0,"operations":2992,"p999_us":2630.0090000000005,"p99_us":1800.1800000001313,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.15367989679496,"total_duration_us":5000531.0,"transferred_bytes":200789721088}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.log new file mode 100644 index 0000000000..3bdfcf4d78 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-1.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:11:07.327808 2232113 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:11:07.327888 2232113 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:11:07.328307 2232113 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:11:07.332800 2232113 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:11:07.332806 2232113 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:11:07.336367 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:11:07.341008 2232113 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:11:07.341017 2232113 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:11:07.344523 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:11:07.349125 2232113 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:11:07.349133 2232113 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:11:07.352630 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:11:07.357265 2232113 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:11:07.357272 2232113 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:11:07.360749 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:11:07.365381 2232113 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:11:07.365387 2232113 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:11:07.368930 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:11:07.373584 2232113 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:11:07.373591 2232113 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:11:07.377125 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:11:07.381755 2232113 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:11:07.381762 2232113 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:11:07.385306 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:11:07.389922 2232113 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:11:07.389930 2232113 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:11:07.393457 2232113 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2232113-2zgoGf.jsonl +I0924 19:11:07.403134 2232113 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:11:07.403142 2232113 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:11:07.403146 2232113 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:15344 started successfully +I0924 19:11:07.539314 2232113 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 133.08 ms, registered in 3.07029 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:11:07.539356 2232113 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:11:07.540269 2232113 tent_backend.cpp:356] Opened 1 target segments +I0924 19:11:07.541702 2232314 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.464863 0.526056 8.8 7.9 10.0 19.0 +8192 1 0.935271 1.029826 8.8 8.0 10.0 12.0 +16384 1 1.822381 2.028475 9.0 8.1 10.0 12.0 +32768 1 3.435800 3.921149 9.5 8.5 11.0 22.0 +65536 1 6.271211 7.118117 10.5 9.4 12.0 41.0 +I0924 19:11:37.760926 2232315 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.170130 11.151413 12.9 11.8 15.0 21.0 +I0924 19:11:43.792387 2232316 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:11:43.792415 2232317 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 15.572790 17.023883 16.8 15.4 17.0 21.0 +I0924 19:11:49.837131 2232318 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:11:49.837146 2232319 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 21.487849 23.007293 24.4 22.9 27.0 32.0 +1048576 1 28.187441 29.771722 37.2 35.3 58.0 60.0 +2097152 1 29.951099 31.177730 70.0 67.4 99.0 103.0 +4194304 1 34.703609 35.542731 120.9 118.3 178.0 183.0 +8388608 1 37.688531 38.228264 222.6 219.9 257.5 342.5 +16777216 1 39.317007 39.751497 426.7 424.1 658.0 911.1 +33554432 1 40.077600 40.289908 837.2 834.6 1010.4 1290.0 +67108864 1 40.153680 40.308651 1671.3 1668.7 1800.2 2630.0 +TENT trace incomplete: 505420 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2232113-2zgoGf.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.command.txt new file mode 100644 index 0000000000..e170b2b28b --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=write --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/write-2.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.jsonl new file mode 100644 index 0000000000..f24ecf0195 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":569790,"aggregate_throughput_gbps":0.46677140787431054,"aggregate_transferred_bytes":2333859840,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5295679339307293,"avg_latency_us":8.77517330946489,"avg_transfer_us":7.831869636181751,"index":0,"operations":569790,"p999_us":14.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.46677140787431054,"total_duration_us":5000006.0,"transferred_bytes":2333859840}]} +{"aggregate_operations":577868,"aggregate_throughput_gbps":0.946777795066646,"aggregate_transferred_bytes":4733894656,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.0512268942220189,"avg_latency_us":8.65250541646189,"avg_transfer_us":7.849700969771644,"index":0,"operations":577868,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.946777795066646,"total_duration_us":5000006.0,"transferred_bytes":4733894656}]} +{"aggregate_operations":561476,"aggregate_throughput_gbps":1.8398423489891813,"aggregate_transferred_bytes":9199222784,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.0408080293887614,"avg_latency_us":8.905110815066005,"avg_transfer_us":8.060777308380056,"index":0,"operations":561476,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.8398423489891813,"total_duration_us":5000006.0,"transferred_bytes":9199222784}]} +{"aggregate_operations":526898,"aggregate_throughput_gbps":3.4530787328000003,"aggregate_transferred_bytes":17265393664,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.9502698591864323,"avg_latency_us":9.489502712099876,"avg_transfer_us":8.374089482214774,"index":0,"operations":526898,"p999_us":18.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.4530787328000003,"total_duration_us":5000000.0,"transferred_bytes":17265393664}]} +{"aggregate_operations":487297,"aggregate_throughput_gbps":6.387091573890111,"aggregate_transferred_bytes":31935496192,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.228450178612315,"avg_latency_us":10.26069522283125,"avg_transfer_us":9.192984976308082,"index":0,"operations":487297,"p999_us":35.0,"p99_us":12.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.387091573890111,"total_duration_us":5000006.0,"transferred_bytes":31935496192}]} +{"aggregate_operations":388953,"aggregate_throughput_gbps":10.196167483966503,"aggregate_transferred_bytes":50980847616,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.122271100809694,"avg_latency_us":12.855026185683103,"avg_transfer_us":11.838036986473945,"index":0,"operations":388953,"p999_us":18.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.196167483966503,"total_duration_us":5000001.0,"transferred_bytes":50980847616}]} +{"aggregate_operations":293072,"aggregate_throughput_gbps":15.3654132736,"aggregate_transferred_bytes":76827066368,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":16.57416211034705,"avg_latency_us":17.06065403723317,"avg_transfer_us":15.854643227602773,"index":0,"operations":293072,"p999_us":22.0,"p99_us":18.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":15.3654132736,"total_duration_us":5000000.0,"transferred_bytes":76827066368}]} +{"aggregate_operations":203284,"aggregate_throughput_gbps":21.315778568974295,"aggregate_transferred_bytes":106579361792,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":22.826877771656886,"avg_latency_us":24.596239743413157,"avg_transfer_us":23.057431967100214,"index":0,"operations":203284,"p999_us":32.0,"p99_us":27.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":21.315778568974295,"total_duration_us":5000022.0,"transferred_bytes":106579361792}]} +{"aggregate_operations":135896,"aggregate_throughput_gbps":28.499240224974287,"aggregate_transferred_bytes":142497284096,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":30.184189574996363,"avg_latency_us":36.79312121033732,"avg_transfer_us":34.8942794489904,"index":0,"operations":135896,"p999_us":60.0,"p99_us":56.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":28.499240224974287,"total_duration_us":5000038.0,"transferred_bytes":142497284096}]} +{"aggregate_operations":71198,"aggregate_throughput_gbps":29.862402554862623,"aggregate_transferred_bytes":149313028096,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":31.1070841348791,"avg_latency_us":70.2271693025085,"avg_transfer_us":67.54861091603696,"index":0,"operations":71198,"p999_us":103.0,"p99_us":99.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.862402554862623,"total_duration_us":5000034.0,"transferred_bytes":149313028096}]} +{"aggregate_operations":41307,"aggregate_throughput_gbps":34.650234011621805,"aggregate_transferred_bytes":173254115328,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.54844410650208,"avg_latency_us":121.04691698743554,"avg_transfer_us":118.24724622945263,"index":0,"operations":41307,"p999_us":183.0,"p99_us":178.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":34.650234011621805,"total_duration_us":5000085.0,"transferred_bytes":173254115328}]} +{"aggregate_operations":22509,"aggregate_throughput_gbps":37.76248359748721,"aggregate_transferred_bytes":188819177472,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.31593074184945,"avg_latency_us":222.14132124927806,"avg_transfer_us":219.40681505175706,"index":0,"operations":22509,"p999_us":342.0,"p99_us":334.35999999998603,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":37.76248359748721,"total_duration_us":5000179.0,"transferred_bytes":188819177472}]} +{"aggregate_operations":11772,"aggregate_throughput_gbps":39.497670504146726,"aggregate_transferred_bytes":197501386752,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.84968336447385,"avg_latency_us":424.7646958885491,"avg_transfer_us":421.9272001359157,"index":0,"operations":11772,"p999_us":662.0,"p99_us":457.50999999998203,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":39.497670504146726,"total_duration_us":5000330.0,"transferred_bytes":197501386752}]} +{"aggregate_operations":5972,"aggregate_throughput_gbps":40.07619526446396,"aggregate_transferred_bytes":200387067904,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.2994347437774,"avg_latency_us":837.2659075686537,"avg_transfer_us":834.4261553918285,"index":0,"operations":5972,"p999_us":1291.0,"p99_us":1280.29,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.07619526446396,"total_duration_us":5000152.0,"transferred_bytes":200387067904}]} +{"aggregate_operations":2992,"aggregate_throughput_gbps":40.15543048765147,"aggregate_transferred_bytes":200789721088,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.316482586121595,"avg_latency_us":1671.2276069518716,"avg_transfer_us":1668.385360962567,"index":0,"operations":2992,"p999_us":2636.0,"p99_us":2018.5800000000963,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.15543048765147,"total_duration_us":5000313.0,"transferred_bytes":200789721088}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.log new file mode 100644 index 0000000000..c82868dff7 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-2.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:14:15.376253 2233390 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:14:15.376325 2233390 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:14:15.376713 2233390 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:14:15.381251 2233390 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:14:15.381258 2233390 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:14:15.384708 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:14:15.389348 2233390 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:14:15.389355 2233390 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:14:15.392874 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:14:15.397509 2233390 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:14:15.397522 2233390 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:14:15.401007 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:14:15.405615 2233390 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:14:15.405622 2233390 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:14:15.409132 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:14:15.413748 2233390 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:14:15.413755 2233390 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:14:15.417279 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:14:15.421927 2233390 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:14:15.421934 2233390 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:14:15.425487 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:14:15.430200 2233390 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:14:15.430207 2233390 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:14:15.433723 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:14:15.438374 2233390 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:14:15.438380 2233390 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:14:15.441932 2233390 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2233390-uky9P4.jsonl +I0924 19:14:15.452504 2233390 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:14:15.452512 2233390 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:14:15.452514 2233390 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:15723 started successfully +I0924 19:14:15.592624 2233390 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 137.077 ms, registered in 3.0184 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:14:15.592664 2233390 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:14:15.593487 2233390 tent_backend.cpp:356] Opened 1 target segments +I0924 19:14:15.594870 2233590 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.466771 0.529568 8.8 7.8 10.0 14.0 +8192 1 0.946778 1.051227 8.7 7.8 10.0 12.0 +16384 1 1.839842 2.040808 8.9 8.1 10.0 12.0 +32768 1 3.453079 3.950270 9.5 8.4 10.0 18.0 +65536 1 6.387092 7.228450 10.3 9.2 12.0 35.0 +I0924 19:14:45.914480 2233591 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.196167 11.122271 12.9 11.8 15.0 18.0 +I0924 19:14:51.949473 2233593 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:14:51.949506 2233592 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 15.365413 16.574162 17.1 15.9 18.0 22.0 +I0924 19:14:57.976567 2233595 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:14:57.976572 2233594 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 21.315779 22.826878 24.6 23.1 27.0 32.0 +1048576 1 28.499240 30.184190 36.8 34.9 56.0 60.0 +2097152 1 29.862403 31.107084 70.2 67.5 99.0 103.0 +4194304 1 34.650234 35.548444 121.0 118.2 178.0 183.0 +8388608 1 37.762484 38.315931 222.1 219.4 334.4 342.0 +16777216 1 39.497671 39.849683 424.8 421.9 457.5 662.0 +33554432 1 40.076195 40.299435 837.3 834.4 1280.3 1291.0 +67108864 1 40.155430 40.316483 1671.2 1668.4 2018.6 2636.0 +TENT trace incomplete: 593004 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2233390-uky9P4.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.command.txt b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.command.txt new file mode 100644 index 0000000000..37a573baf0 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.command.txt @@ -0,0 +1 @@ +/mnt/home/kimbo/networkx/Mooncake/build/tebench-pyxis/mooncake-transfer-engine/benchmark/tebench --backend=tent --xport_type=rdma --metadata_type=p2p --rpc_server_port=0 --seg_type=VRAM --local_gpu_id=0 --target_gpu_id=0 --total_buffer_size=268435456 --target_seg_name=10.0.3.49:15309 --op_type=write --start_block_size=4096 --max_block_size=67108864 --start_batch_size=1 --max_batch_size=1 --start_num_threads=1 --max_num_threads=1 --duration=5 --result_output_jsonl=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/write-3.jsonl diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.jsonl b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.jsonl new file mode 100644 index 0000000000..8d17bfeb40 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.jsonl @@ -0,0 +1,15 @@ +{"aggregate_operations":575162,"aggregate_throughput_gbps":0.47117176805646394,"aggregate_transferred_bytes":2355863552,"backend":"tent","batch_size":1,"block_size":4096,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":0.5362855343045319,"avg_latency_us":8.693220344876748,"avg_transfer_us":7.735691857250653,"index":0,"operations":575162,"p999_us":19.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.47117176805646394,"total_duration_us":5000010.0,"transferred_bytes":2355863552}]} +{"aggregate_operations":573389,"aggregate_throughput_gbps":0.9394401618239353,"aggregate_transferred_bytes":4697202688,"backend":"tent","batch_size":1,"block_size":8192,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":1.036151716784646,"avg_latency_us":8.720087061314395,"avg_transfer_us":7.956723969242521,"index":0,"operations":573389,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":0.9394401618239353,"total_duration_us":5000002.0,"transferred_bytes":4697202688}]} +{"aggregate_operations":553975,"aggregate_throughput_gbps":1.815263101684278,"aggregate_transferred_bytes":9076326400,"backend":"tent","batch_size":1,"block_size":16384,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":2.0264264917015598,"avg_latency_us":9.025688884877477,"avg_transfer_us":8.125704228530168,"index":0,"operations":553975,"p999_us":12.0,"p99_us":10.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":1.815263101684278,"total_duration_us":5000006.0,"transferred_bytes":9076326400}]} +{"aggregate_operations":528431,"aggregate_throughput_gbps":3.4631240163503936,"aggregate_transferred_bytes":17315627008,"backend":"tent","batch_size":1,"block_size":32768,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":3.9607125060948323,"avg_latency_us":9.461977060391991,"avg_transfer_us":8.346489891773949,"index":0,"operations":528431,"p999_us":17.570000000065193,"p99_us":11.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":3.4631240163503936,"total_duration_us":5000002.0,"transferred_bytes":17315627008}]} +{"aggregate_operations":479551,"aggregate_throughput_gbps":6.285569610086077,"aggregate_transferred_bytes":31427854336,"backend":"tent","batch_size":1,"block_size":65536,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":7.148256100238229,"avg_latency_us":10.426421798724224,"avg_transfer_us":9.328757525268427,"index":0,"operations":479551,"p999_us":42.0,"p99_us":12.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":6.285569610086077,"total_duration_us":5000001.0,"transferred_bytes":31427854336}]} +{"aggregate_operations":388399,"aggregate_throughput_gbps":10.181636563963435,"aggregate_transferred_bytes":50908233728,"backend":"tent","batch_size":1,"block_size":131072,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":11.10340264060429,"avg_latency_us":12.873372485511034,"avg_transfer_us":11.860233934690871,"index":0,"operations":388399,"p999_us":18.0,"p99_us":15.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":10.181636563963435,"total_duration_us":5000005.0,"transferred_bytes":50908233728}]} +{"aggregate_operations":288932,"aggregate_throughput_gbps":15.148351982259207,"aggregate_transferred_bytes":75741790208,"backend":"tent","batch_size":1,"block_size":262144,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":16.42676767918529,"avg_latency_us":17.305116774881288,"avg_transfer_us":16.00646172801905,"index":0,"operations":288932,"p999_us":22.0,"p99_us":19.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":15.148351982259207,"total_duration_us":5000002.0,"transferred_bytes":75741790208}]} +{"aggregate_operations":200376,"aggregate_throughput_gbps":21.010883424949725,"aggregate_transferred_bytes":105054732288,"backend":"tent","batch_size":1,"block_size":524288,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":22.571327024104725,"avg_latency_us":24.953163053459495,"avg_transfer_us":23.4099892202659,"index":0,"operations":200376,"p999_us":37.0,"p99_us":32.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":21.010883424949725,"total_duration_us":5000015.0,"transferred_bytes":105054732288}]} +{"aggregate_operations":132832,"aggregate_throughput_gbps":27.856722306066164,"aggregate_transferred_bytes":139284447232,"backend":"tent","batch_size":1,"block_size":1048576,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":29.53676493923068,"avg_latency_us":37.641758010118046,"avg_transfer_us":35.61988075162611,"index":0,"operations":132832,"p999_us":60.0,"p99_us":58.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":27.856722306066164,"total_duration_us":5000030.0,"transferred_bytes":139284447232}]} +{"aggregate_operations":70859,"aggregate_throughput_gbps":29.720204728125957,"aggregate_transferred_bytes":148602093568,"backend":"tent","batch_size":1,"block_size":2097152,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":30.97854545447461,"avg_latency_us":70.56317475550036,"avg_transfer_us":67.84469157058383,"index":0,"operations":70859,"p999_us":103.0,"p99_us":99.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":29.720204728125957,"total_duration_us":5000036.0,"transferred_bytes":148602093568}]} +{"aggregate_operations":41123,"aggregate_throughput_gbps":34.49596903725205,"aggregate_transferred_bytes":172482363392,"backend":"tent","batch_size":1,"block_size":4194304,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":35.36690798476519,"avg_latency_us":121.58823529411765,"avg_transfer_us":118.86499039466965,"index":0,"operations":41123,"p999_us":184.0,"p99_us":176.77999999999884,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":34.49596903725205,"total_duration_us":5000073.0,"transferred_bytes":172482363392}]} +{"aggregate_operations":22411,"aggregate_throughput_gbps":37.59786222610384,"aggregate_transferred_bytes":187997093888,"backend":"tent","batch_size":1,"block_size":8388608,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":38.15450038371747,"avg_latency_us":223.1139618937129,"avg_transfer_us":220.37927803310873,"index":0,"operations":22411,"p999_us":344.0,"p99_us":336.0,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":37.59786222610384,"total_duration_us":5000207.0,"transferred_bytes":187997093888}]} +{"aggregate_operations":11767,"aggregate_throughput_gbps":39.481336557156666,"aggregate_transferred_bytes":197417500672,"backend":"tent","batch_size":1,"block_size":16777216,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":39.819018703311556,"avg_latency_us":424.9404266168097,"avg_transfer_us":422.27832072745815,"index":0,"operations":11767,"p999_us":663.0,"p99_us":652.3400000000001,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":39.481336557156666,"total_duration_us":5000274.0,"transferred_bytes":197417500672}]} +{"aggregate_operations":5970,"aggregate_throughput_gbps":40.06262967859092,"aggregate_transferred_bytes":200319959040,"backend":"tent","batch_size":1,"block_size":33554432,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.277397672636226,"avg_latency_us":837.5494137353434,"avg_transfer_us":834.8100502512563,"index":0,"operations":5970,"p999_us":1290.0310000000009,"p99_us":1000.9800000000632,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.06262967859092,"total_duration_us":5000170.0,"transferred_bytes":200319959040}]} +{"aggregate_operations":2992,"aggregate_throughput_gbps":40.14541884691976,"aggregate_transferred_bytes":200789721088,"backend":"tent","batch_size":1,"block_size":67108864,"num_threads":1,"op_type":"write","record_type":"target_metrics","schema_version":1,"targets":[{"avg_instant_gbps":40.3010037172349,"avg_latency_us":1671.644385026738,"avg_transfer_us":1669.0183823529412,"index":0,"operations":2992,"p999_us":2635.0090000000005,"p99_us":1897.100000000115,"segment_name":"10.0.3.49:15309","threads":1,"throughput_gbps":40.14541884691976,"total_duration_us":5001560.0,"transferred_bytes":200789721088}]} diff --git a/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.log b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.log new file mode 100644 index 0000000000..ab0d2bd8b5 --- /dev/null +++ b/experimental/mooncake-transfer-traces/results/tebench-1695.6UKR6R/write-3.log @@ -0,0 +1,59 @@ +WARNING: Logging before InitGoogleLogging() is written to STDERR +I0924 19:17:23.257109 2234663 transport_selector.cpp:129] No 'policy' configured, using default transport selection +I0924 19:17:23.257181 2234663 ibv_loader.cpp:40] libibverbs lacks optional symbol ibv_query_port_speed; the fallback path will be used +I0924 19:17:23.257588 2234663 ibv_loader.cpp:123] IbvLoader: libibverbs loaded successfully, devices=8 +W0924 19:17:23.262072 2234663 context.cpp:270] No network-attached GID found on ibp0, using non-network GID[0] +I0924 19:17:23.262079 2234663 context.cpp:938] Resolved GID index 0 for device ibp0: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:84:44 +I0924 19:17:23.265558 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp0 +W0924 19:17:23.270175 2234663 context.cpp:270] No network-attached GID found on ibp1, using non-network GID[0] +I0924 19:17:23.270182 2234663 context.cpp:938] Resolved GID index 0 for device ibp1: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:a2 +I0924 19:17:23.273674 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp1 +W0924 19:17:23.278211 2234663 context.cpp:270] No network-attached GID found on ibp2, using non-network GID[0] +I0924 19:17:23.278218 2234663 context.cpp:938] Resolved GID index 0 for device ibp2: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:fc +I0924 19:17:23.281709 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp2 +W0924 19:17:23.286295 2234663 context.cpp:270] No network-attached GID found on ibp3, using non-network GID[0] +I0924 19:17:23.286302 2234663 context.cpp:938] Resolved GID index 0 for device ibp3: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:f4 +I0924 19:17:23.289790 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp3 +W0924 19:17:23.294361 2234663 context.cpp:270] No network-attached GID found on ibp4, using non-network GID[0] +I0924 19:17:23.294368 2234663 context.cpp:938] Resolved GID index 0 for device ibp4: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7e:2c +I0924 19:17:23.297894 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp4 +W0924 19:17:23.302476 2234663 context.cpp:270] No network-attached GID found on ibp5, using non-network GID[0] +I0924 19:17:23.302484 2234663 context.cpp:938] Resolved GID index 0 for device ibp5: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:96:7d:a4 +I0924 19:17:23.305974 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp5 +W0924 19:17:23.310549 2234663 context.cpp:270] No network-attached GID found on ibp6, using non-network GID[0] +I0924 19:17:23.310556 2234663 context.cpp:938] Resolved GID index 0 for device ibp6: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:43:d2 +I0924 19:17:23.314056 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp6 +W0924 19:17:23.318684 2234663 context.cpp:270] No network-attached GID found on ibp7, using non-network GID[0] +I0924 19:17:23.318691 2234663 context.cpp:938] Resolved GID index 0 for device ibp7: fe:80:00:00:00:00:00:00:a0:88:c2:03:00:99:48:b2 +I0924 19:17:23.322201 2234663 context.cpp:408] [RDMA] Relaxed ordering is supported and enabled for ibp7 +TENT transfer events: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2234663-oNUhoa.jsonl +I0924 19:17:23.330044 2234663 tent_metrics.cpp:789] TENT metrics disabled at compile time (TENT_METRICS_ENABLED=0) +I0924 19:17:23.330055 2234663 transfer_engine_impl.cpp:560] TENT Metrics system initialized +I0924 19:17:23.330060 2234663 transfer_engine_impl.cpp:577] Transfer Engine 10.0.1.136:16369 started successfully +I0924 19:17:23.475998 2234663 tent_backend.cpp:281] Allocated 268435456 bytes across 1 buffers (1 seg_types) in 142.843 ms, registered in 3.07708 ms +BlkSize (B) Batch BW (GB/s) Avg Inst GB/s Avg Lat (us) Avg Tx (us) P99 Tx (us) P999 Tx (us) +---------------------------------------------------------------------------------------------------------------------------------------------------------------- +I0924 19:17:23.476038 2234663 segment_manager.cpp:56] Opened segment #1: 10.0.3.49:15309 +I0924 19:17:23.476894 2234663 tent_backend.cpp:356] Opened 1 target segments +I0924 19:17:23.478582 2234863 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +4096 1 0.471172 0.536286 8.7 7.7 10.0 19.0 +8192 1 0.939440 1.036152 8.7 8.0 10.0 12.0 +16384 1 1.815263 2.026426 9.0 8.1 10.0 12.0 +32768 1 3.463124 3.960713 9.5 8.3 11.0 17.6 +65536 1 6.285570 7.148256 10.4 9.3 12.0 42.0 +I0924 19:17:53.753320 2234864 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +131072 1 10.181637 11.103403 12.9 11.9 15.0 18.0 +I0924 19:17:59.785759 2234866 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:17:59.785818 2234865 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +262144 1 15.148352 16.426768 17.3 16.0 19.0 22.0 +I0924 19:18:05.811924 2234868 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +I0924 19:18:05.811941 2234867 rail_monitor.cpp:83] RailMonitor: error_threshold=3 error_window=10s cooldown=30s +524288 1 21.010883 22.571327 25.0 23.4 32.0 37.0 +1048576 1 27.856722 29.536765 37.6 35.6 58.0 60.0 +2097152 1 29.720205 30.978545 70.6 67.8 99.0 103.0 +4194304 1 34.495969 35.366908 121.6 118.9 176.8 184.0 +8388608 1 37.597862 38.154500 223.1 220.4 336.0 344.0 +16777216 1 39.481337 39.819019 424.9 422.3 652.3 663.0 +33554432 1 40.062630 40.277398 837.5 834.8 1001.0 1290.0 +67108864 1 40.145419 40.301004 1671.6 1669.0 1897.1 2635.0 +TENT trace incomplete: 378892 dropped records, write_failed=0, file=/mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1695.6UKR6R/events/tent-events-slurm-h100-206-025-2234663-oNUhoa.jsonl diff --git a/experimental/mooncake-transfer-traces/scripts/analyze_traces.py b/experimental/mooncake-transfer-traces/scripts/analyze_traces.py new file mode 100644 index 0000000000..cf0d560860 --- /dev/null +++ b/experimental/mooncake-transfer-traces/scripts/analyze_traces.py @@ -0,0 +1,117 @@ +import argparse +import collections +import concurrent.futures +import json +import re +from pathlib import Path + +import orjson + + +def analyze(item: tuple[str, Path]) -> dict: + label, path = item + counts = collections.Counter() + status = collections.Counter() + transports = collections.Counter() + sizes = {} + peers = set() + masks = bytearray() + maxid = 0 + duplicates = 0 + first = None + last = None + bad_duration = 0 + footer = None + header = None + for line in path.open("rb", buffering=8 * 1024 * 1024): + r = orjson.loads(line) + event = r["event"] + if event == "trace_start": + header = r + continue + if event == "trace_end": + footer = r + continue + counts[event] += 1 + rid = r["request_id"] + maxid = max(maxid, rid) + if rid >= len(masks): + masks.extend(b"\0" * (rid + 65536 - len(masks))) + bit = 1 if event == "submit" else 2 + if masks[rid] & bit: + duplicates += 1 + masks[rid] |= bit + peers.add((r["initiator"], r["peer"], r["operation"], r["scope"])) + t = r["monotonic_ns"] + first = t if first is None else min(first, t) + last = t if last is None else max(last, t) + b = r["bytes"] + if b not in sizes: + sizes[b] = { + "submit": 0, + "complete": 0, + "duration_ns_sum": 0, + "duration_ns_max": 0, + "first_ns": t, + "last_ns": t, + } + g = sizes[b] + g["first_ns"] = min(g["first_ns"], t) + g["last_ns"] = max(g["last_ns"], t) + if event in ("submit", "complete"): + g[event] += 1 + if event == "complete": + status[r["status"]] += 1 + transports[r["transport"]] += 1 + d = r["duration_ns"] + bad_duration += int(d < 0) + g["duration_ns_sum"] += d + g["duration_ns_max"] = max(g["duration_ns_max"], d) + mask_counts = collections.Counter(masks[1 : maxid + 1]) + result = { + "label": label, + "file": path.name, + "file_bytes": path.stat().st_size, + "header": header, + "footer": footer, + "counts": dict(counts), + "status": dict(status), + "transports": dict(transports), + "peers": sorted(peers), + "max_request_id": maxid, + "matched_pairs": mask_counts[3], + "submit_only": mask_counts[1], + "complete_only": mask_counts[2], + "unobserved_ids_before_last": mask_counts[0], + "duplicate_events": duplicates, + "invalid_duration": bad_duration, + "span_seconds": (last - first) / 1e9, + "sizes": sizes, + } + assert footer and footer["written_records"] == sum(counts.values()) + print(label, "done", counts, "drops", footer["dropped_records"], flush=True) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Stream complete event files and audit retained request IDs." + ) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--events-dir", type=Path, required=True) + parser.add_argument("--workers", type=int, required=True) + args = parser.parse_args() + if args.workers < 1: + parser.error("--workers must be positive") + ROOT = args.run_dir.resolve() + items = [] + for op in ["read", "write"]: + for n in range(1, 4): + label = f"{op}-{n}" + match = re.search( + r"TENT transfer events: (.+)", (ROOT / f"{label}.log").read_text() + ) + items.append((label, args.events_dir / Path(match.group(1)).name)) + with concurrent.futures.ProcessPoolExecutor(max_workers=args.workers) as pool: + results = list(pool.map(analyze, items)) + (ROOT / "trace-analysis.json").write_text(json.dumps(results, indent=2) + "\n") diff --git a/experimental/mooncake-transfer-traces/scripts/extract_trace_windows.py b/experimental/mooncake-transfer-traces/scripts/extract_trace_windows.py new file mode 100644 index 0000000000..8b60089dd3 --- /dev/null +++ b/experimental/mooncake-transfer-traces/scripts/extract_trace_windows.py @@ -0,0 +1,112 @@ +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser( + description="Extract matched request windows from sequential single-thread traces." +) +parser.add_argument("--run-dir", type=Path, required=True) +parser.add_argument("--events-dir", type=Path, required=True) +args = parser.parse_args() +R = args.run_dir.resolve() +traces = json.loads((R / "trace-analysis.json").read_text()) + + +def window(path: Path, target: int, size: int) -> tuple[list, float]: + with path.open("rb") as f: + lo = 0 + hi = path.stat().st_size + while hi - lo > 16384: + mid = (lo + hi) // 2 + f.seek(mid) + f.readline() + line = f.readline() + if not line: + hi = mid + continue + r = json.loads(line) + if "monotonic_ns" not in r: + hi = mid + continue + if r["monotonic_ns"] < target: + lo = f.tell() + else: + hi = mid + f.seek(lo) + if lo: + f.readline() + pending = {} + result = [] + for line in f: + r = json.loads(line) + if "monotonic_ns" not in r: + continue + if r["monotonic_ns"] < target: + continue + if r["bytes"] != size: + if result: + break + continue + key = r["request_id"] + if r["event"] == "submit": + pending[key] = r + elif r["event"] == "complete" and key in pending: + s = pending.pop(key) + assert r["duration_ns"] == r["monotonic_ns"] - s["monotonic_ns"] + assert s["bytes"] == r["bytes"] and r["status"] == "completed" + result.append([key, s["monotonic_ns"], r["monotonic_ns"]]) + if len(result) == 16: + break + assert len(result) == 16, (path, size, len(result)) + origin = result[0][1] + return [ + [rid, round((start - origin) / 1000, 3), round((end - origin) / 1000, 3)] + for rid, start, end in result + ], round((origin - target) / 1e6, 3) + + +output = [] +for t in traces: + origin = min(g["first_ns"] for g in t["sizes"].values()) + phases = [] + for size, g in sorted(t["sizes"].items(), key=lambda x: int(x[0])): + size = int(size) + target = (g["first_ns"] + g["last_ns"]) // 2 + samples, offset = window(args.events_dir / t["file"], target, size) + phases.append( + { + "bytes": size, + "start": round((g["first_ns"] - origin) / 1e9, 6), + "end": round((g["last_ns"] - origin) / 1e9, 6), + "completed": g["complete"], + "mean_us": round(g["duration_ns_sum"] / g["complete"] / 1000, 3), + "observed_GBps": round( + g["complete"] * size / (g["last_ns"] - g["first_ns"]), 3 + ), + "samples": samples, + "window_offset_ms": offset, + } + ) + a = t["footer"] + output.append( + { + "run": t["label"], + "file": t["file"], + "op": t["peers"][0][2], + "loss": round( + 100 + * a["dropped_records"] + / (a["written_records"] + a["dropped_records"]), + 3, + ), + "phases": phases, + } + ) +(R / "trace-windows.json").write_text(json.dumps(output, separators=(",", ":")) + "\n") +print( + "Extracted", + sum(len(p["samples"]) for t in output for p in t["phases"]), + "validated spans across", + sum(len(t["phases"]) for t in output), + "phase windows", +) diff --git a/experimental/mooncake-transfer-traces/scripts/requirements.txt b/experimental/mooncake-transfer-traces/scripts/requirements.txt new file mode 100644 index 0000000000..bbbc2c6dc6 --- /dev/null +++ b/experimental/mooncake-transfer-traces/scripts/requirements.txt @@ -0,0 +1 @@ +orjson==3.12.0 diff --git a/experimental/mooncake-transfer-traces/validation/ctest.log b/experimental/mooncake-transfer-traces/validation/ctest.log new file mode 100644 index 0000000000..91df69856d --- /dev/null +++ b/experimental/mooncake-transfer-traces/validation/ctest.log @@ -0,0 +1,283 @@ +Start testing: Sep 24 19:03 UTC +---------------------------------------------------------- +62/113 Testing: request_merge_test +62/113 Test: request_merge_test +Command: "/mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests/request_merge_test" +Directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests +"request_merge_test" start time: Sep 24 19:03 UTC +Output: +---------------------------------------------------------- +Running main() from /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/_deps/googletest-src/googletest/src/gtest_main.cc +[==========] Running 8 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 8 tests from RequestMergeTest +[ RUN ] RequestMergeTest.KeepsRequestsSplitAcrossRegisteredBufferBoundaries +[ OK ] RequestMergeTest.KeepsRequestsSplitAcrossRegisteredBufferBoundaries (0 ms) +[ RUN ] RequestMergeTest.MergesAdjacentRequestsInsideSameRegisteredBuffers +[ OK ] RequestMergeTest.MergesAdjacentRequestsInsideSameRegisteredBuffers (0 ms) +[ RUN ] RequestMergeTest.KeepsNonContiguousRequestsSplit +[ OK ] RequestMergeTest.KeepsNonContiguousRequestsSplit (0 ms) +[ RUN ] RequestMergeTest.DoesNotMergeAcrossDifferentTransportHint +[ OK ] RequestMergeTest.DoesNotMergeAcrossDifferentTransportHint (0 ms) +[ RUN ] RequestMergeTest.MergesAdjacentRequestsWithSameTransportHint +[ OK ] RequestMergeTest.MergesAdjacentRequestsWithSameTransportHint (0 ms) +[ RUN ] RequestMergeTest.KeepsDifferentSelectionAttributesSplit +[ OK ] RequestMergeTest.KeepsDifferentSelectionAttributesSplit (0 ms) +[ RUN ] RequestMergeTest.MergesMatchingSelectionAttributes +[ OK ] RequestMergeTest.MergesMatchingSelectionAttributes (0 ms) +[ RUN ] RequestMergeTest.MixedHintBatchStillMergesPerHint +[ OK ] RequestMergeTest.MixedHintBatchStillMergesPerHint (0 ms) +[----------] 8 tests from RequestMergeTest (0 ms total) + +[----------] Global test environment tear-down +[==========] 8 tests from 1 test suite ran. (0 ms total) +[ PASSED ] 8 tests. + +Test time = 0.03 sec +---------------------------------------------------------- +Test Passed. +"request_merge_test" end time: Sep 24 19:03 UTC +"request_merge_test" time elapsed: 00:00:00 +---------------------------------------------------------- + +96/113 Testing: tent_engine_failover_e2e_test +96/113 Test: tent_engine_failover_e2e_test +Command: "/mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests/tent_engine_failover_e2e_test" +Directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests +"tent_engine_failover_e2e_test" start time: Sep 24 19:03 UTC +Output: +---------------------------------------------------------- +Running main() from /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/_deps/googletest-src/googletest/src/gtest_main.cc +[==========] Running 30 tests from 3 test suites. +[----------] Global test environment set-up. +[----------] 26 tests from EngineFailoverE2E +[ RUN ] EngineFailoverE2E.HpTcpStaleMetadataRetriesSameTransportOnce +[ OK ] EngineFailoverE2E.HpTcpStaleMetadataRetriesSameTransportOnce (39 ms) +[ RUN ] EngineFailoverE2E.HpTcpPermanentFailureDoesNotFailOver +[ OK ] EngineFailoverE2E.HpTcpPermanentFailureDoesNotFailOver (35 ms) +[ RUN ] EngineFailoverE2E.HpTcpPollErrorDoesNotInspectStaleStatusOutput +[ OK ] EngineFailoverE2E.HpTcpPollErrorDoesNotInspectStaleStatusOutput (34 ms) +[ RUN ] EngineFailoverE2E.StatusCorruptionTriggersFailoverToSecondary +[ OK ] EngineFailoverE2E.StatusCorruptionTriggersFailoverToSecondary (36 ms) +[ RUN ] EngineFailoverE2E.AutoFailoverOnPollDisabledLeavesTaskFailed +[ OK ] EngineFailoverE2E.AutoFailoverOnPollDisabledLeavesTaskFailed (34 ms) +[ RUN ] EngineFailoverE2E.AutoFailoverOnPollDisabledAppliesToVectorStatus +[ OK ] EngineFailoverE2E.AutoFailoverOnPollDisabledAppliesToVectorStatus (34 ms) +[ RUN ] EngineFailoverE2E.AutoFailoverOnPollDisabledAppliesToOverallStatus +[ OK ] EngineFailoverE2E.AutoFailoverOnPollDisabledAppliesToOverallStatus (34 ms) +[ RUN ] EngineFailoverE2E.ProgressBatchRetriesWhenPollAutoFailoverDisabled +[ OK ] EngineFailoverE2E.ProgressBatchRetriesWhenPollAutoFailoverDisabled (34 ms) +[ RUN ] EngineFailoverE2E.ProgressBatchDoesNotReviveObservedFailedTask +[ OK ] EngineFailoverE2E.ProgressBatchDoesNotReviveObservedFailedTask (34 ms) +[ RUN ] EngineFailoverE2E.ProgressBatchHonorsMaxFailoverAttemptsZero +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0924 19:03:59.487087 2229394 transfer_engine_impl.cpp:2776] Task failover limit reached (0), last transport=rdma +[ OK ] EngineFailoverE2E.ProgressBatchHonorsMaxFailoverAttemptsZero (34 ms) +[ RUN ] EngineFailoverE2E.ProgressBatchKeepsOverallPendingWithMixedOutcomes +W0924 19:03:59.521595 2229394 transfer_engine_impl.cpp:2776] Task failover limit reached (0), last transport=rdma +[ OK ] EngineFailoverE2E.ProgressBatchKeepsOverallPendingWithMixedOutcomes (34 ms) +[ RUN ] EngineFailoverE2E.OverallStatusUsesWorstFailureNotGenericFailed +[ OK ] EngineFailoverE2E.OverallStatusUsesWorstFailureNotGenericFailed (34 ms) +[ RUN ] EngineFailoverE2E.WaitTransferCompletionUsesProgressBatchWhenPollDisabled +[ OK ] EngineFailoverE2E.WaitTransferCompletionUsesProgressBatchWhenPollDisabled (34 ms) +[ RUN ] EngineFailoverE2E.TransferSyncUsesProgressBatchWhenPollDisabled +[ OK ] EngineFailoverE2E.TransferSyncUsesProgressBatchWhenPollDisabled (34 ms) +[ RUN ] EngineFailoverE2E.TransferSyncFreesBatchWhenSubmitFails +W0924 19:03:59.659814 2229394 transfer_engine_impl.cpp:2151] Unable to find registered buffer for request: opcode 1 source 0x5b228b0fb8a0 target_id 0 target_offset 0x5b228b0fb8a0 length 4096 transport_hint unspec +[ OK ] EngineFailoverE2E.TransferSyncFreesBatchWhenSubmitFails (34 ms) +[ RUN ] EngineFailoverE2E.ProgressBatchAdvancesExactlyOneStepPerCall +[ OK ] EngineFailoverE2E.ProgressBatchAdvancesExactlyOneStepPerCall (34 ms) +[ RUN ] EngineFailoverE2E.BothTransportsFailExhaustsFailoverBudget +W0924 19:03:59.730998 2229394 transfer_engine_impl.cpp:2793] No more transports available after tcp failed +[ OK ] EngineFailoverE2E.BothTransportsFailExhaustsFailoverBudget (36 ms) +[ RUN ] EngineFailoverE2E.MixedFaultsAcrossManySubmissions +[ OK ] EngineFailoverE2E.MixedFaultsAcrossManySubmissions (44 ms) +[ RUN ] EngineFailoverE2E.MaxFailoverAttemptsZeroDisablesFailover +W0924 19:03:59.810497 2229394 transfer_engine_impl.cpp:2776] Task failover limit reached (0), last transport=rdma +[ OK ] EngineFailoverE2E.MaxFailoverAttemptsZeroDisablesFailover (34 ms) +[ RUN ] EngineFailoverE2E.MaxFailoverAttemptsOneAllowsSingleFailover +[ OK ] EngineFailoverE2E.MaxFailoverAttemptsOneAllowsSingleFailover (36 ms) +[ RUN ] EngineFailoverE2E.PerTaskFailoverCountsAreIndependent +[ OK ] EngineFailoverE2E.PerTaskFailoverCountsAreIndependent (36 ms) +[ RUN ] EngineFailoverE2E.SubmitStageFailureFailsOverToSecondary +[ OK ] EngineFailoverE2E.SubmitStageFailureFailsOverToSecondary (34 ms) +[ RUN ] EngineFailoverE2E.SubmitStageFailureExhaustsBudget +[ OK ] EngineFailoverE2E.SubmitStageFailureExhaustsBudget (34 ms) +[ RUN ] EngineFailoverE2E.SubmitStageFailoverWithDerivedTasks +[ OK ] EngineFailoverE2E.SubmitStageFailoverWithDerivedTasks (34 ms) +[ RUN ] EngineFailoverE2E.QueuedPathSubmitStageFailover +[ OK ] EngineFailoverE2E.QueuedPathSubmitStageFailover (34 ms) +[ RUN ] EngineFailoverE2E.SubmitStagePartialAcceptanceExecutesExactlyOnce +[ OK ] EngineFailoverE2E.SubmitStagePartialAcceptanceExecutesExactlyOnce (34 ms) +[----------] 26 tests from EngineFailoverE2E (923 ms total) + +[----------] 2 tests from TransferStatusSeverityTest +[ RUN ] TransferStatusSeverityTest.KnownRanksMatchFormerMap +[ OK ] TransferStatusSeverityTest.KnownRanksMatchFormerMap (0 ms) +[ RUN ] TransferStatusSeverityTest.UnknownValueRanksWithFailedAndDoesNotThrow +[ OK ] TransferStatusSeverityTest.UnknownValueRanksWithFailedAndDoesNotThrow (0 ms) +[----------] 2 tests from TransferStatusSeverityTest (0 ms total) + +[----------] 2 tests from EngineFailoverE2E/PinnedFailoverPolicyTest +[ RUN ] EngineFailoverE2E/PinnedFailoverPolicyTest.PolicyIsPinnedAtSubmit/0 +W0924 19:04:00.093765 2229394 transfer_engine_impl.cpp:2776] Task failover limit reached (0), last transport=rdma +[ OK ] EngineFailoverE2E/PinnedFailoverPolicyTest.PolicyIsPinnedAtSubmit/0 (36 ms) +[ RUN ] EngineFailoverE2E/PinnedFailoverPolicyTest.PolicyIsPinnedAtSubmit/1 +[ OK ] EngineFailoverE2E/PinnedFailoverPolicyTest.PolicyIsPinnedAtSubmit/1 (36 ms) +[----------] 2 tests from EngineFailoverE2E/PinnedFailoverPolicyTest (73 ms total) + +[----------] Global test environment tear-down +[==========] 30 tests from 3 test suites ran. (996 ms total) +[ PASSED ] 30 tests. + +Test time = 1.02 sec +---------------------------------------------------------- +Test Passed. +"tent_engine_failover_e2e_test" end time: Sep 24 19:04 UTC +"tent_engine_failover_e2e_test" time elapsed: 00:00:01 +---------------------------------------------------------- + +104/113 Testing: tent_runtime_queue_dispatch_test +104/113 Test: tent_runtime_queue_dispatch_test +Command: "/mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests/tent_runtime_queue_dispatch_test" +Directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests +"tent_runtime_queue_dispatch_test" start time: Sep 24 19:04 UTC +Output: +---------------------------------------------------------- +Running main() from /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/_deps/googletest-src/googletest/src/gtest_main.cc +[==========] Running 14 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 14 tests from RuntimeQueueDispatch +[ RUN ] RuntimeQueueDispatch.RejectsOverfullBatchBeforePublishingTasks +[ OK ] RuntimeQueueDispatch.RejectsOverfullBatchBeforePublishingTasks (36 ms) +[ RUN ] RuntimeQueueDispatch.RejectsOwnerLargerThanDispatchByteWindow +[ OK ] RuntimeQueueDispatch.RejectsOwnerLargerThanDispatchByteWindow (34 ms) +[ RUN ] RuntimeQueueDispatch.RejectsEmptyDispatchWindowConfig +WARNING: Logging before InitGoogleLogging() is written to STDERR +E0924 19:04:00.232915 2229676 transfer_engine_impl.cpp:302] Failed to construct Transfer Engine instance: InvalidArgument: runtime queue dispatch window must be non-zero + Raised at /mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp:423 +E0924 19:04:00.232957 2229676 transfer_engine_impl.cpp:302] Failed to construct Transfer Engine instance: InvalidArgument: runtime queue dispatch window must be non-zero + Raised at /mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp:423 +[ OK ] RuntimeQueueDispatch.RejectsEmptyDispatchWindowConfig (0 ms) +[ RUN ] RuntimeQueueDispatch.DispatchesOnlyOneWindowOnSubmit +[ OK ] RuntimeQueueDispatch.DispatchesOnlyOneWindowOnSubmit (34 ms) +[ RUN ] RuntimeQueueDispatch.KeepsDispatchWindowUntilOwnerIsTerminal +[ OK ] RuntimeQueueDispatch.KeepsDispatchWindowUntilOwnerIsTerminal (34 ms) +[ RUN ] RuntimeQueueDispatch.CancelsQueuedOwnerWithoutDispatchingIt +[ OK ] RuntimeQueueDispatch.CancelsQueuedOwnerWithoutDispatchingIt (34 ms) +[ RUN ] RuntimeQueueDispatch.CancelsDispatchedRdmaTaskIdempotently +[ OK ] RuntimeQueueDispatch.CancelsDispatchedRdmaTaskIdempotently (34 ms) +[ RUN ] RuntimeQueueDispatch.RejectsCancellationForUnsupportedTransport +[ OK ] RuntimeQueueDispatch.RejectsCancellationForUnsupportedTransport (34 ms) +[ RUN ] RuntimeQueueDispatch.ProgressWorkerRefillsWindowFromTransportNotify +[ OK ] RuntimeQueueDispatch.ProgressWorkerRefillsWindowFromTransportNotify (36 ms) +[ RUN ] RuntimeQueueDispatch.RuntimeQueueDrainsWithoutUserPolling +[ OK ] RuntimeQueueDispatch.RuntimeQueueDrainsWithoutUserPolling (38 ms) +[ RUN ] RuntimeQueueDispatch.EarlyFreeReclaimsAfterQueuedCompletion +[ OK ] RuntimeQueueDispatch.EarlyFreeReclaimsAfterQueuedCompletion (36 ms) +[ RUN ] RuntimeQueueDispatch.PollingDerivedTaskCompletesMergedOwner +[ OK ] RuntimeQueueDispatch.PollingDerivedTaskCompletesMergedOwner (67 ms) +[ RUN ] RuntimeQueueDispatch.BatchNotificationsFollowSubmitIntervals +[ OK ] RuntimeQueueDispatch.BatchNotificationsFollowSubmitIntervals (136 ms) +[ RUN ] RuntimeQueueDispatch.TcpCapsPreservePublicMappingAndOwnerBytes +[ OK ] RuntimeQueueDispatch.TcpCapsPreservePublicMappingAndOwnerBytes (410 ms) +[----------] 14 tests from RuntimeQueueDispatch (967 ms total) + +[----------] Global test environment tear-down +[==========] 14 tests from 1 test suite ran. (967 ms total) +[ PASSED ] 14 tests. + +Test time = 1.00 sec +---------------------------------------------------------- +Test Passed. +"tent_runtime_queue_dispatch_test" end time: Sep 24 19:04 UTC +"tent_runtime_queue_dispatch_test" time elapsed: 00:00:00 +---------------------------------------------------------- + +106/113 Testing: causal_chain_test +106/113 Test: causal_chain_test +Command: "/mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests/causal_chain_test" +Directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests +"causal_chain_test" start time: Sep 24 19:04 UTC +Output: +---------------------------------------------------------- +Running main() from /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/_deps/googletest-src/googletest/src/gtest_main.cc +[==========] Running 3 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 3 tests from CausalChain +[ RUN ] CausalChain.TimestampsPopulatedOnQueuePath +[ OK ] CausalChain.TimestampsPopulatedOnQueuePath (35 ms) +[ RUN ] CausalChain.MultipleTransfersAllComplete +[ OK ] CausalChain.MultipleTransfersAllComplete (34 ms) +[ RUN ] CausalChain.TimestampsPopulatedOnDirectPath +[ OK ] CausalChain.TimestampsPopulatedOnDirectPath (34 ms) +[----------] 3 tests from CausalChain (105 ms total) + +[----------] Global test environment tear-down +[==========] 3 tests from 1 test suite ran. (105 ms total) +[ PASSED ] 3 tests. + +Test time = 0.13 sec +---------------------------------------------------------- +Test Passed. +"causal_chain_test" end time: Sep 24 19:04 UTC +"causal_chain_test" time elapsed: 00:00:00 +---------------------------------------------------------- + +109/113 Testing: tent_transfer_event_recorder_test +109/113 Test: tent_transfer_event_recorder_test +Command: "/mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests/tent_transfer_event_recorder_test" +Directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/mooncake-transfer-engine/tent/tests +"tent_transfer_event_recorder_test" start time: Sep 24 19:04 UTC +Output: +---------------------------------------------------------- +Running main() from /mnt/home/kimbo/networkx/Mooncake/build/trace-tests/_deps/googletest-src/googletest/src/gtest_main.cc +[==========] Running 9 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 9 tests from TransferEventTest +[ RUN ] TransferEventTest.DisabledAndInvalidConfiguration +TENT trace disabled: invalid TENT_TRACE_BUFFER_RECORDS +TENT trace disabled: cannot create /tmp/tent-event-test-ZkjS9Y/missing/tent-events-slurm-h100-206-025-2230001-XXXXXX.jsonl +[ OK ] TransferEventTest.DisabledAndInvalidConfiguration (0 ms) +[ RUN ] TransferEventTest.ConcurrentWritersAndDuplicateCompletion +TENT transfer events: /tmp/tent-event-test-aBrzbM/tent-events-slurm-h100-206-025-2230001-BF5M7g.jsonl +[ OK ] TransferEventTest.ConcurrentWritersAndDuplicateCompletion (10 ms) +[ RUN ] TransferEventTest.OverflowIsExplicitAndFilesAreUnique +TENT transfer events: /tmp/tent-event-test-UmRKA2/tent-events-slurm-h100-206-025-2230001-DaJcsS.jsonl +TENT transfer events: /tmp/tent-event-test-UmRKA2/tent-events-slurm-h100-206-025-2230001-uaAzbI.jsonl +TENT trace incomplete: 4 dropped records, write_failed=0, file=/tmp/tent-event-test-UmRKA2/tent-events-slurm-h100-206-025-2230001-DaJcsS.jsonl +[ OK ] TransferEventTest.OverflowIsExplicitAndFilesAreUnique (0 ms) +[ RUN ] TransferEventTest.MergedDescriptorsAndAppendKeepOriginalSizes +TENT transfer events: /tmp/tent-event-test-zeNDDA/tent-events-slurm-h100-206-025-2230001-nrAL6C.jsonl +[ OK ] TransferEventTest.MergedDescriptorsAndAppendKeepOriginalSizes (36 ms) +[ RUN ] TransferEventTest.QueueCompletionAndQueuedCancellation +TENT transfer events: /tmp/tent-event-test-AVTWxP/tent-events-slurm-h100-206-025-2230001-nZsXWC.jsonl +[ OK ] TransferEventTest.QueueCompletionAndQueuedCancellation (34 ms) +[ RUN ] TransferEventTest.SubmitFailureIsRecordedWithoutMetrics +TENT transfer events: /tmp/tent-event-test-gaXZOJ/tent-events-slurm-h100-206-025-2230001-FwvnWE.jsonl +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0924 19:04:01.407263 2230001 transfer_engine_impl.cpp:2793] No more transports available after rdma failed +[ OK ] TransferEventTest.SubmitFailureIsRecordedWithoutMetrics (35 ms) +[ RUN ] TransferEventTest.RejectionAndUnobservedTeardown +TENT transfer events: /tmp/tent-event-test-dem2yK/tent-events-slurm-h100-206-025-2230001-LvOXIa.jsonl +[ OK ] TransferEventTest.RejectionAndUnobservedTeardown (34 ms) +[ RUN ] TransferEventTest.QueuedSubmitFailureRecordedOnce +TENT transfer events: /tmp/tent-event-test-1IYgVi/tent-events-slurm-h100-206-025-2230001-26bfBU.jsonl +[ OK ] TransferEventTest.QueuedSubmitFailureRecordedOnce (34 ms) +[ RUN ] TransferEventTest.FailoverKeepsOneLogicalRequest +TENT transfer events: /tmp/tent-event-test-ufMe9o/tent-events-slurm-h100-206-025-2230001-m4KSTA.jsonl +[ OK ] TransferEventTest.FailoverKeepsOneLogicalRequest (34 ms) +[----------] 9 tests from TransferEventTest (221 ms total) + +[----------] Global test environment tear-down +[==========] 9 tests from 1 test suite ran. (221 ms total) +[ PASSED ] 9 tests. + +Test time = 0.25 sec +---------------------------------------------------------- +Test Passed. +"tent_transfer_event_recorder_test" end time: Sep 24 19:04 UTC +"tent_transfer_event_recorder_test" time elapsed: 00:00:00 +---------------------------------------------------------- + +End testing: Sep 24 19:04 UTC diff --git a/experimental/mooncake-transfer-traces/validation/package-checks.txt b/experimental/mooncake-transfer-traces/validation/package-checks.txt new file mode 100644 index 0000000000..abbde2e063 --- /dev/null +++ b/experimental/mooncake-transfer-traces/validation/package-checks.txt @@ -0,0 +1,4 @@ +All 3 patches applied to base fe0d23e332f41c7d54b5c217194dce5f8e751b21; all 17 resulting files matched the tested working-tree hashes. +Packaged streaming audit reproduced 2 matched pairs, 1 submit-only ID, 1 complete-only ID, and 1 wholly absent ID on each controlled fixture. +Packaged extractor returned the expected 16 mid-phase spans (IDs 33–48, each 6 microseconds) from complete sequential fixtures. +README local links and bilingual command blocks checked. diff --git a/experimental/mooncake-transfer-traces/validation/rdma-smoke-job-1693.log b/experimental/mooncake-transfer-traces/validation/rdma-smoke-job-1693.log new file mode 100644 index 0000000000..c5a15281ad --- /dev/null +++ b/experimental/mooncake-transfer-traces/validation/rdma-smoke-job-1693.log @@ -0,0 +1,7 @@ +Results: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1693.zlYY3h +Initiator: slurm-h100-206-025; target: slurm-h100-206-035; RDMA VRAM +Compute-node container build log: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1693.zlYY3h/build.log +Transfer event traces: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1693.zlYY3h/events +Sweep: read, repeat 1/1, 8388608..8388608 bytes +Sweep: write, repeat 1/1, 8388608..8388608 bytes +Completed all sweeps. JSON results and logs: /mnt/home/kimbo/networkx/Mooncake/benchmark-results/tebench-1693.zlYY3h diff --git a/experimental/mooncake-transfer-traces/validation/regression-job-1694.log b/experimental/mooncake-transfer-traces/validation/regression-job-1694.log new file mode 100644 index 0000000000..4d869874e7 --- /dev/null +++ b/experimental/mooncake-transfer-traces/validation/regression-job-1694.log @@ -0,0 +1,197 @@ +-- glog version '0.4.0' (< 0.6.0 or unknown): IsGoogleLoggingInitialized() unavailable +CMAKE_BUILD_TYPE: Release +-- [limit_jobs] Available memory: 2020066 MB, CPU cores: 128 +-- [limit_jobs] Compile jobs: 128 (1500 MB/job), Link jobs: 128 (4000 MB/job) +-- [limit_jobs] Hint: use -G Ninja for automatic compile/link parallelism separation +-- [limit_jobs] With Make, recommend: cmake --build . -j128 +-- Event-driven completion is disabled +-- None of USE_ETCD, USE_REDIS, USE_HTTP is selected, only "P2PHANDSHAKE" is supported as metadata server +-- metrics is enabled +-- -------------YLT INSTALL SETTING------------ +-- INSTALL_THIRDPARTY: ON +-- INSTALL_STANDALONE: ON +-- INSTALL_INDEPENDENT_THIRDPARTY: ON +-- INSTALL_INDEPENDENT_STANDALONE: ON +-- -------------------------------------------- +-- -------------YLT CONFIG SETTING------------- +YLT is imported by fetchContent or add_subdirectory +-- YLT_ENABLE_DL: ON +-- YLT_ENABLE_SSL: OFF +-- YLT_ENABLE_NTLS: OFF +-- Enable ibverbs support +-- YLT_ENABLE_PMR: OFF +-- YLT_ENABLE_IO_URING: OFF +-- YLT_ENABLE_STRUCT_PACK_UNPORTABLE_TYPE: OFF +-- YLT_ENABLE_STRUCT_PACK_OPTIMIZE: OFF +-- -------------------------------------------- +-- pybind11 v2.13.6 +-- USE_NOF=OFF, NoF SSD pool support is disabled +-- CUDA: Disabled +-- ROCm/HIP: Disabled +-- GDS: Disabled +-- RDMA: Enabled +-- Uring: Enabled +-- Could NOT find CURL (missing: CURL_LIBRARY CURL_INCLUDE_DIR) +-- Http metastore support: Disabled +-- Redis metastore support: Disabled +-- TENT metrics: DISABLED (zero overhead) +/usr/local/lib/python3.10/dist-packages +-- CUDA: Disabled +-- ROCm/HIP: Disabled +-- Sunrise/Tang: Disabled +-- Mooncake Python install directory: /usr/local/lib/python3.10/dist-packages/mooncake +-- Configuring done +-- Generating done +-- Build files have been written to: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests +Consolidate compiler generated dependencies of target asio_shared +Consolidate compiler generated dependencies of target gtest +[ 4%] Built target gtest +Consolidate compiler generated dependencies of target gtest_main +[ 4%] Built target gtest_main +[ 4%] Built target asio_shared +Consolidate compiler generated dependencies of target mooncake_common +Consolidate compiler generated dependencies of target tent_common +[ 9%] Built target tent_common +Consolidate compiler generated dependencies of target tent_rpc +Consolidate compiler generated dependencies of target tent_platform_all +Consolidate compiler generated dependencies of target tent_xport_uring +Consolidate compiler generated dependencies of target tent_xport_bufio +Consolidate compiler generated dependencies of target tent_metrics +[ 13%] Built target tent_platform_all +Consolidate compiler generated dependencies of target tent_xport_hp_tcp_core +Consolidate compiler generated dependencies of target tent_xport_rdma +[ 18%] Built target tent_metrics +[ 22%] Built target mooncake_common +[ 31%] Built target tent_xport_uring +[ 31%] Built target tent_xport_bufio +[ 31%] Built target tent_rpc +[ 36%] Built target tent_xport_hp_tcp_core +[ 50%] Built target tent_xport_rdma +Consolidate compiler generated dependencies of target tent_xport_tcp +Consolidate compiler generated dependencies of target tent_xport_shm +Consolidate compiler generated dependencies of target tent_xport_mpcomm +Consolidate compiler generated dependencies of target tent_xport_hp_tcp +[ 54%] Built target tent_xport_tcp +[ 54%] Built target tent_xport_shm +[ 68%] Built target tent_xport_hp_tcp +[ 68%] Built target tent_xport_mpcomm +Consolidate compiler generated dependencies of target tent_runtime +[ 68%] Building CXX object mooncake-transfer-engine/tent/src/runtime/CMakeFiles/tent_runtime.dir/transfer_engine_impl.cpp.o +[ 68%] Building CXX object mooncake-transfer-engine/tent/src/runtime/CMakeFiles/tent_runtime.dir/transfer_event_recorder.cpp.o +[ 95%] Built target tent_runtime +Consolidate compiler generated dependencies of target tent +[ 95%] Linking CXX static library libtent.a +[100%] Built target tent +Consolidate compiler generated dependencies of target tent_transfer_event_recorder_test +[100%] Building CXX object mooncake-transfer-engine/tent/tests/CMakeFiles/tent_transfer_event_recorder_test.dir/transfer_event_recorder_test.cpp.o +/mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp: In member function ‘virtual void mooncake::tent::{anonymous}::TransferEventTest_FailoverKeepsOneLogicalRequest_Test::TestBody()’: +/mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/tests/transfer_event_recorder_test.cpp:505:12: warning: suggest explicit braces to avoid ambiguous ‘else’ [-Wdangling-else] + 505 | if (row["event"] == "complete") EXPECT_EQ(row["transport"], "tcp"); + | ^ +[100%] Linking CXX executable tent_transfer_event_recorder_test +[100%] Built target tent_transfer_event_recorder_test +[ 4%] Built target gtest +[ 4%] Built target asio_shared +[ 4%] Built target gtest_main +[ 13%] Built target tent_common +[ 13%] Built target mooncake_common +[ 21%] Built target tent_platform_all +[ 21%] Built target tent_metrics +[ 26%] Built target tent_xport_uring +[ 30%] Built target tent_xport_bufio +[ 30%] Built target tent_rpc +[ 34%] Built target tent_xport_hp_tcp_core +[ 47%] Built target tent_xport_rdma +[ 52%] Built target tent_xport_shm +[ 52%] Built target tent_xport_tcp +[ 65%] Built target tent_xport_mpcomm +[ 65%] Built target tent_xport_hp_tcp +Consolidate compiler generated dependencies of target tent_runtime +[ 91%] Built target tent_runtime +[ 95%] Built target tent +[100%] Building CXX object mooncake-transfer-engine/tent/tests/CMakeFiles/request_merge_test.dir/request_merge_test.cpp.o +/mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/tests/request_merge_test.cpp: In function ‘mooncake::tent::Request mooncake::tent::{anonymous}::makeWriteRequest(char*, size_t, uint64_t, size_t)’: +/mnt/home/kimbo/networkx/Mooncake/mooncake-transfer-engine/tent/tests/request_merge_test.cpp:45:5: warning: missing initializer for member ‘mooncake::tent::Request::policy_name’ [-Wmissing-field-initializers] + 45 | }; + | ^ +[100%] Linking CXX executable request_merge_test +[100%] Built target request_merge_test +[ 0%] Built target asio_shared +[ 4%] Built target gtest +[ 4%] Built target gtest_main +[ 13%] Built target tent_common +[ 13%] Built target mooncake_common +[ 27%] Built target tent_rpc +[ 27%] Built target tent_platform_all +[ 31%] Built target tent_xport_uring +[ 36%] Built target tent_xport_bufio +[ 31%] Built target tent_metrics +[ 40%] Built target tent_xport_hp_tcp_core +[ 50%] Built target tent_xport_rdma +[ 54%] Built target tent_xport_tcp +[ 54%] Built target tent_xport_shm +[ 63%] Built target tent_xport_mpcomm +[ 68%] Built target tent_xport_hp_tcp +[ 95%] Built target tent_runtime +[100%] Built target tent +[100%] Building CXX object mooncake-transfer-engine/tent/tests/CMakeFiles/causal_chain_test.dir/causal_chain_test.cpp.o +[100%] Linking CXX executable causal_chain_test +[100%] Built target causal_chain_test +[ 4%] Built target gtest +[ 4%] Built target asio_shared +[ 4%] Built target gtest_main +[ 9%] Built target tent_common +[ 13%] Built target mooncake_common +[ 27%] Built target tent_xport_uring +[ 27%] Built target tent_platform_all +[ 27%] Built target tent_xport_bufio +[ 27%] Built target tent_rpc +[ 40%] Built target tent_xport_rdma +[ 45%] Built target tent_metrics +[ 45%] Built target tent_xport_tcp +[ 54%] Built target tent_xport_shm +[ 59%] Built target tent_xport_mpcomm +[ 63%] Built target tent_xport_hp_tcp_core +[ 68%] Built target tent_xport_hp_tcp +[ 95%] Built target tent_runtime +[100%] Built target tent +[100%] Building CXX object mooncake-transfer-engine/tent/tests/CMakeFiles/tent_runtime_queue_dispatch_test.dir/runtime_queue_dispatch_test.cpp.o +[100%] Linking CXX executable tent_runtime_queue_dispatch_test +[100%] Built target tent_runtime_queue_dispatch_test +[ 0%] Built target asio_shared +[ 4%] Built target gtest +[ 9%] Built target tent_common +[ 9%] Built target gtest_main +[ 13%] Built target mooncake_common +[ 18%] Built target tent_rpc +[ 27%] Built target tent_xport_bufio +[ 27%] Built target tent_xport_uring +[ 27%] Built target tent_platform_all +[ 31%] Built target tent_metrics +[ 36%] Built target tent_xport_hp_tcp_core +[ 50%] Built target tent_xport_tcp +[ 50%] Built target tent_xport_shm +[ 54%] Built target tent_xport_rdma +[ 63%] Built target tent_xport_mpcomm +[ 68%] Built target tent_xport_hp_tcp +[ 95%] Built target tent_runtime +[100%] Built target tent +[100%] Building CXX object mooncake-transfer-engine/tent/tests/CMakeFiles/tent_engine_failover_e2e_test.dir/engine_failover_e2e_test.cpp.o +[100%] Linking CXX executable tent_engine_failover_e2e_test +[100%] Built target tent_engine_failover_e2e_test +Internal ctest changing into directory: /mnt/home/kimbo/networkx/Mooncake/build/trace-tests +Test project /mnt/home/kimbo/networkx/Mooncake/build/trace-tests + Start 62: request_merge_test +1/5 Test #62: request_merge_test .................. Passed 0.03 sec + Start 96: tent_engine_failover_e2e_test +2/5 Test #96: tent_engine_failover_e2e_test ....... Passed 1.02 sec + Start 104: tent_runtime_queue_dispatch_test +3/5 Test #104: tent_runtime_queue_dispatch_test .... Passed 1.00 sec + Start 106: causal_chain_test +4/5 Test #106: causal_chain_test ................... Passed 0.13 sec + Start 109: tent_transfer_event_recorder_test +5/5 Test #109: tent_transfer_event_recorder_test ... Passed 0.25 sec + +100% tests passed, 0 tests failed out of 5 + +Total Test time (real) = 2.52 sec