From 42d7be171a8c9e05c2d17667f93be590e99040ff Mon Sep 17 00:00:00 2001 From: Renggeng Zheng Date: Mon, 30 Mar 2026 11:17:10 -0400 Subject: [PATCH 01/19] correlation files init --- notebooks/astrasim2_correlation/correlation.ipynb | 0 notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml | 0 notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml | 0 notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 notebooks/astrasim2_correlation/correlation.ipynb create mode 100644 notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml create mode 100644 notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml create mode 100644 notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml b/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml b/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml b/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml new file mode 100644 index 00000000..e69de29b From 05407f20774b1adc3732c64730221867a8485b5e Mon Sep 17 00:00:00 2001 From: Renggeng Zheng Date: Mon, 30 Mar 2026 11:35:50 -0400 Subject: [PATCH 02/19] provisional workload and mapping for all_reduce --- .../mapping/all_reduce.mapping.yaml | 34 +++++++++++++++++++ .../mapping/all_reduce.workload.yaml | 15 ++++++++ 2 files changed, 49 insertions(+) diff --git a/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml b/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml index e69de29b..5be5aae6 100644 --- a/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml +++ b/notebooks/astrasim2_correlation/mapping/all_reduce.mapping.yaml @@ -0,0 +1,34 @@ +mapping: + nodes: + - !Storage + tensors: [I, S] + component: Network + - !Spatial + rank_variable: nodes + # bound is nodes + tile_shape: {{ nodes }} + - !Storage + tensors: [S, G] + component: NV100 + - !Sequential + nodes: + - !Nested + nodes: + - !Temporal + rank_variable: vec_len + # Bound is vec_len. + tile_shape: {{ vec_len }} + component: MAC + - !Compute + einsum: scatter + component: MAC + - !Gather + nodes: + - !Temporal + rank_variable: vec_len + # Bound is vec_len. + tile_shape: {{ vec_len }} + component: MAC + - !Compute + einsum: all_gather + component: MAC diff --git a/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml b/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml index e69de29b..d00907d5 100644 --- a/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml +++ b/notebooks/astrasim2_correlation/mapping/all_reduce.workload.yaml @@ -0,0 +1,15 @@ +workload: + iteration_space_shape: + vec_len: 0 <= vec_len < {{ collective_size / (data_width * nodes) }} + nodes: 0 <= nodes < {{ nodes }} + einsums: + - name: scatter + tensor_accesses: + - {name: I, projection: [ vec_len, nodes ]} + - {name: W_scatter, projection: [ vec_len, nodes ]} + - {name: S, projection: [ nodes ], output: True} + - name: all_gather + tensor_accesses: + - {name: S, projection: [nodes]} + - {name: W_gather, projection: [vec_len. nodes]} + - {name: G, projection: [vec_len, nodes]} From 70eade11c78809af5aefb7c85e6925bee63b30e6 Mon Sep 17 00:00:00 2001 From: Renggeng Zheng Date: Mon, 30 Mar 2026 11:38:33 -0400 Subject: [PATCH 03/19] starting off with nvdla --- .../mapping/nvidiav100.arch.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml b/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml index e69de29b..c5894704 100644 --- a/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml +++ b/notebooks/astrasim2_correlation/mapping/nvidiav100.arch.yaml @@ -0,0 +1,44 @@ +# Based on examples/arches/nvdla.yaml +arch: + nodes: + - !Memory + name: MainMemory + size: inf + leak_power: 0 + actions: + # Energy is upper end of the range from the TPU paper. The lower end came from + # their reference, and they said it left out some things. Latency is 38.4 GB/s. + # DDR5-4800. Chip runs at at 1GHz, so divide to get per-cycle bandwidth. + # https://www.jedec.org/news/pressreleases/jedec-updates-standard-low-power-memory-devices-lpddr5 + - {name: read, energy: 7.03e-12, latency: 1 / (8 * 38.4e9)} + - {name: write, energy: 7.03e-12, latency: 1 / (8 * 38.4e9)} + tensors: {keep: ~Intermediates, may_keep: All} + + - !Memory + name: GlobalBuffer + size: 1024*64*8 # 64 kB + total_latency: max(read_latency, write_latency) # Separate ports + leak_power: 0 + actions: + # 512 GB/s read, 128 GB/s write + - {name: read, energy: 0.249e-12, latency: 1 / 512e9 / 8} + - {name: write, energy: 0.293e-12, latency: 1 / 128e9 / 8} + tensors: {keep: All} + + - !Container + name: ProcessingElement + spatial: + - {name: reuse_input, fanout: 32, may_reuse: input, reuse: input, min_usage: 1} + - {name: reuse_output, fanout: 192, may_reuse: output, reuse: output, min_usage: 1} + + - !Memory + name: Register + size: weight.bits_per_value if weight else 1 + component_class: Dummy + tensors: {keep: weight} + + - !Compute + name: MAC + leak_power: 0 + actions: + - {name: compute, energy: 0.084e-12, latency: 1 / 1e9} From afe95abfaa6896727af9c6e1e3d6fe61a29d15c1 Mon Sep 17 00:00:00 2001 From: Renggeng Zheng Date: Mon, 30 Mar 2026 11:45:53 -0400 Subject: [PATCH 04/19] prima facie v100_torus items --- .../mapping/all_reduce.binding.yaml | 0 .../mapping/v100_torus.arch.yaml | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 notebooks/astrasim2_correlation/mapping/all_reduce.binding.yaml create mode 100644 notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml diff --git a/notebooks/astrasim2_correlation/mapping/all_reduce.binding.yaml b/notebooks/astrasim2_correlation/mapping/all_reduce.binding.yaml new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml b/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml new file mode 100644 index 00000000..06c90fe7 --- /dev/null +++ b/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml @@ -0,0 +1,32 @@ +# Based on examples/arches/nvdla.yaml +arch: + nodes: + - !Memory + name: Network + size: inf + leak_power: 0 + actions: + # TODO: Get the network energies for NVLink. + # 150 GB/s NVLink transfer speeds. + - {name: read, energy: 7.03e-12, latency: 1 / (150e9 * 8)} + - {name: write, energy: 7.03e-12, latency: 1 / (150e9 * 8)} + tensors: {keep: All, may_keep: All} + + - !Memory + name: NV100 + size: 16*1024*1024*8 # 16 GB + total_latency: max(read_latency, write_latency) # Separate ports + leak_power: 0 + actions: + # Prima facie network analysis assuming NV100 mem hierarchy characteristics + # are dominated by the NVLink connection. + - {name: read, energy: 0, latency: 0} + - {name: write, energy: 0, latency: 0} + tensors: {keep: All} + + - !Compute + name: MAC + leak_power: 0 + actions: + # Prima facie assume compute is dominated by NVLink data mvmt. + - {name: compute, energy: 0, latency: 0} From 8f3bb4512b22676cb0f8a4facca8da5d25c42955 Mon Sep 17 00:00:00 2001 From: Renggeng Zheng Date: Wed, 8 Apr 2026 15:13:25 -0400 Subject: [PATCH 05/19] my correl stuff --- .../astrasim2_correlation/correlation.ipynb | 99 +++++++++++++++++++ .../mapping/v100_torus.arch.yaml | 4 +- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index e69de29b..7915e160 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -0,0 +1,99 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3aa122f8", + "metadata": {}, + "source": [ + "# Network Sim Correlation\n", + "\n", + "The following are attempts to correlate the network model in accelforge to actual networks measured in real architectures." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3a04b7c", + "metadata": {}, + "outputs": [], + "source": [ + "import accelforge as af\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "id": "95cf92d9", + "metadata": {}, + "source": [ + "## Astrasim-2.0\n", + "\n", + "The [Astrasim-2.0 paper](https://arxiv.org/abs/2303.14006) has correlation to a torus of 4 and 16 V100s on page 6 to the network latency of data transfers of an all-reduce.\n", + "\n", + "On top of this, on page 9 they have data on their simulation framework correlated to the above and the latencies of certain operation of their model.\n", + "\n", + "We aim to show that we can match these numbers with our analytical model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f70059c4", + "metadata": {}, + "outputs": [], + "source": [ + "from numbers import Number\n", + "from typing import Dict, Sequence\n", + "\n", + "def ring_4xV100():\n", + "\t\"\"\"\n", + "\tGenerates a graph with the latencies of 4xV100s.\n", + "\t\"\"\"\n", + "\t# Collective size in MB to latency TODO: stop eyeballing latency.\n", + "\tGROUND: Dict[int, int] = {\n", + "\t\t64: 500,\n", + "\t\t96: 750,\n", + "\t\t128: 1000,\n", + "\t\t192: 2000,\n", + "\t\t750: 10_000,\n", + "\t\t1500: 20_000\n", + "\t}\n", + "\n", + "\tsize: Sequence[Number] = tuple(GROUND.keys())\n", + "\ttruth: Sequence[Number] = tuple(GROUND.values())\n", + "\testimate: Sequence[Number] = []\n", + "\n", + "\t# Generates estimates from model\n", + "\tfor mb in size:\n", + "\t\t# TODO: Read and Jinja2 these items.\n", + "\t\t\n", + "\n", + "\tx = np.arange(len(size)) # positions\n", + "\twidth = 0.35 # bar width\n", + "\n", + "\tfig, ax = plt.subplots()\n", + "\t# Position bars side-by-side using offset\n", + "\tax.bar(x - width/2, men, width, label='Ground', color=\"blue\")\n", + "\tax.bar(x + width/2, women, width, label='Model', color=\"red\")\n", + "\n", + "\tax.set_xticks(x) # Center labels\n", + "\tax.set_xticklabels(labels)\n", + "\tax.legend()\n", + "\tplt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "accelforge", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml b/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml index 06c90fe7..96d60735 100644 --- a/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml +++ b/notebooks/astrasim2_correlation/mapping/v100_torus.arch.yaml @@ -8,8 +8,8 @@ arch: actions: # TODO: Get the network energies for NVLink. # 150 GB/s NVLink transfer speeds. - - {name: read, energy: 7.03e-12, latency: 1 / (150e9 * 8)} - - {name: write, energy: 7.03e-12, latency: 1 / (150e9 * 8)} + - {name: read, energy: 0, latency: 1 / (150e9 * 8)} + - {name: write, energy: 0, latency: 1 / (150e9 * 8)} tensors: {keep: All, may_keep: All} - !Memory From c4945e157a25f2045c92248d6f6ea4804efe5f04 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Thu, 11 Jun 2026 13:00:15 -0400 Subject: [PATCH 06/19] accelforge islpy-barvinok fix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c3b0ec2b..c5eccfde 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "pydantic_core>=2.33.0", "ruamel.yaml>=0.18.0", "jinja2>=3.1.0", - "islpy-barvinok == 2025.2.5", + "islpy-barvinok >= 2025.2.5", "sympy>=1.14.0", "symengine>=0.14.0", "paretoset>=1.2.5", From 7099e75e9ffb211c5edac9e787c46c7fda97693c Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Thu, 11 Jun 2026 14:33:24 -0400 Subject: [PATCH 07/19] all_to_all model implementation first approximation in ISL --- .../_looptree/reuse/isl/distributed/bind.py | 2 +- .../astrasim2_correlation/correlation.ipynb | 55 +------- .../distribuffers/multicast/test_cases.yaml | 119 +++++++++++++++++- .../{ => distribuffers}/test_multicast.py | 0 tests/not_working/distribuffers/util.py | 56 +++++++++ 5 files changed, 176 insertions(+), 56 deletions(-) rename tests/not_working/{ => distribuffers}/test_multicast.py (100%) create mode 100644 tests/not_working/distribuffers/util.py diff --git a/accelforge/model/_looptree/reuse/isl/distributed/bind.py b/accelforge/model/_looptree/reuse/isl/distributed/bind.py index 7e5f000d..37346659 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/bind.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/bind.py @@ -1,6 +1,6 @@ """Applies the binding layer into one that can be used for later analysis,""" -from accelforge.frontend.binding import Binding +from accelforge.frontend._binding import Binding from accelforge.frontend.mapping import Mapping from accelforge.frontend.workload import Workload diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index 7915e160..2848fc58 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -26,60 +26,7 @@ "id": "95cf92d9", "metadata": {}, "source": [ - "## Astrasim-2.0\n", - "\n", - "The [Astrasim-2.0 paper](https://arxiv.org/abs/2303.14006) has correlation to a torus of 4 and 16 V100s on page 6 to the network latency of data transfers of an all-reduce.\n", - "\n", - "On top of this, on page 9 they have data on their simulation framework correlated to the above and the latencies of certain operation of their model.\n", - "\n", - "We aim to show that we can match these numbers with our analytical model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f70059c4", - "metadata": {}, - "outputs": [], - "source": [ - "from numbers import Number\n", - "from typing import Dict, Sequence\n", - "\n", - "def ring_4xV100():\n", - "\t\"\"\"\n", - "\tGenerates a graph with the latencies of 4xV100s.\n", - "\t\"\"\"\n", - "\t# Collective size in MB to latency TODO: stop eyeballing latency.\n", - "\tGROUND: Dict[int, int] = {\n", - "\t\t64: 500,\n", - "\t\t96: 750,\n", - "\t\t128: 1000,\n", - "\t\t192: 2000,\n", - "\t\t750: 10_000,\n", - "\t\t1500: 20_000\n", - "\t}\n", - "\n", - "\tsize: Sequence[Number] = tuple(GROUND.keys())\n", - "\ttruth: Sequence[Number] = tuple(GROUND.values())\n", - "\testimate: Sequence[Number] = []\n", - "\n", - "\t# Generates estimates from model\n", - "\tfor mb in size:\n", - "\t\t# TODO: Read and Jinja2 these items.\n", - "\t\t\n", - "\n", - "\tx = np.arange(len(size)) # positions\n", - "\twidth = 0.35 # bar width\n", - "\n", - "\tfig, ax = plt.subplots()\n", - "\t# Position bars side-by-side using offset\n", - "\tax.bar(x - width/2, men, width, label='Ground', color=\"blue\")\n", - "\tax.bar(x + width/2, women, width, label='Model', color=\"red\")\n", - "\n", - "\tax.set_xticks(x) # Center labels\n", - "\tax.set_xticklabels(labels)\n", - "\tax.legend()\n", - "\tplt.show()" + "## We are testing an 8 GPU All-to-All to Correlate Later, simulating an NVLink Switch\n" ] } ], diff --git a/tests/not_working/distribuffers/multicast/test_cases.yaml b/tests/not_working/distribuffers/multicast/test_cases.yaml index 32dbc8eb..00afc2e0 100755 --- a/tests/not_working/distribuffers/multicast/test_cases.yaml +++ b/tests/not_working/distribuffers/multicast/test_cases.yaml @@ -479,4 +479,121 @@ # dist_fn: *ring_dist_size_8 # expected: # latency: 1 -# total_hops: 4 \ No newline at end of file +# total_hops: 4 +################################################### +# 8-GPU fully-connected (NVLink/NVSwitch-style) # +# all-to-all, one-hot GPU encoding. # +# # +# GPU i sits at one-hot coordinate e_i so every # +# src!=dst cast has extent 1 along exactly the # +# src and dst dims: cost = (1+1)(1+1)-1 = 3, # +# uniform across all pairs (fully-connected). # +# Self-chunks never cross the fabric (cost 0). # +# dist_fn is unit-cost (matching only). # +################################################### + +# All-to-all over 8 GPUs: each GPU holds chunks data[self, d] and requests data[s, self]. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and 0 <= d < 8 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and 0 <= s < 8 + } + dims: &8d_onehot_spatial + - type: Spatial + spatial_dim: 0 + target: 0 + - type: Spatial + spatial_dim: 1 + target: 0 + - type: Spatial + spatial_dim: 2 + target: 0 + - type: Spatial + spatial_dim: 3 + target: 0 + - type: Spatial + spatial_dim: 4 + target: 0 + - type: Spatial + spatial_dim: 5 + target: 0 + - type: Spatial + spatial_dim: 6 + target: 0 + - type: Spatial + spatial_dim: 7 + target: 0 + dist_fn: &fully_connected_unit | + { + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[0] : + xd0 = xs0 and xd1 = xs1 and xd2 = xs2 and xd3 = xs3 and xd4 = xs4 and xd5 = xs5 and xd6 = xs6 and xd7 = xs7; + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[1] : + (xd0 < xs0) or (xd0 > xs0) or (xd1 < xs1) or (xd1 > xs1) or (xd2 < xs2) or (xd2 > xs2) or (xd3 < xs3) or (xd3 > xs3) or (xd4 < xs4) or (xd4 > xs4) or (xd5 < xs5) or (xd5 > xs5) or (xd6 < xs6) or (xd6 > xs6) or (xd7 < xs7) or (xd7 > xs7) + } + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 168 + extent_DOR_hops: null + +# Single chunk GPU0 -> GPU3: one unicast cast, cost (1+1)(1+1)-1 = 3. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 0 and d = 3 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 0 and d = 3 + } + dims: *8d_onehot_spatial + + dist_fn: *fully_connected_unit + + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 3 + extent_DOR_hops: null + +# Self chunk GPU5 -> GPU5: never crosses the fabric, cost 0. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 5 and d = 5 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 5 and d = 5 + } + dims: *8d_onehot_spatial + + dist_fn: *fully_connected_unit + + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 0 + extent_DOR_hops: null diff --git a/tests/not_working/test_multicast.py b/tests/not_working/distribuffers/test_multicast.py similarity index 100% rename from tests/not_working/test_multicast.py rename to tests/not_working/distribuffers/test_multicast.py diff --git a/tests/not_working/distribuffers/util.py b/tests/not_working/distribuffers/util.py new file mode 100644 index 00000000..bcb86ffa --- /dev/null +++ b/tests/not_working/distribuffers/util.py @@ -0,0 +1,56 @@ +""" +Utility functions common to testing the isl mapper functions. +""" + +from pathlib import Path +import islpy as isl + +from ruamel.yaml import YAML + + +def to_isl_maps(obj: str | list | dict) -> dict: + """ + Given an object, attempt to reduce all strings in tree with isl.Map + + Parameters + ---------- + obj: + A DAG which can be explored and contains isl.Map strings within it. + + Returns + ------- + `obj` but all strings are converted to isl.Map. + """ + + def _to_isl_maps(obj: str | dict | list) -> isl.Map | dict | list: + """Recursively convert string ISL maps to isl.Map; leave others alone.""" + if isinstance(obj, str): + return isl.Map.read_from_str(isl.DEFAULT_CONTEXT, obj) + if isinstance(obj, dict): + return {k: (_to_isl_maps(v) if k != "type" else v) for k, v in obj.items()} + if isinstance(obj, list): + return [_to_isl_maps(v) for v in obj] + return obj + + return _to_isl_maps(obj) # type: ignore + + +def load_solutions(path: Path) -> dict: + """ + Loads in a dictionary with the isl solutions to a workload problem. + + Parameters + ---------- + path: + The path to the solutions. + + Returns + ------- + A dictionary relating Python-based keys generated by the mapper (e.g., + `BufferTensorEinsum` to their corresponding isl.Map.) + """ + # Load expected solutions (YAML file with string ISL maps) + yaml: YAML = YAML(typ="safe") + + with open(path, "r", encoding="utf-8") as f: + return to_isl_maps(yaml.load(f)) From 785e9c75453d414421bd3beb44b67f5530dc1b14 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Thu, 11 Jun 2026 14:57:50 -0400 Subject: [PATCH 08/19] good motivation to fix hypercube, we are overestimating hops 3x in all-to-all due to extent formulation --- .../astrasim2_correlation/correlation.ipynb | 567 +++++++++++++++++- 1 file changed, 564 insertions(+), 3 deletions(-) diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index 2848fc58..047f3531 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -12,9 +12,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "e3a04b7c", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:47.222682Z", + "iopub.status.busy": "2026-06-11T18:41:47.222573Z", + "iopub.status.idle": "2026-06-11T18:41:48.167242Z", + "shell.execute_reply": "2026-06-11T18:41:48.166287Z" + } + }, "outputs": [], "source": [ "import accelforge as af\n", @@ -28,6 +35,552 @@ "source": [ "## We are testing an 8 GPU All-to-All to Correlate Later, simulating an NVLink Switch\n" ] + }, + { + "cell_type": "markdown", + "id": "195ecb91", + "metadata": {}, + "source": [ + "### Encoding: one-hot GPU coordinates on a fully-connected fabric\n", + "\n", + "GPU $i$ sits at one-hot coordinate $e_i$ of an $N$-dimensional `noc` space. `data[s, d]` is the chunk sent by GPU $s$ to GPU $d$: each GPU *holds* the chunks it sends (occupancy) and *requests* the chunks addressed to it (fill).\n", + "\n", + "Every $s \\neq d$ cast then has extent 1 along exactly the src and dst dimensions, so `HypercubeMulticastModel`'s bounding-box cost is $(1+1)(1+1)-1 = 3$ **uniformly for every pair** — the fully-connected property (no pair privileged). The distance function is unit-cost (0 if same GPU, 1 otherwise) and only influences source matching in `identify_mesh_casts`. Self-chunks match at distance 0 and cost 0 hops: they never cross the fabric." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c5b2c7bf", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:48.169250Z", + "iopub.status.busy": "2026-06-11T18:41:48.169045Z", + "iopub.status.idle": "2026-06-11T18:41:48.177262Z", + "shell.execute_reply": "2026-06-11T18:41:48.176277Z" + } + }, + "outputs": [], + "source": [ + "import math\n", + "\n", + "import islpy as isl\n", + "import pandas as pd\n", + "\n", + "from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import (\n", + " HypercubeMulticastModel,\n", + ")\n", + "from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import (\n", + " Fill,\n", + " Occupancy,\n", + " SpatialTag,\n", + ")\n", + "\n", + "CTX = isl.DEFAULT_CONTEXT" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e343cac3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:48.178504Z", + "iopub.status.busy": "2026-06-11T18:41:48.178391Z", + "iopub.status.idle": "2026-06-11T18:41:48.183047Z", + "shell.execute_reply": "2026-06-11T18:41:48.182522Z" + } + }, + "outputs": [], + "source": [ + "def onehot_constraints(prefix: str, n: int) -> str:\n", + " \"\"\"One-hot constraints over dims ``{prefix}0..{prefix}{n-1}``.\"\"\"\n", + " bounds = \" and \".join(f\"0 <= {prefix}{i} <= 1\" for i in range(n))\n", + " hot = \" + \".join(f\"{prefix}{i}\" for i in range(n)) + \" = 1\"\n", + " return f\"{bounds} and {hot}\"\n", + "\n", + "\n", + "def linear_id(prefix: str, n: int) -> str:\n", + " \"\"\"Affine recovery of the GPU id from a one-hot vector: id = sum i*g_i.\"\"\"\n", + " return \" + \".join(f\"{i}*{prefix}{i}\" for i in range(1, n))\n", + "\n", + "\n", + "def all_to_all_maps(n: int) -> tuple[isl.Map, isl.Map, isl.Map]:\n", + " \"\"\"Build (occupancy, fill, dist_fn) for an N-GPU fully-connected all-to-all.\n", + "\n", + " data[s, d] is the chunk sent by GPU s to GPU d. Each GPU holds the\n", + " chunks it sends (occ) and requests the chunks addressed to it (fill).\n", + " \"\"\"\n", + " gs = \", \".join(f\"gs{i}\" for i in range(n))\n", + " gd = \", \".join(f\"gd{i}\" for i in range(n))\n", + " occ = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gs}] -> data[s, d] : {onehot_constraints('gs', n)} \"\n", + " f\"and s = {linear_id('gs', n)} and 0 <= d < {n} }}\",\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gd}] -> data[s, d] : {onehot_constraints('gd', n)} \"\n", + " f\"and d = {linear_id('gd', n)} and 0 <= s < {n} }}\",\n", + " )\n", + "\n", + " xd = \", \".join(f\"xd{i}\" for i in range(n))\n", + " xs = \", \".join(f\"xs{i}\" for i in range(n))\n", + " same = \" and \".join(f\"xd{i} = xs{i}\" for i in range(n))\n", + " diff = \" or \".join(f\"(xd{i} < xs{i}) or (xd{i} > xs{i})\" for i in range(n))\n", + " dist_fn = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ [noc[{xd}] -> noc[{xs}]] -> hops[0] : {same}; \"\n", + " f\" [noc[{xd}] -> noc[{xs}]] -> hops[1] : {diff} }}\",\n", + " )\n", + " return occ, fill, dist_fn" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a2a2df5f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:48.184434Z", + "iopub.status.busy": "2026-06-11T18:41:48.184302Z", + "iopub.status.idle": "2026-06-11T18:41:48.188107Z", + "shell.execute_reply": "2026-06-11T18:41:48.187196Z" + } + }, + "outputs": [], + "source": [ + "def eval_total(poly: isl.PwQPolynomial) -> int:\n", + " \"\"\"Evaluate a fully-summed PwQPolynomial (zero-dimensional domain).\"\"\"\n", + " return int(poly.eval(isl.Point.zero(poly.domain().get_space())).to_python())\n", + "\n", + "\n", + "def model_hops_per_element(n: int) -> int:\n", + " \"\"\"Run the real HypercubeMulticastModel on the all-to-all at V=1.\"\"\"\n", + " occ_map, fill_map, dist_fn = all_to_all_maps(n)\n", + " tags = [SpatialTag(i, 0) for i in range(n)]\n", + " model = HypercubeMulticastModel(dist_fn)\n", + " info = model.apply(0, Fill(tags, fill_map), Occupancy(tags, occ_map))\n", + " return eval_total(info.hops)\n", + "\n", + "\n", + "def single_chunk_hops(n: int, src: int, dst: int) -> int:\n", + " \"\"\"Hops for one (src, dst) chunk — uniformity / self-chunk probe.\"\"\"\n", + " occ_map, fill_map, dist_fn = all_to_all_maps(n)\n", + " chunk = isl.Set.read_from_str(CTX, f\"{{ data[{src}, {dst}] }}\")\n", + " tags = [SpatialTag(i, 0) for i in range(n)]\n", + " model = HypercubeMulticastModel(dist_fn)\n", + " info = model.apply(\n", + " 0,\n", + " Fill(tags, fill_map.intersect_range(chunk)),\n", + " Occupancy(tags, occ_map.intersect_range(chunk)),\n", + " )\n", + " return eval_total(info.hops)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "06d40ac3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:48.189890Z", + "iopub.status.busy": "2026-06-11T18:41:48.189717Z", + "iopub.status.idle": "2026-06-11T18:41:48.193103Z", + "shell.execute_reply": "2026-06-11T18:41:48.192370Z" + } + }, + "outputs": [], + "source": [ + "NODES = 8 # GPUs on the fabric\n", + "LINK_BW_GBPS = 150.0 # per-GPU per-direction NVLink bandwidth, GB/s (V100 generation)\n", + "ALPHA_S = 0.0 # per-operation latency overhead, seconds\n", + "MIN_MIB, MAX_MIB = 1, 1024 # collective size sweep bounds, powers of 2\n", + "PER_RANK = False # interpret swept sizes as per-rank NCCL sizes instead of totals" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "60364739", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:48.194288Z", + "iopub.status.busy": "2026-06-11T18:41:48.194178Z", + "iopub.status.idle": "2026-06-11T18:41:49.888588Z", + "shell.execute_reply": "2026-06-11T18:41:49.888067Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ISL network model: 8-GPU fully-connected all-to-all (one-hot)\n", + " hops/element : 168 (uniform 3 per pair)\n", + " fabric-crossing chunks: 56 of 64\n" + ] + } + ], + "source": [ + "n = NODES\n", + "\n", + "# --- Run the tool at V=1 and validate the fully-connected structure. ---\n", + "hops_per_elem = model_hops_per_element(n)\n", + "expected = 3 * n * (n - 1)\n", + "assert hops_per_elem == expected, (\n", + " f\"model returned {hops_per_elem} hops/element, expected {expected} \"\n", + " f\"(= 3 per off-diagonal pair x {n*(n-1)} pairs)\"\n", + ")\n", + "probe_off = single_chunk_hops(n, 0, min(3, n - 1))\n", + "probe_self = single_chunk_hops(n, n - 1, n - 1)\n", + "assert probe_off == 3, f\"off-diagonal chunk cost {probe_off} != 3\"\n", + "assert probe_self == 0, f\"self chunk cost {probe_self} != 0\"\n", + "crossing_chunks = hops_per_elem // 3 # = N(N-1) fabric-crossing chunks\n", + "\n", + "print(f\"ISL network model: {n}-GPU fully-connected all-to-all (one-hot)\")\n", + "print(f\" hops/element : {hops_per_elem} (uniform 3 per pair)\")\n", + "print(f\" fabric-crossing chunks: {crossing_chunks} of {n*n}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c89a8e17", + "metadata": {}, + "source": [ + "### Size and latency conventions\n", + "\n", + "`collective_size` $S$ = **total** bytes moved by the collective across all ranks (ASTRA-sim style). Per-rank NCCL buffer size = $S/N$; each $(s, d)$ pair exchanges a chunk of $S/N^2$ bytes. Set `PER_RANK = True` to interpret the swept sizes as per-rank NCCL sizes instead.\n", + "\n", + "Latency derives from the model's hop count: hops are uniform 3 per fabric-crossing chunk, so crossing chunks/element $= \\mathrm{hops}/3 = N(N-1)$, i.e. each GPU receives $N-1$ chunks $\\Rightarrow$ per-GPU wire bytes $= S(N-1)/N^2$. With a non-blocking switch and full-duplex ports (send overlaps receive), port drain time gives\n", + "\n", + "$$ t = \\alpha + \\frac{S\\,(N-1)}{N^2 \\cdot BW} $$\n", + "\n", + "NCCL conventions: $\\mathrm{algbw} = (S/N)/t$, $\\mathrm{busbw} = \\mathrm{algbw} \\cdot (N-1)/N$. With $\\alpha = 0$, busbw $\\equiv$ BW exactly — used as a per-row sanity invariant below.\n", + "\n", + "**Correlation notes** — this is a flat bandwidth model with no launch/protocol overhead, so expect empirical underprediction at small sizes; calibrate `ALPHA_S` (per-operation latency) and `LINK_BW_GBPS` (effective port bandwidth) against nccl-tests `alltoall_perf`, and overlay the ASTRA-sim 2.0 (FullyConnected) and EC2 series on the plot below." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6d5e39c9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:49.890046Z", + "iopub.status.busy": "2026-06-11T18:41:49.889928Z", + "iopub.status.idle": "2026-06-11T18:41:49.909301Z", + "shell.execute_reply": "2026-06-11T18:41:49.908706Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodescollective_size_bytesper_rank_bytesper_gpu_wire_byteshops_per_elementfabric_chunks_per_elementmodel_latency_salgbw_GBpsbusbw_GBps
081048576131072114688168567.645867e-07171.428571150.0
182097152262144229376168561.529173e-06171.428571150.0
284194304524288458752168563.058347e-06171.428571150.0
3883886081048576917504168566.116693e-06171.428571150.0
481677721620971521835008168561.223339e-05171.428571150.0
583355443241943043670016168562.446677e-05171.428571150.0
686710886483886087340032168564.893355e-05171.428571150.0
781342177281677721614680064168569.786709e-05171.428571150.0
882684354563355443229360128168561.957342e-04171.428571150.0
985368709126710886458720256168563.914684e-04171.428571150.0
1081073741824134217728117440512168567.829367e-04171.428571150.0
\n", + "
" + ], + "text/plain": [ + " nodes collective_size_bytes per_rank_bytes per_gpu_wire_bytes \\\n", + "0 8 1048576 131072 114688 \n", + "1 8 2097152 262144 229376 \n", + "2 8 4194304 524288 458752 \n", + "3 8 8388608 1048576 917504 \n", + "4 8 16777216 2097152 1835008 \n", + "5 8 33554432 4194304 3670016 \n", + "6 8 67108864 8388608 7340032 \n", + "7 8 134217728 16777216 14680064 \n", + "8 8 268435456 33554432 29360128 \n", + "9 8 536870912 67108864 58720256 \n", + "10 8 1073741824 134217728 117440512 \n", + "\n", + " hops_per_element fabric_chunks_per_element model_latency_s algbw_GBps \\\n", + "0 168 56 7.645867e-07 171.428571 \n", + "1 168 56 1.529173e-06 171.428571 \n", + "2 168 56 3.058347e-06 171.428571 \n", + "3 168 56 6.116693e-06 171.428571 \n", + "4 168 56 1.223339e-05 171.428571 \n", + "5 168 56 2.446677e-05 171.428571 \n", + "6 168 56 4.893355e-05 171.428571 \n", + "7 168 56 9.786709e-05 171.428571 \n", + "8 168 56 1.957342e-04 171.428571 \n", + "9 168 56 3.914684e-04 171.428571 \n", + "10 168 56 7.829367e-04 171.428571 \n", + "\n", + " busbw_GBps \n", + "0 150.0 \n", + "1 150.0 \n", + "2 150.0 \n", + "3 150.0 \n", + "4 150.0 \n", + "5 150.0 \n", + "6 150.0 \n", + "7 150.0 \n", + "8 150.0 \n", + "9 150.0 \n", + "10 150.0 " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bw = LINK_BW_GBPS * 1e9 # bytes/s\n", + "\n", + "rows = []\n", + "mib = MIN_MIB\n", + "while mib <= MAX_MIB:\n", + " size = mib * (1 << 20)\n", + " total = size * n if PER_RANK else size\n", + " per_rank = total / n\n", + " chunk_bytes = total / (n * n)\n", + " # Latency derived from the model output: hops/3 crossing chunks\n", + " # spread evenly, (N-1) into each GPU's port.\n", + " per_gpu_wire = (crossing_chunks / n) * chunk_bytes\n", + " t = ALPHA_S + per_gpu_wire / bw\n", + " # Closed form cross-check: t = alpha + S(N-1)/(N^2 BW)\n", + " assert math.isclose(t, ALPHA_S + total * (n - 1) / (n * n * bw), rel_tol=1e-12)\n", + " algbw = per_rank / t / 1e9\n", + " busbw = algbw * (n - 1) / n\n", + " rows.append((n, int(total), int(per_rank), int(per_gpu_wire),\n", + " hops_per_elem, crossing_chunks, t, algbw, busbw))\n", + " mib *= 2\n", + "\n", + "sweep = pd.DataFrame(rows, columns=[\n", + " \"nodes\", \"collective_size_bytes\", \"per_rank_bytes\", \"per_gpu_wire_bytes\",\n", + " \"hops_per_element\", \"fabric_chunks_per_element\",\n", + " \"model_latency_s\", \"algbw_GBps\", \"busbw_GBps\",\n", + "])\n", + "sweep" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cbdaefb9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:49.910815Z", + "iopub.status.busy": "2026-06-11T18:41:49.910686Z", + "iopub.status.idle": "2026-06-11T18:41:49.913503Z", + "shell.execute_reply": "2026-06-11T18:41:49.913097Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "busbw invariant holds: busbw == 150 GB/s on every row\n" + ] + } + ], + "source": [ + "if ALPHA_S == 0:\n", + " assert (sweep[\"busbw_GBps\"] - LINK_BW_GBPS).abs().max() < 1e-9 * LINK_BW_GBPS\n", + " print(f\"busbw invariant holds: busbw == {LINK_BW_GBPS:g} GB/s on every row\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "11751aab", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-11T18:41:49.914710Z", + "iopub.status.busy": "2026-06-11T18:41:49.914592Z", + "iopub.status.idle": "2026-06-11T18:41:50.198575Z", + "shell.execute_reply": "2026-06-11T18:41:50.197933Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAHbCAYAAACQmw0xAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQAAmoVJREFUeJzs3XlcVFX/B/DPzLArDLKjIpCYgiiYuOWGiqKZZu5tmtoq5kKb/ipxqSzNtYcy60lLK81My7Xcl9RcccNdXFKBUWRfZ+b+/qCZh3FYZoM7A5/36+Wr5t5z7jl35svly517zpEIgiCAiIiIiKyKVOwOEBEREZE+JmlEREREVohJGhEREZEVYpJGREREZIWYpBERERFZISZpRERERFaISRoRERGRFWKSRkRERGSFmKQRERERWSEmaWRRSqUS77zzDgICAiCVSjFo0CCj6s+YMQMSiURnW1BQEF588UXLdZIsorzPylqZ01dN3Xv37lm4V/+zZ88eSCQS7NmzR7vtxRdfRFBQULW1aUnm9l+tViM8PBwfffRR9XSQ9ERHRyM8PFzsbphkxYoVkEgkuH79utF1H74WlJSUICAgAF988YUFe2g5TNJq0OXLlzFy5Eg0btwYLi4uaNGiBWbNmoX8/HyjjpOSkoIJEybg0UcfhYuLC1xcXBAWFoa4uDicPn1ap6wmIDX/NGXff/99ZGdn65Wr6BdReHg4oqOjq+zbt99+i3nz5mHo0KH47rvvMGXKFKPOjSzrzp07mDFjBpKSkkTrw4svvgiJRILWrVujvFXoJBIJJkyYAABYsGABJBIJduzYUeHxvv76a0gkEvz+++8AbPuXjSVYw2dsrp9++gm3bt3SxgHwv1/Ex44d0yl74MAB9OvXD40aNYKTkxOaNGmCAQMG4Mcff9QpVzauxPbFF19gxYoVYneDymFvb4/4+Hh89NFHKCwsFLs7epik1ZBbt26hffv2OHz4MCZMmIBFixahU6dOSEhIwDPPPGPwcTZt2oTw8HCsXLkSMTExWLhwIRYvXox+/fphy5YtiIyMxI0bN/Tqffnll1i5ciUWLFiAFi1a4KOPPkLfvn3L/aVpjl27dqFRo0ZYuHAhXnjhBXTv3t2ixyfj3LlzBzNnzrSKX+BnzpzBr7/+WmmZkSNHQiqV6v3CLevHH3+Ep6cn+vXrZ3Db77//PgoKCgwub0us6TM21bx58zBy5EjI5fJKy61duxbdunVDWloaJk2ahM8//xzPP/88Hjx4gK+//rqGems8JmnWbcyYMbh3716l1x2x2Indgbpi5cqVyMzMxIEDB9CyZUsAwCuvvAK1Wo3vv/8eDx48QIMGDSo9xtWrVzFy5EgEBgZi586d8Pf319n/6aef4osvvoBUqp97Dx06FF5eXgCA1157DUOGDMGvv/6Kw4cPo1OnThY6SyA9PR3u7u4WOx7VDs7OzggICMCsWbMwePDgCr96bNiwIXr06IFff/0VX375JRwdHXX23759G/v27cMrr7wCe3t7g9u3s7ODnR0vd9bo5MmTOHXqFObPn19l2RkzZiAsLAyHDx+Gg4ODzr709PTq6mKtkpeXh3r16ondDavi7u6OPn36YMWKFRg7dqzY3dHBO2k1RPPVoq+vr852f39/SKVSvQtOeebOnYu8vDwsX75cL0EDSn8RTZw4EQEBAVUeq2fPngBKvzq1hOvXr0MikWD37t04d+6c9uvVPXv2lPu8Stk6xvyFee3aNUgkEixcuFBv38GDByGRSPDTTz9VeZytW7eie/fucHV1hZubG9q1a6f3V9TatWvRtm1bODs7w8vLC88//zxu376tU+bFF19E/fr1cfv2bQwaNAj169eHt7c33nrrLahUKr1z/eyzz7Bs2TI0bdoUjo6OaNeuHY4eParXvwsXLmDo0KHw8PCAk5MToqKitF/vlZWZmYkpU6YgKCgIjo6OaNy4MUaNGoV79+5hz549aNeuHYDSvxQ1n0nZ9/vvv/9G3759IZfL4eLigu7du+Ovv/7Sa+fAgQNo164dnJyc0LRpU3z11VdVvsdlSaVSvP/++zh9+jTWr19fadnnn38eWVlZ2Lx5s96+1atXQ61W47nnnjOq/fKeSdN8HbZhwwaEh4fD0dERLVu2xLZt26o83o0bNxASEoLw8HCkpaVVWm78+PFo3rw5nJ2d4enpiWHDhpn0LE15DPmMDYljsfoPABs2bICDgwO6detWZdmrV6+iXbt25V4vfXx8LNIfQ3+mgdJn6RYtWoSWLVvCyckJvr6+ePXVV/HgwQNtmaCgIJw7dw579+7Vfj7R0dHIzMyETCbDkiVLtGXv3bsHqVQKT09PnW85Xn/9dfj5+em0bcz16erVq3jiiSfg6upa6c/On3/+CRcXFzzzzDNQKpUVltM8YnD69Gl0794dLi4uCAkJwS+//AIA2Lt3Lzp06ABnZ2c0b9683McXTp48iX79+sHNzQ3169dHr169cPjwYb1y586dQ8+ePeHs7IzGjRvjww8/hFqtLrdfW7duRdeuXVGvXj24urqif//+OHfuXIXnUVbv3r1x4MABZGRkGFS+pjBJqyGa57nGjRuHpKQk3Lp1C2vWrMGXX36JiRMnGvSXzaZNmxASEoIOHTqY3Z+rV68CADw9Pc0+FgB4e3tj5cqVaNGiBRo3boyVK1di5cqVCA0NtcjxNR555BF07twZP/zwg96+H374Aa6urnjqqacqPcaKFSvQv39/ZGRkYNq0afjkk08QGRmp88t5xYoVGD58OGQyGebMmYOXX34Zv/76K7p06YLMzEyd46lUKsTGxsLT0xOfffYZunfvjvnz52PZsmV6bf/444+YN28eXn31VXz44Ye4fv06Bg8ejJKSEm2Zc+fOoWPHjjh//jymTp2K+fPno169ehg0aJBOgpObm4uuXbvi888/R58+fbB48WK89tpruHDhAv755x+EhoZi1qxZAErv2mo+E80vw127dqFbt27Izs5GQkICPv74Y2RmZqJnz544cuSItp0zZ86gT58+SE9Px4wZMzBmzBgkJCRUmWw97Nlnn0WzZs0wa9asSr9mHzx4MJycnMr96uHHH39EYGAgOnfubFTbFTlw4ADGjx+PkSNHYu7cuSgsLMSQIUNw//79CutcvXoV3bp1g6urK/bs2aP3h1dZR48excGDBzFy5EgsWbIEr732Gnbu3Ino6Gijn0UtT1WfsTFxLEb/gdI/rsLDww26M6r5FuGff/6xSNsVMfRn+tVXX8Xbb7+Nzp07Y/HixRgzZgx++OEHxMbGan+mFy1ahMaNG6NFixbaz+e9996Du7s7wsPDsW/fPu3xDhw4AIlEgoyMDCQnJ2u379+/H127dtW+NuZzVSqViI2NhY+PDz777DMMGTKk3HPetGkTBg4ciGHDhmHVqlVV3nl+8OABnnzySXTo0AFz586Fo6MjRo4ciTVr1mDkyJF44okn8MknnyAvLw9Dhw5FTk6Otu65c+fQtWtXnDp1Cu+88w4++OADpKSkIDo6Gn///be2XGpqKnr06IGkpCRMnToVkydPxvfff4/Fixfr9WflypXo378/6tevj08//RQffPABkpOT0aVLF4P+qGjbti0EQcDBgwerLFujBKoxs2fPFpydnQUA2n/vvfeeQXWzsrIEAMKgQYP09j148EBQKBTaf/n5+dp9CQkJAgDh4sWLgkKhEFJSUoSvvvpKcHR0FHx9fYW8vDydcgqFotz2W7ZsKXTv3r3Kfnbv3l1o2bKlzrbdu3cLAITdu3frbE9JSREACMuXL9frb1mBgYHC6NGjta+/+uorAYBw/vx57bbi4mLBy8tLp1x5MjMzBVdXV6FDhw5CQUGBzj61Wq09lo+PjxAeHq5TZtOmTQIAYfr06dpto0ePFgAIs2bN0jlWmzZthLZt2+qdq6enp5CRkaHd/ttvvwkAhI0bN2q39erVS2jVqpVQWFio07fHH39caNasmXbb9OnTBQDCr7/+qneemnM5evSo3nus2d+sWTMhNjZWW1YQBCE/P18IDg4Wevfurd02aNAgwcnJSbhx44Z2W3JysiCTyfQ+q/KMHj1aqFevniAIgvDdd9/p9RmAEBcXp1Nn2LBhgpOTk5CVlaXdduHCBQGAMG3aNJ2y5cXcw8qLKwCCg4ODcOXKFe22U6dOCQCEzz//XK+uQqEQzp8/LzRs2FBo166dzudYkbI/ixqHDh0SAAjff/+9dlt5PyOjR48WAgMDq2yjos/YmDgWs/+NGzcWhgwZord9+fLlAgDh6NGj2m3//e9/tZ9bjx49hA8++EDYv3+/oFKp9OqXF1eGMPRnev/+/QIA4YcfftApt23bNr3tFV0/4+LiBF9fX+3r+Ph4oVu3boKPj4/w5ZdfCoIgCPfv3xckEomwePFiQRBMuz5NnTpVr+2yPzfr1q0T7O3thZdffrnc97K8ugCEH3/8UbtN8/MplUqFw4cPa7f/8ccfevE5aNAgwcHBQbh69ap22507dwRXV1ehW7du2m2TJ08WAAh///23dlt6erogl8sFAEJKSoogCIKQk5MjuLu7Cy+//LJOP1NTUwW5XK6zvbxrgaZ9AMKnn35a5fnXJN5Jq0FBQUHo1q0bli1bhnXr1mHs2LH4+OOP8Z///KfKupqvS+vXr6+3Lzo6Gt7e3tp/iYmJemWaN28Ob29vBAcH49VXX0VISAg2b94MFxcX80+shg0fPhxOTk46d9P++OMP3Lt3D88//3yldbdv346cnBxMnToVTk5OOvs0X4cdO3YM6enpGD9+vE6Z/v37o0WLFuV+Dffaa6/pvO7atSuuXbumV27EiBE6zx5q/jrWlM3IyMCuXbswfPhw5OTk4N69e7h37x7u37+P2NhYXL58WfuVxrp16xAREYGnn35ar52qpptISkrC5cuX8eyzz+L+/fvadvLy8tCrVy/s27cParUaKpUKf/zxBwYNGoQmTZpo64eGhiI2NrbSNsrz3HPPGXQ37fnnn0dhYaHOQAPNnTVjv+qsTExMDJo2bap93bp1a7i5uZX72Z09exbdu3dHUFAQduzYUeUzpEDps3gaJSUluH//PkJCQuDu7o4TJ05Y5iQqYEocP6wm+n///n2D3ksAGDt2LLZt24bo6GgcOHAAs2fPRteuXdGsWTOL3wGp6md67dq1kMvl6N27t/bn5969e2jbti3q16+P3bt3V9lG165dkZaWhosXLwIovWPWrVs3dO3aFfv37wdQendNEATttcKUz/X111+vsA8//fQTRowYgVdffRVfffVVuc80l6d+/foYOXKk9nXz5s3h7u6O0NBQnW97NP+vee9UKhX+/PNPDBo0CI888oi2nL+/P5599lkcOHBA+/tuy5Yt6NixI9q3b68t5+3trXcN2L59OzIzM/HMM8/ofBYymQwdOnQw6LPQxGB1TrVjCj5JW0NWr16NV155BZcuXULjxo0BlH6to1ar8e677+KZZ56Bp6cnMjIyUFxcrK3n7OwMuVwOV1dXAKVfcT3sq6++Qk5ODtLS0ipMUtatWwc3NzfY29ujcePGOr+YDGUtc2K5u7trh9zPnj0bQOlXnY0aNdI+a5ebm6vzXslkMnh7e2u/5q1sygbN6NjmzZvr7WvRogUOHDigs83JyQne3t462xo0aKDzXIpG2URHUw6AtuyVK1cgCAI++OADfPDBB+X2Lz09HY0aNcLVq1cr/OqiKpcvXwYAjB49usIyWVlZKCoqQkFBAZo1a6a3v3nz5tiyZYtR7cpkMrz//vsYPXo0NmzYUG6CCQD9+vWDh4cHfvzxR+0ceT/99BMiIiK0A28s4eHPA6j4sxswYAB8fX3xxx9/lPvHUnkKCgowZ84cLF++HLdv39ZJTLOysozqq0Kh0Hkmqn79+pX2w9A4VqlUUCgUOvs9PDzg4OBg0f5XprKE/WGxsbGIjY1Ffn4+jh8/jjVr1mDp0qV48sknceHCBYs8m2bIz/Tly5eRlZVVYXuGDGTQJF779+9H48aNcfLkSXz44Yfw9vbGZ599pt3n5uaGiIgIAMZfn+zs7LS/cx6WkpKC559/HsOGDcPnn39eZX/Laty4sd7vBLlcrvdMtGbErua9UygUyM/PL7f/oaGhUKvVuHXrFlq2bIkbN26U+3jPw3U11zPN9f9hbm5uVZ6PJgat5fecBpO0GvLFF1+gTZs2ej8sAwcOxIoVK3Dy5EnExMRg8ODB2Lt3r3b/6NGjsWLFCsjlcvj7++Ps2bN6x9YEcWXfu3fr1k07urM8mr/IKpqmID8/X+/Ok6EqCvqHH8I1xqhRo7B27VocPHgQrVq1wu+//47x48dr/wr87LPPMHPmTG35wMBAiz7sXJZMJjO7rOYCoXkg9q233qrwTlVISIiRPdSnaWfevHmIjIwst0z9+vVRVFRkdlsPe+655zB79mzMmjWrwsmO7e3tMXz4cHz99ddIS0vDzZs3cfnyZcydO9eifanq8yhryJAh+O677/DDDz/g1VdfNej4b7zxBpYvX47JkyejU6dOkMvlkEgkGDlyZIUPP1ekXbt2OtPrJCQkYMaMGUYdozy3bt1CcHCwzrbdu3cjOjraov2viKenZ7lJcVVcXFzQtWtXdO3aFV5eXpg5cya2bt1a6R8ehjLkZ1qtVsPHx6fc52MB6CV55WnYsCGCg4Oxb98+BAUFQRAEdOrUCd7e3pg0aRJu3LiB/fv34/HHHzf4DtfDHB0dK6zr7+8Pf39/bNmyBceOHUNUVJTBx63oPTLmZ8pSNLG4cuVKvQEWAAwa2a2Jwcp+T4qBSVoNSUtLK/eWvubhUs1Imvnz5+tcsBo2bKj9//79++Obb77BkSNHdG7/WkJgYCAA4OLFi3p/CeXn5+PWrVvo06ePScfWnPfDD7SWN5+bofr27Qtvb2/88MMP6NChA/Lz8/HCCy9o948aNQpdunTRvtZ8baO5g3j27NkKk52y78XDf5ldvHhRu786aG7/29vbIyYmptKyTZs2LTdpL6uiBFnzPri5uVXajre3N5ydnbV/qZal+YrGWJq7aS+++CJ+++23Css999xzWLp0KdasWYOUlBRIJBKj5hS0tHnz5sHOzg7jx4+Hq6srnn322Srr/PLLLxg9erTO9BKFhYUGPbT/sB9++EHnjyhNrFT0GRsax35+fti+fbvOfs1dG0v2vyItWrQwe5S5Jrm4e/euJbpkkKZNm2LHjh3o3LmzztfC5ans7kzXrl2xb98+BAcHIzIyEq6uroiIiIBcLse2bdtw4sQJvT84Actcn5ycnLBp0yb07NkTffv2xd69ey16p7o83t7ecHFxKff6ceHCBUilUu3voMDAQIOuPZrrmY+PT5XXzYpoYtDSg93MxWfSasijjz6KkydP4tKlSzrbf/rpJ0ilUrRu3RpA6QiTmJgY7b+wsDBt2XfeeQcuLi4YO3ZsucP+zflLpVevXnBwcMCXX36p9xfysmXLoFQqjZo8tKzAwEDIZDKdUUwAzFqGw87ODs888wx+/vlnrFixAq1atdK+h0DpL7Cy76NmNGCfPn3g6uqKOXPm6M0urXn/oqKi4OPjg6VLl+rcSdq6dSvOnz+P/v37m9zvqvj4+CA6OhpfffVVub9wyn4tNWTIEJw6darcUZaac9GMGn74l2rbtm3RtGlTfPbZZ+V+ha5pRyaTITY2Fhs2bMDNmze1+8+fP48//vjD+BP81/PPP4+QkBCdXz4P69y5M4KCgrBq1SqsWbMG3bt3r/Brm5ogkUiwbNkyDB06FKNHjy53SpSHyWQyvZ/Lzz//3KS7yJ07d9aJaU2SVtFnbGgcOzk56Rw3JiZG+4eVJftfkU6dOuHs2bMG3bXduXNnuds1X7uX9xVadRk+fDhUKpX2kYuylEqlzudRr169ChPbrl274vr161izZo3260+pVIrHH38cCxYsQElJic7ITktfn+RyOf744w/4+Pigd+/e2kdCqotMJkOfPn3w22+/6Xy7kZaWhh9//BFdunTRfj35xBNP4PDhwzqjzRUKhd7dy9jYWLi5ueHjjz/WGSlftk5Vjh8/DolEYtF5Qy2Bd9JqyNtvv62dw2XChAnw9PTEpk2bsHXrVrz00ks6d8wq0qxZM/z444945pln0Lx5czz33HOIiIiAIAhISUnBjz/+CKlUatIvMh8fH0yfPh3vv/8+unXrhoEDB8LFxQUHDx7ETz/9hD59+mDAgAGmnDrkcrn2mQeJRIKmTZti06ZNZk8+OWrUKCxZsgS7d+/Gp59+alAdNzc3LFy4EC+99BLatWuHZ599Fg0aNMCpU6eQn5+P7777Dvb29vj0008xZswYdO/eHc888wzS0tKwePFiBAUFVftSV4mJiejSpQtatWqFl19+GY888gjS0tJw6NAh/PPPPzh16hSA0pj65ZdfMGzYMIwdOxZt27ZFRkYGfv/9dyxduhQRERFo2rQp3N3dsXTpUri6uqJevXro0KEDgoOD8c0336Bfv35o2bIlxowZg0aNGuH27dvYvXs33NzcsHHjRgDAzJkzsW3bNnTt2hXjx4+HUqnE559/jpYtW+otQ2YomUyG9957D2PGjKmwjEQiwbPPPouPP/4YALRTTZRHoVDgww8/1NseHBxs0YEGUqkUq1atwqBBgzB8+HBs2bKlwudgAODJJ5/EypUrIZfLERYWhkOHDmHHjh0Wm/oGQKWfsblxXBP9f+qppzB79mzs3bu3yrv1Tz31FIKDgzFgwAA0bdoUeXl52LFjBzZu3Ih27drpXaOOHTtWblxER0fr3Gk3Rffu3fHqq69izpw5SEpKQp8+fWBvb4/Lly9j7dq1WLx4MYYOHQqg9I+iL7/8Eh9++CFCQkLg4+OjjRtNAnbx4kVtrAOlj6hs3bpVO5+iRnVcn7y8vLB9+3Z06dIFMTExOHDgABo1amTO21OpDz/8UNve+PHjYWdnh6+++gpFRUU6jzS88847WLlyJfr27YtJkyahXr16WLZsGQIDA3WuPW5ubvjyyy/xwgsv4LHHHsPIkSPh7e2NmzdvYvPmzejcuXOVA/S2b9+Ozp07WzS2LaLGx5PWYX///bfQr18/wc/PT7C3txceffRR4aOPPhJKSkqMOs6VK1eE119/XQgJCRGcnJwEZ2dnoUWLFsJrr70mJCUl6ZStamqNh61atUro2LGjUK9ePcHR0VFo0aKFMHPmTJ3pICpT0XQICoVCGDJkiODi4iI0aNBAePXVV4WzZ8+aNAVHWS1bthSkUqnwzz//GNQ/jd9//114/PHHBWdnZ8HNzU1o37698NNPP+mUWbNmjdCmTRvB0dFR8PDwEJ577jm9dspOL1HWw+ehmYJj3rx5emUBCAkJCTrbrl69KowaNUobK40aNRKefPJJ4ZdfftEpd//+fWHChAlCo0aNBAcHB6Fx48bC6NGjhXv37mnL/Pbbb0JYWJhgZ2en936fPHlSGDx4sODp6Sk4OjoKgYGBwvDhw4WdO3fqtLN3716hbdu2goODg/DII48IS5curXAo+8Mqeo9KSkqEpk2bVjpVwrlz5wQAgqOjo/DgwYNyy2imAyjvX69evQRBqHgKjvLafTjeyvsZys/PF7p37y7Ur19fZ7qBhz148EAYM2aM4OXlJdSvX1+IjY0VLly4oNeGOVNYCELln7EhcSx2/1u3bi2MGzdOZ1t5U3D89NNPwsiRI4WmTZsKzs7OgpOTkxAWFia89957QnZ2tk79imICgDB79uwK+2Loz7TGsmXLhLZt2wrOzs6Cq6ur0KpVK+Gdd94R7ty5oy2Tmpoq9O/fX3B1dRUA6E3H4ePjIwAQ0tLStNsOHDggABC6du1abj/NuT4JQvnX6itXrgj+/v5CaGhopb8zKrrOBwYGCv3799fbXt7P2okTJ4TY2Fihfv36gouLi9CjRw/h4MGDenVPnz4tdO/eXXBychIaNWokzJ49WzsVi2YKDo3du3cLsbGxglwuF5ycnISmTZsKL774onDs2DFtmfI+x8zMTMHBwUH45ptvKjxnsUgEoRqf5iOqZm3atIGHh0eFX4MQkfVbuXIl4uLicPPmTS4rRzVu0aJFmDt3Lq5evVrl84U1jc+kkc06duwYkpKSMGrUKLG7QkRmeO6559CkSZNy53gkqk4lJSVYsGAB3n//fatL0ACAd9LI5pw9exbHjx/H/Pnzce/ePVy7ds3k6UGIiIisFe+kkc355ZdfMGbMGJSUlOCnn35igkZERLUS76QRERERWSHeSSMiIiKyQkzSiIiIiKyQzU9mm5mZiZiYGCiVSiiVSkyaNAkvv/yywfXVajXu3LkDV1dXq1tYlYiIiGoXQRCQk5ODhg0bVrkmq80/k6ZSqVBUVAQXFxfk5eUhPDwcx44dM3jW4H/++UdvrUoiIiKi6nTr1q0qVwiy+TtpMpkMLi4uAICioiIIgmDUGpaurq4ASt8szXpharUaCoUC3t7e5Wa55u63BWKcg6XbNPd4ptQ3po6hZRmPjEdT6zMeqwfj0Tri0dwyYsVidnY2AgICtPlHZURP0vbt24d58+bh+PHjuHv3LtavX49BgwbplElMTMS8efOQmpqKiIgIfP7552jfvr12f2ZmJrp3747Lly9j3rx58PLyMrh9zVecbm5uOklaYWEh3NzcKrzImLPfFohxDpZu09zjmVLfmDqGlmU8Mh5Nrc94rB6MR+uIR3PLiB2LhjxiJfpPSF5eHiIiIiqcaXrNmjWIj49HQkICTpw4gYiICMTGxuoszu3u7o5Tp05pFxlPS0urqe4TERERVQvR76T169cP/fr1q3D/ggUL8PLLL2PMmDEAgKVLl2Lz5s349ttvMXXqVJ2yvr6+iIiIwP79+zF06NByj1dUVISioiLt6+zsbAClGbVardb+vyAI2tcPM3e/LRDjHCzdprnHM6W+MXUMLct4ZDyaWp/xWD0Yj9YRj+aWESsWjWlP9CStMsXFxTh+/DimTZum3SaVShETE4NDhw4BANLS0uDi4gJXV1dkZWVh3759eP311ys85pw5czBz5ky97QqFAoWFhQBK38CsrCwIglDh7Xpz9tsCMc7B0m2aezxT6htTx9CyjEfGo6n1GY/Vg/FoHfFobhmxYjEnJ8fgsladpN27dw8qlQq+vr462319fXHhwgUAwI0bN/DKK69oBwy88cYbaNWqVYXHnDZtGuLj47WvNQ/weXt76zyTJpFItA8TqtVqFBcXa+uo1WoolcpKn7mobL8tEOMcLN2mucczpb4xdQwta2681cZ4tLe3h0wmq/Y2y14HxD6eKfWNqWNo2arKmbvfFohxDoxHy5cRKxaNWcrQqpM0Q7Rv3x5JSUkGl3d0dISjo6PedqlUqvMhSSQSSKVSKJVKpKSk6Nye1Nwezc3NLffBv6r22wIxzsHSbZp7PFPqG1PH0LLmxlttjUd3d3f4+flV6zlprgOWuoCbezxT6htTx9CyVZUzd78tEOMcGI+WLyPG52hMW1adpHl5eUEmk+kNBEhLS4Ofn1+1ty8IAu7evQuZTIaAgADtGysIApRKJezs7Cr8pVjZflsgxjlYuk1zj2dKfWPqGFrW3HirbfEIAPn5+drBQ/7+/mJ2jYio2lh1kubg4IC2bdti586d2mk51Go1du7ciQkTJlR7+0qlEvn5+WjYsKF2Ljag7v1SZJLGJE1sD5+Ds7MzACA9PR0+Pj7V/tUnEZEYRE/ScnNzceXKFe3rlJQUJCUlwcPDA02aNEF8fDxGjx6NqKgotG/fHosWLUJeXp52tKelqNX6ozuVSiUEQYC9vb3eBLma1xVNnFvVflsgxjlYuk1zj2dKfWPqGFrW3HirjfHo7OwMQRBQVFRk1DMehuJoOtPKcXSnbbRZG+KRoztrwLFjx9CjRw/ta81D/aNHj8aKFSswYsQIKBQKTJ8+HampqYiMjMS2bdv0BhMYKzExEYmJiVCpVADKH91ZUlICtVoNlUoFpVKprSsIgrZeRXcuKttvC8Q4B0u3ae7xTKlvTB1Dy5obb7U1HlUqFdRqNe7fvw97e3uLt8nRdKaV4+hO22izNsQjR3fWgOjo6Cr/up8wYYLFv96Mi4tDXFwcsrOzIZfLyx3d6erqitzcXNjZ2WmfhSmrql8M1fGLo6aJcQ6WbtPc45lS35g6hpY1N95qWzza2dlBKpXC09Oz2u6kcTSd8eU4utM22qwN8cjRnXVIRaM7JRKJ9p+GIAja1xXduahsvy0Q4xwqajMoKAiTJ0/G5MmTDTrOjBkzsGHDBpw8ebLSc/jggw+QlpaGZcuWGdUfU87BnLLmxpsp56F5DzUjp1988UVkZmZiw4YNBtXXtFXeMm+GGDlyJNq1a4c333wTQPnnoPm5rM6RWRxNZ1o5ju60jTZrQzxacnSnSi3gSEoG0nMK4ePqhPbBHpBJLf/7z5j3y3Z/QmyISi3g0NX7+C3pNg5dvQ+VunqfC3rxxRd1fjEqFAq8/vrraNKkCRwdHeHn54fY2Fj89ddf2jJBQUFYtGhRtfbLmqSmpmLx4sV47733tNv27duHAQMGoGHDhpBIJOUmJC+++KJO4i6RSNC3b1+dMhkZGXj++efh5uYGd3d3jBs3Drm5uVX26eTJkxgxYgT8/f3h6OiIwMBAPPnkk9i4caP2bvP169d12nZwcECzZs3w8ccfl3tHeubMmXjhhReMfHf0LV68GCtWrDD7OIZ6//338dFHHyErK6vG2iSiumvb2bvo8ukuPPP1YUxanYRnvj6MLp/uwrazd0XtF++kVbNtZ+9i5sZk3M0q1G7zlzshYUAY+obXzNQBQ4YMQXFxMb777js88sgjSEtLw86dO3H//v0aad8affPNN3j88ccRGBio3aZZR3bs2LEYPHhwhXX79u2L5cuXa18/PO/e6NGjkZqaiu3bt6OkpARjxozBK6+8gh9//LHCY/72228YPnw4YmJi8N133yEkJARFRUU4ePAgPvjgA3Tq1AleXl7a8jt27EDLli1RVFSE/fv34+WXX0ajRo3w0ksv6R333XffNfh9qYhcLjf7GMYIDw9H06ZNsWrVKsTFxdVo20RUt2w7m4q4H0/i4T9zU7MK8fqqE/jy+cdq7Pf1w3gn7V+a0Z2af5oRH5qVDMr+A3RHmlW0f9vZVLy+6oROggb874PfeuZuufXN/achCAIePHiA/fv345NPPkF0dDSaNGmCdu3aYerUqRgwYIBenw09x6CgIMyePRujRo1C/fr1ERgYiN9++w3p6el46qmnUL9+fbRu3RpHjx7VqffLL7+gZcuWcHR0RFBQED777DOdNtPS0jBw4EA4OzsjODgYq1at0uvDgwcPMG7cOO1zhD179kRSUlKF70F557B69Wo8+eSTOtv69u2L2bNn69yFfLg+UJqU+fr6av+5u7tr958/fx5//PEHvv76a7Rv3x6dO3fGkiVLsHr1aty+fbvc9zgvLw/jxo1D//79sWnTJvTu3RvBwcFo0aIFxo4di6SkJO3zkpp6Hh4e8PX1RZMmTfDcc8+hU6dOOHHihM7xb968iXPnzqFv375Qq9WYMWOG9m5qw4YN8cYbb1QZR5r/19yd1byOjo7GG2+8gbfffhseHh7w8/NDQkJChfUFQcD06dPh7++PU6dOQRAEJCYmolmzZnBycoKvry+GDh2qU/7JJ5/E6tWrq4zHh392LfnP0sc393im1DemjqFlqypn7n5b+CfGOTAeLV9GqVJj1qZkvQQNgHbbzI3JKFGqLPreG6rO3kkzZnSnUqnUTsdRUKKCIJSOLJMVq1DeIz6CABSXKDHj93MVfvASADM2nkOHILlB33k728sMfp5IG3xKJZycnFC/fn2sX78eUVFR5a62ULaeZhSrIFQ9InDRokWYPXs2pk6diiVLlmDUqFHo1KkTRo8ejY8//hj/93//h1GjRuHUqVOQSCQ4ceIERowYgQ8++ADDhg3D4cOH8cYbb6BBgwYYNWoUBEHAuHHjtHeh7O3tMWXKFKSnp+v0bdiwYXBycsLGjRvh5uaGb775BjExMTh37hw8PDy0P5QlJSXlnkNGRgaSk5PRpk0bnVG7D1MqlXr11Wo19uzZo03OevTogZkzZ8LT0xMAcODAAbi7uyMiIkJ77OjoaEilUhw8eFAvAVSpVNi6dSvu37+P+Pj4cvujucCUlJRo92tiEigdIX3y5Ek8//zzOvU3bNiA7t27w9nZGb/88gsWLVqEVatWISwsDGlpaTh9+nSF5695DzX7y8aUpk/ff/89Jk2ahAMHDuDw4cN46aWX0LFjR8TExGiPo1KpUFJSgilTpmDLli3YtWsXQkJC8Pfff2PSpElYvnw5OnXqhIyMDPz11186/Wnbti0+/vhj5OXlwcHBQe+zUCqVUKs5utNSdQwtW1U5c/fbAjHOgfFo+TJqtRp/XUpFanZRhf0QANzNKsSfJ6+hbYBr5SdoIJsa3SkWU0Z35hcrETF7l0XaFwCkZRfhsY92G1T+3Mw+cLE37OPSPASp6ffy5cvxyiuvYNmyZXjsscfQrVs3jBw5Eq1bt9ar9/Ao1sp++T3xxBPaxewTEhLw1VdfoV27dhg5ciQAYOrUqXj88cdx//59+Pn5YcmSJejVqxcSEhIAAGFhYbhw4QIWLFiAsWPH4tKlS/jzzz/x999/o127dgCA//73vwgLC9P27cCBAzh69CjS0tK0Cef8+fPx+++/Y8OGDXjllVe0Az40fX/4HO7cuQNBEBAQEFDuqF0NOzs7yGQynfr9+vXDkCFDEBwcjKtXr+K9997DwIEDcfDgQchkMigUCnh7e+uNQvTw8IBCoSi3vatXr2rfD83+o0ePomfPntoyK1euxKBBg7T7u3fvDqlUiuLiYpSUlGDcuHF6cwdu2rQJAwcOhL29Pf755x/ts4j29vZ45JFH0KlTpwrPXfMeatorG1NAaaLUunVrzJw5EwAQGhqKpUuXYs+ePTrP6AmCgDFjxuDkyZPYv38/GjVqBAC4ffs26tWrh6eeegqurq5o2rSp9jPXCAgIQHFxMe7du6f9WpqjOzm60xqIcQ6MR8uXUavVKLz0wKC+l9g5w8fHx6CyVeHoThMYMrpTzJGaprSvKT906FA8+eST2L9/Pw4fPoytW7di3rx5+Oabb/Diiy+W24YgVD0isHXr1tp9mmW6ytumUCjg7++P8+fP46mnntI5XpcuXbB48WKo1WqcP38ednZ2aNu2rbZMaGgo3N3dtX07ffo0cnNzdZ7PAoCCggJcu3ZN730q7xw0d0ydnZ2rfE8frv/MM8/onH9ERASaNm2KvXv3olevXuXWLfu6olHCD++PiIjQjqxs1qyZ3l2kNWvWIDQ0FCUlJThz5gwmTpyIqVOn4tNPPwUAZGdnY+/evfjvf/8LoDQG/vOf/6Bp06bo27cvnnjiCQwYMKDCJLWiz77s67KfNVC6PJNCodDZFh8fD0dHRxw+fFjnM+vTpw8CAwO1/enbty+efvppnZU9NP9fUFBQbp84utPydTi603Ac3Wkd8WhumfxilUH99nVzttj7bsxxmKQZwdlehuRZsdqvgSpbhufQFQVeWnmyymOuGNMO7YM9DGrbHE5OTujduzd69+6NDz74AC+99BISEhJ0kjRjlb2roXkfyttmzPfvVcnNzYW/vz/27Nmjt8/d3d2gY2iShQcPHsDb29us/jzyyCPw8vLClStX0KtXL/j5+UGhUOiUUSqVyMjIqHC92WbNmgEALl68iI4dOwIofe4tJCSkwnYDAgK0+1u0aIHLly9jxowZmDlzJpycnLB161aEhYUhICBAe9fwwoUL2LlzJ7Zv347x48dj3rx52Lt3r8lfFT5cTyKR6H3WvXv3xk8//YQ//vgDzz33nHa7q6srTpw4gT179uDPP//E9OnTMWPGDBw9elT7OWZkZACA2Z8REdHDVGoBX+29is9236q0nASAn9zJoN/T1cF2/4wRgUQigYuDnUH/uoR4wU/uhIru00hQOsqzazNvg45n6bt4YWFhyMvLs+gxqxIaGqoz7QcA/PXXX3j00Uchk8nQokULKJVKHD9+XLv/4sWLyMzM1L5+7LHHkJqaCjs7O4SEhOj8e/juWkWaNm0KNzc3JCcnm31O//zzD+7fv69d5LtTp07IzMzUOYddu3ZBrVajQ4cO5R6jT58+8PDw0N4FM4VMJoNSqURxcTGA0lGdTz31lE4ZZ2dnDBgwAEuWLMGePXtw6NAhnDlzxuQ2DTFw4ED8+OOPeOmll7B69WqdfXZ2doiJicHcuXNx+vRpXL9+Hbt2/e9xgrNnz6Jx48YGf65ERIa4k1mA5745jLl/XIJaACID5JAAer+vNa8TBoRVy3xphuCdtH+VHXFRdjRIeSMGgarXQpRKgIQnwzD+hxOQADoDCDQf9fQnwyCVVN96ioIg4P79+xg+fDjGjBmD1q1bw9XVFceOHcPcuXMxcOBAnbb/+ecfnDz5v7t/SqUSTZs2RYMGDSo8fnnvS0Wj8eLj49G+fXvMmjULI0aMwKFDh/Cf//wHiYmJEAQBzZs3R58+ffDaa6/hiy++gJ2dHaZMmaJdo1EQBPTq1QudOnXCoEGD8Omnn+LRRx/FnTt3sHnzZjz99NOIioqqcHSnhkQiQUxMDPbv36+TyFS0jqyPjw+aNGmC3NxczJw5E0OGDIGfnx+uXr2Kd999FyEhIejTpw8EQUBoaCj69OmDV155BV9++SVKSkowYcIEjBw5Ev7+/uW+X/Xr18fXX3+NkSNHon///njjjTfQrFkz5ObmYtu2bQD+d3tcU//evXu4e/culEolzpw5g//85z/o0aMHXF1dUVJSgq1bt+LNN9/Ulv/uu+8AAB06dICLiwtWrlwJZ2dnNGnSpNz4qyi+H35vK6ur+f9Bgwbh+++/x6hRoyCTyTB06FBs2rQJ165dQ7du3dCgQQNs2bIFarUajz76qLb+/v370bt373LjqWz7xo6WMlTZ64A1HM+U+sbUMbRsVeXM3W8LxDgHxqNlymw+fRfvbTiL7EIlXBxkmNytEcZ0b4Ht5xWYtek8UrP/NxuDn9wJH/QPRZ8wX4t+1hzdaQBTRndqVDXyUbO/V3NPfD4yAh9uuaAzesRP7oj3+rVATAuvSkcXmurh0Z1RUVFYuHAhrl27hpKSEjRu3Bhjx47F1KlTddqfP38+5s+fr3Os5cuX63xN9XA7D/e/7DqnD49EbN26NX788UfMnDkTH374Ifz9/ZGQkKAdlSgIApYuXYq4uDhER0fD19cXM2bMwK1bt3Ta+u233zB9+nSMHTsWCoUCfn5+6NKlCzw9PbUj/iob3QmUTkr7+uuv4+OPP9YmQH///Td69+6tLaOZ7f6FF17Af//7XwiCgNOnT+P7779HZmYmGjZsiJiYGMyYMUN7J0sQBHz77bd48803ERMTA6lUiqeffhoLFy7Ue6/KxtGAAQOwb98+zJs3D6NHj0ZGRgbkcjnatm2LVatWoW/fvjqjOzX9lMlk8Pf3R58+fTB79mwolUrs2rVLOwWKpk+urq5YsGAB3nzzTahUKoSHh2P9+vWQy+XlxqAhozvL7i+vTNl4GDRoEP773/9qR/H6+Pjg119/xcyZM1FYWIiQkBCsXLkSzZs3h1KpRGFhITZs2IBNmzZpz4GjO8UfTWdIOY7utI02a0M8GlMmt1CJhftuY8v50vlBw3xdkBAbCDdJERQKBR7zkWLdi2FIup2L+3kl8Kxnj8hG9SGTSpCenm7Q+RnKmNGdEqG6buPYCM3ozgcPHuiM7lQoFHB1dcXNmzcRHBysNxqjpKSk0l8MZfer1AKOXs9AenYRfNwc0S6oepaasLSqztEW2qzoeIIgoGPHjpg8ebLOYABL9MeYOoaWNSbeJk6cCKVSiS+++MKkPlmDL7/8Ehs2bMAff/yh3fbwORQWFiIlJQVBQUHVNrpTM1rXUr8UzTmeKfWNqWNo2arKmbvfFohxDoxH08vsOp2C2dtv4daDAkglwPjopnijZwhkEogSi9nZ2WjQoAGysrK0eUdF6uydtIcZMrpTo6qRjw/vt5NJ0KmpbT1XY8joTmtvs7LjSSQSLFu2DGfOnKmwLVP6Y0wdQ8saG2+tWrVCp06djBqpa20cHBzw+eefV3oOHN1p+Toc3Wk4ju60jnisrIxSpcbnu6/iP7uuQCUAjdydsXBEpHYQgGZ6jpr+HDm6k8gAkZGRiIyMFLsbFvfKK6+I3QWzPby8FRGRMW7ez8fkNSdx4mYmAOCpyIaYPSgcbk62840CwCSNiIiIaglBEPDridtI+P0ccouUqO9oh7d6NMaobqE2eeeWSRoRERHZvKz8Ery34Qw2nb4LAGgX1ADzh7WGQ0muyD0zHZM0A9TxsRVEVok/l0Skcejqfbz5cxLuZBVCJpVgSkwzvB4dAgkEpKczSbN55c2TJpFIIAgCiouL9UaPVTVPWlX7bYEY52DpNs09nin1jaljaFlz4602xmNeXh4EQYBMJuM8aRaow3nSDMd50qwjHtVqNYqVKny67QKW7U+BIACBni5YNDwCEQHuAIRKjyNWLHKeNAMYMk+aZuRHWlqa3rqWarVaO/rzYVXttwVinIOl2zT3eKbUN6aOoWXNjbfaFo9A6fQb6enpcHR0xP3796ulTc5LZVo5zpNmG23WhnhMuZ+P6Vuu4vL90pVWBrT0xJTuAXBxKNbObVbZccSKRWPmSauzSVpcXBzi4uK086R5e3vrzJMmkUjg7e0NT09PXL9+Hf/8849O/bK/MMpT1X5bIMY5WLpNc49nSn1j6hha1tx4q43x6OHhAV9f32pLPMteByz1S9Gc45lS35g6hpatqpy5+22BGOfAePwfQRCw+ugtfLj5EgpKVJA72+Hjp1uhX7j++siVHUesWDRmXsc6m6Q9rKJ50pycnPDoo49q10QESj/Y+/fvw9PTs8KLUGX7bYEY52DpNs09nin1jaljaFlz4602xqO9vT1kMlm1t8t5qUwrx3nSbKNNW4zH+7lFmPrrGWxPTgMARAW4YsmzUWjYwMWkNjlPWi2gSdY01Go17O3t4eTkVOEvxcr22wIxzsHSbZp7PFPqG1PH0LLmxhvjkYhqg32XFHhz7SkocopgL5Pg7T7N8eSjLvCTW37FEWvBJI2IiIisVmGJCnO3XcS3f6UAAEJ86mPxyEiE+rlafF1Na8MkjYiIiKzShdQcxP98ChdSSx+2H9UpENP6hcLZoXpGdVsbJmlERERkVQRBwJqT6Uj86zaKlWp41XfA3KGt0bOFr9hdq1FM0oiIiMhqpOcU4u21p7D30j0AQI/m3pg7NALero4i96zmMUn7V3mT2XKyRk7WKPZkjYaUYzzaRpuMR+P7ZK0Yj9UXjzvOp2HqujPIyC+Bg0yCaf1aYFSnQEgkEr16hhyvsjJixaIx7dXZJM2QyWw5WSMnaxR7skZDyjEebaNNxqPp52FtGI+Wj8fCEjUW77uF9WdK754183LGm5090TrQCQqFwuQ+VFZGrFjkZLYGMHQyW07WyMkaOXmo+BiPjEdrwni0bDyevZ2FyWtO4dq9PADAS12CMSUmBNkP7ld6fEP6UFkZsWKRk9maoKLJbDlZIydr5OSh1oHxyHi0JoxH8+NRpRbw9f5rmP/nRZSoBPi6OWL+sEh0aeYFtVqNHAOOb0gfOJktERERkYHuZBYg/uckHL6WAQDo29IPcwa3QoN6DiL3zLowSSMiIqIas+XMXfzf+rPILlTCxUGGGQNaYlhU42pbh9eWMUkjIiKiapdbpMSsP65jy/n7AICIxnIsGtkGwV71RO6Z9WKSRkRERNXq+I0HmLLmJG5mFEAqAcZHh2BSTDPYy2z3ucSawCSNiIiIqoVSpUbi7qtYsusyVGoBfq4OWDSyDTo29RK7azaBSRoRERGZRaUWcCQlA+k5hfBxdUL7YA/cflCAKT8n4fiNBwCAgRH+eKOTD5o28RC5t7aDSRoRERGZbNvZu5i5MRl3swq12+TO9igqUaFQqYarox1mDwrHwAh/pKeni9hT28MkjYiIiEyy7Wwq4n48CeGh7VkFJQCApt71sGJMewR4uNj0UmBiYZL2L67dqYtr03GtRGvCeGQ8WhPGY2l9pUqNWZuS9RK0svKLVfBzc9T+jrVkPHLtzlqMa3dWjmvTca1Ea8J4ZDxaE8Zjaf2/LqUiNbuo0nJ3swrx58lraBvgavF45NqdtRjX7qwc16bjWonWhPHIeLQmjMfS+oWXHhhUtsTOGT4+PhaPR67dWYdw7U59XJuOayVaE8Yj49Ga1PV4FAQBF9LzDSrr6+asbcPS8ci1O4mIiIj+lZFXjHd/OYXt5ysfqSkB4CcvnY6DTGO7f8YQERFRjdp3SYHYRfuw/Xw67KQSDG7TCBKUJmRlaV4nDAiDTMo1OU3FO2lERERUqcISFeZuu4hv/0oBAIR418P03k3QJTwIfVr66s2T5id3QsKAMPQN9xery7UCkzQiIiKq0MXUHExafRIXUktHJY7qFIh3Y5sjJ7N0ofS+4f7oHeant+IA76CZj0kaERER6REEASsOXsecrRdQrFTDs54D5g5tjV6hvlCr1Sg7kYRMKkGnpp6i9bW2YpJGREREOtJzCvH22tPYe0kBAOjR3Btzh0bA29VR5J7VLUzSiIiISGt7chreXXcaGXnFcLST4r3+oXihYyAkEn59WdOYpBEREREKilX4cHMyfvj7JgAg1N8NS0ZGopmvq8g9q7uYpBEREdVxZ29nYeLqk7imyAMAvNw1GG/FNoejnUzkntVtTNKIiIjqKJVawLL9VzH/z4soUQnwdXPE/GGR6NLMS+yuEZikaanVau3K9Gq1GoIgVLhSvbn7bYEY52DpNs09nin1jaljaFnGI+PR1PqMx+pRW+IxNbsIk377G3+nlK7BGdvSFx8/HY4GLg5mx4G5dQwpa24ZsWLRmPbqbJKWmJiIxMREqFQqAIBCoUBhYelEfGq1GllZWRAEocIFgs3ZbwvEOAdLt2nu8Uypb0wdQ8syHhmPptZnPFaP2hCPOy7exyc7byK3WA1neyniowPwZJgnSnIzkZ5bPf2xdDyaW0asWMzJyam60L/qbJIWFxeHuLg4ZGdnQy6Xw9vbG25ubgBKPziJRAJvb+8KLzLm7LcFYpyDpds093im1DemjqFlGY+MR1PrMx6rhy3HY26REjM3JmPdidsAgNaN3LBwRCSCvepVe38sHY/mlhErFp2cnAwuW2eTtIdJpVKdD0kikehtK8vc/bZAjHOwdJvmHs+U+sbUMbQs45HxaGp9xmP1sMV4PHHzASavTsLNjHxIJcCodn6YNiACjvampQLWEI/mlhHjczSmLSZpREREtZhSpUbi7qtYsusyVGoBjdydMX9YawTVU8JeZruJcl3AJI2IiKiWunk/H1N+TsLxG6WDA56KbIjZg8JR30GG9PR0kXtHVWGSRkREVMsIgoBfT9xGwu/nkFukhKujHWYPCsegNo0AGDfCkMTDJI2IiKgWycovwXsbzmDT6bsAgHZBDbBgeCQCPFxE7hkZi0kaERFRLXH42n3Er0nCnaxCyKQSTIlphtejQyCTct1NW8QkjYiIyMYVK9VYuOMSlu69CkEAAj1dsHhkG0QGuIvdNTIDkzQiIiIbdlWRi8mrk3DmdhYAYERUAKYPCEM9R/6Kt3X8BImIiGyQIAhYffQWZm1MRkGJCnJne3wyuBX6tfIXu2tkIUzSiIiIbExGXjHeXXca25PTAACdQzwxf1gk/OSGz2ZP1o9JGhERkQ3Zd0mBN9eegiKnCPYyCd6JbYFxXYIh5eCAWodJGhERkQ0oUqoxe/N5LP/rOgAgxKc+Fo+MRMuGcnE7RtWGSRoREZGVu5iagwmrL+DqvQIAwKhOgZjWLxTODjKRe0bViUkaERGRlRIEASsOXsecrRdQrFTDs54D5g1rjZ4tfMXuGtUAJmlERERWKD2nEG+vPY29lxQAgMeD3LDwmSj4yp1F7hnVFCZpREREIlKpBRxJyUB6TiF8XJ3QPtgDuy+k4511p5GRVwxHOymm9WuB2Eec4O3qKHZ3qQYxSSMiIhLJtrN3MXNjMu5mFWq3uTjIkF+sAgCE+rthychINPWuh/T0dLG6SSJhkkZERCSCbWfv4vVVJyA8tF2ToPUO88V/nm0DRzsZ1Gp1zXeQRMck7V9qtVr7Q6BWqyEIQoU/FObutwVinIOl2zT3eKbUN6aOoWUZj4xHU+szHquHJc5BpRYw4/dkvQStrLO3syDF/34/MR4tW0asWDSmvTqbpCUmJiIxMREqVelfLAqFAoWFpbeb1Wo1srKyIAgCpFKpXl1z99sCMc7B0m2aezxT6htTx9CyjEfGo6n1GY/VwxLncPxWDlKzCystczerEH+evIa2Aa6Mx2ooI1Ys5uTkGFy2ziZpcXFxiIuLQ3Z2NuRyOby9veHm5gag9IOTSCTw9vau8CJjzn5bIMY5WLpNc49nSn1j6hhalvHIeDS1PuOxeljiHEruKg0rZ+cMHx8fxmM1lBErFp2cDF+6q84maQ+TSqU6H5JEItHbVpa5+22BGOdg6TbNPZ4p9Y2pY2hZxiPj0dT6jMfqYc455BYp8evJ2waV9XVz1rbBeLR8GTFi0Zi2mKQRERHVkBM3H2Dy6iTczMivtJwEgJ+8dDoOqrts988YIiIiG6FUqbF4x2UMW3oINzPy0cjdGW/2fhQSlCZkZWleJwwIg4yLptdpvJNGRERUjW7ez8eUn5Nw/MYDAMCgyIaYNSgcbk72aOZbX2+eND+5ExIGhKFvuL9YXSYrwSSNiIioGgiCgPUnb2P6b+eQW6SEq6MdZg8Kx6A2jbRl+ob7o3eYn96KA7yDRgCTNCIiIovLKijB+xvOYuOpOwCAdkENsGB4JAI8XPTKyqQSdGrqWdNdJBvAJI2IiMiCDl+7j/g1SbiTVQiZVIIpMc3wenQI746R0ZikERERWUCxUo2FOy5h6d6rEAQg0NMFi0e2QWSAu9hdIxvFJI2IiMhMVxW5mLw6CWduZwEARkQFYPqAMNRz5K9ZMh2jh4iIyESCIGD10VuYtTEZBSUqyJ3t8cngVujXiiMzyXxM0oiIiEyQkVeMaevPYntyGgCgc4gn5g+LhJ/c8GV/iCrDJI2IiMhIf9/Ixoc7zkKRUwR7mQTvxLbAuC7BkHJwAFkQkzQiIiIDFZao8Om2C1j+13UAQIhPfSweGYmWDeXidoxqJSZpREREBriYmoNJq0/iQmoOAOCFjk3wf0+EwdlBJnLPqLZikkZERFQJQRCw4uB1zNl6AcVKNTzrOeD/Yprg6Q7NIJVyCWyqPkzSiIiIKpCeU4i3157G3ksKAECP5t74ZHArCAVZIveM6gImaUREROXYnpyGd9edRkZeMRztpHivfyhe6BgIQRCQXiB276guYJJGRERURkGxCh9uTsYPf98EAIT6u2HJyEg083UFUPr1J1FNYJJGRET0r7O3szBx9UlcU+QBAF7uGoy3YpvD0Y6DA6jmMUkjIqI6T6UW8PX+a5j/50WUqAT4ujli/rBIdGnmJXbXqA5jkkZERHXancwCxP+chMPXMgAAfVv6Yc7gVmhQz0HknlFdxySNiIjqrM2n72Lar6eRXaiEi4MMMwa0xLCoxpBIuHIAiY9JGhER1Tm5RUok/HYO6078AwCIaCzHopFtEOxVT+SeEf0PkzQiIqpTjt94gClrknAzIx9SCTA+OgSTYprBXsaJacm6MEkjIqI6QalSI3H3VSzZdRkqtYBG7s5YOCIS7YM9xO4aUbmYpBERUa13834+pvychOM3HgAAnopsiNmDwuHmZC9yz4gqxiSNiIhqLUEQ8OuJ20j4/Rxyi5RwdbTD7EHhGNSmkdhdI6qSzX8Bf+vWLURHRyMsLAytW7fG2rVrxe4SERFZgaz8Erzx00m8ufYUcouUaBfUAFsmdWWCRjbD5u+k2dnZYdGiRYiMjERqairatm2LJ554AvXqcYQOEVFddfjafcSvScKdrELIpBJMiWmG16NDIJNyag2yHTafpPn7+8Pf3x8A4OfnBy8vL2RkZDBJIyKq5VRqAUdSMpCeUwgfVye0D/aASi1g4Y5LWLr3KgQBCPR0waIRkWjTpIHY3SUymuhJ2r59+zBv3jwcP34cd+/exfr16zFo0CCdMomJiZg3bx5SU1MRERGBzz//HO3bt9c71vHjx6FSqRAQEFBDvSciIjFsO3sXMzcm425WoXabd30HODvY4WZGPgBgRFQApg8IQz1H0X/VEZlE9GfS8vLyEBERgcTExHL3r1mzBvHx8UhISMCJEycQERGB2NhYpKen65TLyMjAqFGjsGzZsproNhERiWTb2bt4fdUJnQQNABS5xbiZkQ8XBxm+fO4xfDq0NRM0smmiR2+/fv3Qr1+/CvcvWLAAL7/8MsaMGQMAWLp0KTZv3oxvv/0WU6dOBQAUFRVh0KBBmDp1Kh5//PFK2ysqKkJRUZH2dXZ2NgBArVZDrVZr/18QBO3rh5m73xaIcQ6WbtPc45lS35g6hpZlPDIeTa1fG+NRpRYw4/dkCJWUqe9oh5hQn2qLF8ajdcSjuWXEujYa057oSVpliouLcfz4cUybNk27TSqVIiYmBocOHQJQOrz6xRdfRM+ePfHCCy9Uecw5c+Zg5syZetsVCgUKC0v/KlOr1cjKyoIgCJBK9W82mrvfFohxDpZu09zjmVLfmDqGlmU8Mh5NrV8b4/H4rRykZhdWWiY9pwh/nryGtgGuRh/fEIxH64hHc8uIdW3MyckxuKxJSVpJSQlSU1ORn58Pb29veHhUz2zN9+7dg0qlgq+vr852X19fXLhwAQDw119/Yc2aNWjdujU2bNgAAFi5ciVatWpV7jGnTZuG+Ph47evs7GwEBATA29sbbm5uAEo/OIlEAm9v7wovMubstwVinIOl2zT3eKbUN6aOoWUZj4xHU+vXxngsuas0rJydM3x8fIw+viEYj9YRj+aWEeva6OTkZHBZg5O0nJwcrFq1CqtXr8aRI0dQXFwMQRAgkUjQuHFj9OnTB6+88gratWtnUqdN1aVLF6NuHTo6OsLR0VFvu1Qq1fmQJBKJ3rayzN1vC8Q4B0u3ae7xTKlvTB1DyzIeGY+m1q9t8VisrOyLzv/xdXOu1lhhPFpHPJpbRozP0Zi2DCq5YMECBAUFYfny5YiJicGGDRuQlJSES5cu4dChQ0hISIBSqUSfPn3Qt29fXL582eTOl+Xl5QWZTIa0tDSd7WlpafDz87NIG0REZP0EQcDyv1Lw3oYzlZaTAPCXO3E9TqoVDLqTdvToUezbtw8tW7Ysd3/79u0xduxYLF26FMuXL8f+/fvRrFkzszvn4OCAtm3bYufOndppOdRqNXbu3IkJEyaYffyyOHBAFx+MtY4HYw0px3i0jTYZj8b3SUORU4R31p3G3kv3AAAtG7rh3J1sSACdAQSaaWo/6B8KCQSo1YbddTMW49E64pEDB/71008/GXQwR0dHvPbaawY3DgC5ubm4cuWK9nVKSgqSkpLg4eGBJk2aID4+HqNHj0ZUVBTat2+PRYsWIS8vTzva01SJiYlITEyESqUCwIEDD+ODsdbxYKwh5RiPttEm49G089h3NRMf77iBzAIlHGUSvNGtMYa09saeq5lYuOcW0nNLtGV96ttjcnQAHvOR6k3TZEmMR+uIRw4cMEB2djZ27dqF5s2bIzQ01Oj6x44dQ48ePbSvNQ/1jx49GitWrMCIESOgUCgwffp0pKamIjIyEtu2bdMbTGCsuLg4xMXFITs7G3K5nAMHHsIHY63jwVhDyjEebaNNxqNx7RQUq/DRlvP48cgtAECovysWDY9AM9/SEZsjfH0xtOOjOHo9A+k5RfBxdUS7II8aWfaJ8Wgd8ciBA+UYPnw4unXrhgkTJqCgoABRUVG4fv06BEHA6tWrMWTIEKOOFx0dDUGo/Jb0hAkTLP715sM4cEAfH4y1jgdjDSnHeLSNNhmPhu0/ezsLE1efxDVFHgDg5a7BeCu2ORztZDrlpFLg8RDvqk6rWjAerSMeOXDgIfv27UPXrl0BAOvXr4cgCMjMzMSSJUvw4YcfGns4IiIiAKUT1S7dexVPf/EXriny4OvmiFXjOuC9/mF6CRpRXWB0kpaVlaWdF23btm0YMmQIXFxc0L9/f4uN6iQiorrlTmYBnvvmMD7ZegElKgGxLX2xbVI3dGnmJXbXiERj9NedAQEBOHToEDw8PLBt2zasXr0aAPDgwQOjvme1NhzdqYujl6xj9JIh5RiPttEm47Hi/VvO3MX/rT+L7EIlnO1lmD4gFMPbNoZEIrHKuGU8Wkc8cnRnOSZPnoznnnsO9evXR2BgIKKjowGUfg1a0Sz/1oijOyvH0UvWMXrJkHKMR9tok/Gov79AKWD+7lvYcv4+ACDM1wUz+gajSQNHKBQKg85RDIxH64hHju4sx/jx49G+fXvcunULvXv31p7YI488go8++sjYw4mGozsrx9FL1jF6yZByjEfbaJPxqLv/VoE93vrlNG5mFEAqAV7v3hQTe4XAXmb9Mcp4tI545OjOCkRFRSEqKkpnW5s2bbBq1So8/vjjphxSdBzdqY+jl6xj9JIh5RiPttEm4xFQqtT49u+7+PZIKlRqAY3cnbFwRKTNrRDAeLSOeKztozuNTtLGjh1b7vYbN27gyJEjeOedd4w9JBER1QE37+djypqTOH4zEwDwVGRDzHoqHHJne3E7RmSljE7SHjx4oPNapVLh2rVrSE5OxpdffmmxjhERUe0gCAJ+PXEbCb+fQ26REvUcpJg9KByDHwsQu2tEVs3oJG39+vXlbv/oo4+wYcMGvPrqq2Z3ioiIaoes/BK8t+EMNp2+CwBoG9gA7/VshMhmjUTuGZH1M3tZKI1nnnnGpiez5RQcujjE3DqGmBtSjvFoG23WxXj8+9p9xK89jbtZhZBJJZjUMwSvdA3Cg4z7jEeR26wN8cgpOIxw6tQptGnTxlKHq3acgqNyHGJuHUPMDSnHeLSNNutSPN7LyMQXB25j1fE0CAAayx0xo28wwv3rIeP+PcajFbRZG+KRU3CUQ7MAellpaWn47bff0L9/f539CxYsMPbwNYZTcFSOQ8ytY4i5IeUYj7bRZl2JxytpOXjnz4u4kJ4PABge1Rgf9A9FPUc7k8/D2jAerSMeOQVHOU6ePFnu9nbt2iE9PR3p6ekASoe12hJOwaGPQ8ytY4i5IeUYj7bRZm2OR0EQsProLczamIyCEhXkzvb4ZHAr9Gvlb5HzsDaMR+uIR07B8ZDdu3cbW4WIiGqxjLxivLvuNLYnpwEAogJcsfjZtmjUoJ7IPSOybRZ7Jo2IiOqefZcUeHPtKShyimAvk+DtPs3x5KMu8JM7i901Iptn0D23vn374vDhw1WWy8nJwaefforExESzO0ZERNarqESFWRuTMerbI1DkFCHEpz42xHXGS12DIbWxx12IrJVBd9KGDRuGIUOGQC6XY8CAAYiKikLDhg3h5OSEBw8eIDk5GQcOHMCWLVvQv39/zJs3r7r7bXGcgkMXh5hbxxBzQ8oxHm2jzdoUj1cU+Zj900VcTMsFADzfoQmm9WsBZwcZ49FG2qwN8cgpOP41btw4PP/881i7di3WrFmDZcuWISsrC0DpQ3dhYWGIjY3F0aNHERoaalqvaxin4Kgch5hbxxBzQ8oxHm2jzdoQj4IgYM3JNCQeuIMStYAGznZ4r3cgujzijpzM+8gx4DiMR+toszbEI6fgKMPR0RHPP/88nn/+eQBAVlYWCgoK4OnpCXt721t3jVNwVI5DzK1jiLkh5RiPttGmrcejIqcI7647jb2X7gEAoh/1wqdDWsPb1dGo4zAeraNNW49HS5SplVNwaMjlcsjlclOrWx1OwaGPQ8ytY4i5IeUYj7bRpq3G4/bkNLy77jQy8orhaCfFG10b4fWYlpDJZCa1yXi0jjZtNR4tWabWTcFBRER1Q0GxCh9uTsYPf98EALTwc8WiERFwlxTY3FyYRLaISRoREek5ezsLE1efxDVFHgDg5a7BeCu2OeylEqSnF4jcO6K6gUkaERFpqdUClu2/ivl/XkSJSoCPqyMWDI9El2Ze/+633VGZRLaGSRoRUR2kUgs4kpKB9JxC+Lg6ISrQHWk5xZjy+xEcupYBAIht6YtPBrdGg3oOIveWqG4yOkkbPXo0xo0bh27dulVHf4iIqJptO3sXMzcm425WoXabu7M9CktUKFSq4Wwvw4yBYRgeFcBnz4hEZPRwhqysLMTExKBZs2b4+OOPcfv27eroFxERVYNtZ+/i9VUndBI0AMgsKEGhUo1ATxdsmdQVI9o1YYJGJDKj76Rt2LABCoUCK1euxHfffYeEhATExMRg3LhxeOqpp2xyzjSAKw48jDNqW8eM2oaUYzzaRpvWEI8qtYAZvydDqKROUYkKjd2dqi3eGI/W0aY1xKO5ZbniQAW8vb0RHx+P+Ph4nDhxAsuXL8cLL7yA+vXr4/nnn8f48ePRrFkzUw5dY7jiQOU4o7Z1zKhtSDnGo220aQ3xePxWDlKzCyutk5pdhD9PXkPbAFeT+sF4tI02rSEezS3LFQeqcPfuXWzfvh3bt2+HTCbDE088gTNnziAsLAxz587FlClTzDl8teKKA5XjjNrWMaO2IeUYj7bRpjXEY8ldpUH1Suyc4ePjY1I/GI+20aY1xKO5ZbniQDlKSkrw+++/Y/ny5fjzzz/RunVrTJ48Gc8++6w2yVm/fj3Gjh1r1Unaw7jigD7OqG0dM2obUo7xaBttihmPEokEl/5dEL0qvm7OZs3yzni0jTZrw/WRKw48xN/fH2q1Gs888wyOHDmCyMhIvTI9evSAu7u7sYcmIqJqkF1Qgg9+T8bGU3eqLOsvd0L7YI8a6BURVcXoJG3hwoUYNmxYpbfr3N3dkZKSYlbHiIjIfCf+ycHs7edwN6sQMqkE/cP9sPH0XQDQGUAg+ff1B/1DIZNyVCeRNTA6SRs4cCDy8/P1krSMjAzY2dlpv/IkIiLxFCvVWLD9Ir7aew0CgEBPFywaEYk2TRrgidb686T5yZ0wsWtD9A33E6/TRKTD6CRt5MiRGDBgAMaPH6+z/eeff8bvv/+OLVu2WKxzRERkvKuKXExenYQzt7MAAMPaNsaMgS1Rz7H0kt833B+9w/z0Vhy4f08hZreJ6CFGJ2l///03FixYoLc9Ojoa7733nkU6RURExhMEAauP3sKsjckoKFFB7myPd3sGYGTn5noPK8ukEnRq6ql9bcvzlhHVVkYnaUVFRVAq9Ydxl5SUoKCgwCKdIiIi42TkFePddaexPTkNAPB4U0/MG9oKsiLD52QiIuti9JjT9u3bY9myZXrbly5dirZt21qkU0REZLh9lxSIXbQP25PTYC+T4L0nQrFqXAf4y53F7hoRmcHoO2kffvghYmJicOrUKfTq1QsAsHPnThw9ehR//vmnxTtYU7gslC4ue2Idy54YUo7xaBttVkc8FpWoMPfPS1j+13UAQIh3PSwaEYmwhm4ABMZjNWE8Wsf1kctClaNz5844dOgQ5s2bh59//hnOzs5o3bo1/vvf/1r9UlBlcVmoynHZE+tY9sSQcoxH22jT0vF49V4Bpm9LwdV7pY+ZDGntjTe6NoaTXSHS0w27lpnSP8Yj49HU+lwWqlS1LwsVGRmJH374wZSqVoPLQlWOy55Yx7InhpRjPNpGm5aKRy8vL6z8+xY+2XYRxUo1POs54JMhrdCrhf4yTozH6sF4tI7rI5eFqoBarcaVK1eQnp6ud9uuW7duphxSdFwWSh+XPbGOZU8MKcd4tI02zT1eRr4SU1eewN5L9wAAPZp7Y+7QCHi7OlqkTcaj4RiP1nF95LJQDzl8+DCeffZZ3LhxA4Ig6OyTSCTarw+JiMhydpxPwzu/JCOzQAlHOyne6x+KFzoGQiLh6gBEtZXRSdprr72GqKgobN68Gf7+/rxAEBFVo4JiFT7cnIwf/r4JAGjh54olz7TBo76uIveMiKqb0Una5cuX8csvvyAkJKQ6+kNERP86ezsLE1efxDVFHgDg2cd88cGgCDg72IvcMyKqCUYnaR06dMCVK1eYpBERVROVWsDX+69h/p8XUaIS4OvmiHlDW+NRNzUc7WRid4+IaojRSdobb7yBN998E6mpqWjVqhXs7XX/omvdurXFOkdEVNfcySxA/M9JOHwtAwAQ29IXnwxuDbmzHdLT00XuHRHVJKOTtCFDhgAAxo4dq90mkUggCAIHDhARmWHz6buY9utpZBcq4Wwvw4yBYRgeFQCJRGLTk78SkWmMTtJSUlKqox9ERHVWbpESCb+dw7oT/wAAIhrLsWhkGwR71RO5Z0QkJqOTtMDAwOroBxFRnXT8xgNMWZOEmxn5kEqA8dEhmBTTDPYy251DjIgsw6SrwMqVK9G5c2c0bNgQN27cAAAsWrQIv/32m0U7R0RUWylVaizecRnDvzqEmxn5aOTujNWvdMJbsc2ZoBERABOStC+//BLx8fF44oknkJmZqX0Gzd3dHYsWLbJ0/4iIap2b9/MxYtlhLNxxCSq1gKciG2LLpK5oH+whdteIyIoY/XXn559/jq+//hqDBg3CJ598ot0eFRWFt956y6Kdq0lqtVr7YK5arYYgCBU+qGvuflsgxjlYuk1zj2dKfWPqGFqW8Vh74lGtVmPd8VuYuek8cotUqO9oh1lPhWFQZCNtGUv2h/FYPWpLPNr69dHcMmLFojHtmTRwoE2bNnrbHR0dkZeXZ+zhRJOYmIjExETtnUCFQoHCwkIApW9gVlYWBEGocIFgc/bbAjHOwdJtmns8U+obU8fQsozH2hGPmfnF+PjPa9h3vfQ6GdGwPhJig9BQbm/Q1BqMR+tRG+KxNlwfzS0jVizm5OQYXNboJC04OBhJSUl6Awi2bduG0NBQYw8nmri4OMTFxSE7OxtyuRze3t5wc3MDUPrBSSQSeHt7V3iRMWe/LRDjHCzdprnHM6W+MXUMLct4tP14/PvafcSvvYy7WYWQSSWY3CsEr3VvCpnU8GX1GI/Ww9bj0RLHs4Z4NLeMWLHo5ORkcFmjk7T4+HjExcWhsLAQgiDgyJEj+OmnnzBnzhx88803xh7OakilUp0PSSKR6G0ry9z9tkCMc7B0m+Yez5T6xtQxtCzj0TbjsVipxsIdl7B071UIAtBY7oglzz6GxwJNe/aM8Wg9bDEeLX08a4hHc8uI8Tka05bRSdpLL70EZ2dnvP/++8jPz8ezzz6Lhg0bYvHixRg5cqSxhyMiqpWuKnIxeXUSztzOAgAMj2qM19p7Iaixu7gdIyKbYXSSBgDPPfccnnvuOeTn5yM3Nxc+Pj6W7hcRkU0SBAE/HbmF2ZuSUVCigtzZHp8MboXYlr5c1omIjGL0/b2ePXsiMzMTAODi4qJN0LKzs9GzZ0+Ldo6IyJZk5BXjlZXH8X/rz6CgRIXOIZ74Y3I39GvlL3bXiMgGGX0nbc+ePSguLtbbXlhYiP3791ukU0RE1kylFnAkJQPpOYXwcXVC+2AP/HXlHt5cewqKnCLYyyR4J7YFxnUJhtSIwQFERGUZnKSdPn1a+//JyclITU3VvlapVNi2bRsaNWpk2d4REVmZbWfvYubGZNzNKtRuc3GQIb+4dDqfEJ/6WDwyEi0bysXqIhHVEgYnaZGRkZBIJJBIJOV+rens7IzPP//cop0jIrIm287exeurTkB4aLsmQev+qDeWPt8Wzg6ymu8cEdU6BidpKSkpEAQBjzzyCI4cOQJvb2/tPgcHB/j4+EAm44WJiGonlVrAzI3JeglaWZfScuBgZ7vTShCRdTE4SdNMXmvLS3kQEZnqSEqGzlec5bmbVYgjKRno1NSzhnpFRLWZSVNwAKXPpd28eVNvEMHAgQPN7hQRkbVJz6k8QTO2HBFRVYxO0q5du4ann34aZ86cgUQigSCU3vyXSEpHMGnWwiQiqi0KilXYeOqOQWV9XA1f8oWIqDJGPzwxadIkBAcHIz09HS4uLjh37hz27duHqKgo7Nmzpxq6SEQknrO3s9D/8/3Ycb7yiWglAPzlpdNxEBFZgtFJ2qFDhzBr1ix4eXlp17vq0qUL5syZg4kTJ1ZHH4mIapxKLWDlsVQMWXoI1xR58HVzxKRezSBBaUJWluZ1woAwoxZNJyKqjNFfd6pUKri6ugIAvLy8cOfOHTRv3hyBgYG4ePGixTtIRFTT7mQWIH5NEg6nZAAA+rb0w5zBrdCgngNC/V315knzkzshYUAY+oZzZQEishyjk7Tw8HCcOnUKwcHB6NChA+bOnQsHBwcsW7YMjzzySHX0kYioxmw+fRfTfj2N7EIlnO2lSBgQhhHtmmifu+0b7o/eYX56Kw7wDhoRWZrRSdr777+PvLw8AMCsWbPw5JNPomvXrvD09MSaNWss3kEiopqQW6REwm/nsO7EPwCA1o3leL9XY0Q1D9AmaBoyqYTTbBBRtTM6SYuNjdX+f0hICC5cuICMjAw0aNBA70JGRGQLjt94gClrknAzIx9SCTA+OgRv9GyKB/fvid01IqrDTJ4nrSwPD45mIiLbo1Spkbj7KpbsugyVWkAjd2csHBGJ9sEenLibiERnUJI2ePBggw/466+/mtwZManVau1FWa1WQxCECi/S5u63BWKcg6XbNPd4ptQ3po6hZRmP1XMONzPy8ebPp3D8ZiYAYGCEP2YNbAk3Z3vt9YDxaHw5xqNttFkb4tHcMmLFojHtGZSkyeVykztjrRITE5GYmKidfFehUKCwsHS0llqtRlZWFgRBgFSqP0uJufttgRjnYOk2zT2eKfWNqWNoWcajZc9BEARsPZ+Bz/bcRH6xGvUcpHi7ZxP0beGJwpwHKMyxfJuWOB7j0Xrw+mgd8WhuGbFiMScnx+CyBiVpy5cvN7kz1iouLg5xcXHIzs6GXC6Ht7c33NzcAJR+cBKJBN7e3hVeZMzZbwvEOAdLt2nu8Uypb0wdQ8syHi13DlkFJfhgwzlsOnMXABAV2AALhrdG4wYu1dampY7HeLQevD5aRzyaW0asWHRyMnxVEos8k1YbaCbm1ZBIJHrbyjJ3vy0Q4xws3aa5xzOlvjF1DC3LeDT/HA5fu4/4NUm4k1UImVSCKTHN8Hp0SKVTZzAeTSvHeLSNNmtDPJpbRozP0Zi2mKQRUa1WrFRj4Y5LWLr3KgQBCPR0waIRkWjTpIHYXSMiqhSTNCKqta4qcjF5dRLO3M4CAIyICsD0AWGo58hLHxFZP16piKjWEQQBq4/ewqyNySgoUUHubI9PBrdCv1ZctomIbAeTNCKqVTLyivHuutPYnpwGAHi8qSfmD4+Av9xZ5J4RERnHoCRtyZIlBh9w4sSJJneGiMgc+y4p8ObaU1DkFMFeJsE7sS0wrkswpFxXk4hskEFJ2sKFCw06mEQiYZJGRDWusESFudsu4tu/UgAAIT71sXhkJFo2rH1zPBJR3WFQkpaSklLd/SAiMsnF1BxMWn0SF1JLJ4gc1SkQ0/qFwtlBJnLPiIjMw2fSiMgmCYKAFQevY87WCyhWquFZzwFzh7ZGr1BfsbtGRGQRBiVp8fHxBh9wwYIFJneGiMgQ6TmFeHvtaey9pAAA9GjujblDI+Dt6ihyz4iILMegJO3kyZMGHUwi4cO5RFS9tien4d11p5GRVwxHOyne6x+KFzoG8vpDRLWOQUna7t27q7sfRESVKixR4/0NZ/HjkVsAgBZ+rljyTBs86usqcs+IiKoHn0kjIqt39nYW3vgxGTceFAEAXu4ajLdim8PRjoMDiKj2MilJO3bsGH7++WfcvHkTxcXFOvt+/fVXi3SMiEilFvD1/muY/+dFlKgE+Lo5Yv6wSHRp5iV214iIqp3Ry76vXr0ajz/+OM6fP4/169ejpKQE586dw65duyCXc04iIrKMO5kFeO6bw/hk6wWUqAR0b+qOLRO7MEEjojrD6DtpH3/8MRYuXIi4uDi4urpi8eLFCA4Oxquvvgp/f66LR0Tm23z6Lqb9ehrZhUo428swfUAoogMc0MDFQeyuERHVGKPvpF29ehX9+/cHADg4OCAvLw8SiQRTpkzBsmXLLN5BIqo7couUePPnU4j78QSyC5WIaCzHlkldMSIqgKM3iajOMfpOWoMGDZCTUzqzd6NGjXD27Fm0atUKmZmZyM/Pt3gHiahuOHHzASavTsLNjHxIJEBcdAgmxTSDvUwKtVotdveIiGqc0Ulat27dsH37drRq1QrDhg3DpEmTsGvXLmzfvh29evWqjj4SUS2iUgs4kpKB9JxC+Lg64bEm7li69xqW7LoMlVpAI3dnLBwRifbBHmJ3lYhIVEYnaf/5z39QWFgIAHjvvfdgb2+PgwcPYsiQIXj//fct3kEiqj22nb2LmRuTcTerULvNXiZBiUoAADwV2RCzngqH3NlerC4SEVkNo5M0D4///XUrlUoxdepUi3aIiGqnbWfv4vVVJyA8tF2ToI3pHISEAS1rvmNERFbK6IEDZfXv3x937961VF+IqJZSqQXM3Jisl6CVte1sKlTqykoQEdUtZiVp+/btQ0FBgaX6QkS11JGUDJ2vOMtzN6sQR1IyaqhHRETWz6wkjYjIEHczDftjLj2n8kSOiKguMStJCwwMhL09H/AloopdVeTi891XDCrr4+pUzb0hIrIdZi2wfvbsWUv1g4hqGUEQsProLczamIyCEhUkQIXPpEkA+MmdOO0GEVEZJidpx48fx/nz5wEAYWFheOyxxyzWKSKybRl5xXh33WlsT04DADze1BMDIvzxf7+W/mFXNlnTrCOQMCAMMilXFSAi0jA6SUtPT8fIkSOxZ88euLu7AwAyMzPRo0cPrF69Gt7e3pbuY5Wefvpp7NmzB7169cIvv/xS4+0T0f/su6TAm2tPQZFTBHuZBO/EtsC4LsGQSiVo4OKgN0+an9wJCQPC0Deca/8SEZVldJL2xhtvICcnB+fOnUNoaCgAIDk5GaNHj8bEiRPx008/WbyTVZk0aRLGjh2L7777rsbbJqJShSUqzN12Ed/+lQIACPGpj8UjI9GyoVxbpm+4P3qH+emsONA+2IN30IiIymF0krZt2zbs2LFDm6ABpV93JiYmok+fPhbtnKGio6OxZ88eUdomIuBiag4mrT6JC6ml6/q+0DEQ//dEKJwdZHplZVIJOjX1rOkuEhHZHKNHd6rV6nJHdNrb25u0CPK+ffswYMAANGzYEBKJBBs2bNArk5iYiKCgIDg5OaFDhw44cuSI0e0QkeUJgoDlf6VgwH8O4EJqDjzrOeC/o6Mwe1B4uQkaEREZzugkrWfPnpg0aRLu3Lmj3Xb79m1MmTLFpAXW8/LyEBERgcTExHL3r1mzBvHx8UhISMCJEycQERGB2NhYpKenG90WEVlOek4hXlx+FDM3JqNYqUaP5t7YNrkbeoX6it01IqJawaQF1gcOHIigoCAEBAQAAG7duoXw8HCsWrXK6A7069cP/fr1q3D/ggUL8PLLL2PMmDEAgKVLl2Lz5s349ttvTVo3tKioCEVFRdrX2dnZAErvEGruBKrVagiCUOGdQXP32wIxzsHSbZp7PFPqG1PH0LLWGI87zqdh6rozyMgvgaOdFNP6tcALHZtAIpFUS8wwHhmP1oTxaB3xaG4ZsWLRmPaMTtICAgJw4sQJ7NixAxcuXAAAhIaGIiYmxthDVam4uBjHjx/HtGnTtNukUiliYmJw6NAhk445Z84czJw5U2+7QqFAYWHpiDO1Wo2srCwIggCpVP9mo7n7bYEY52DpNs09nin1jaljaFlrisfCEjUW77uF9WfuAQBCvJwxq18wHvF0hkKhMOvYlWE8Mh6tCePROuLR3DJixWJOTo7BZY1O0r7//nuMGDECvXv3Ru/evbXbi4uLsXr1aowaNcrYQ1bo3r17UKlU8PXV/frE19dXmyACQExMDE6dOoW8vDw0btwYa9euRadOnco95rRp0xAfH699nZ2djYCAAHh7e8PNzQ1A6QcnkUjg7e1d4UXGnP22QIxzsHSb5h7PlPrG1DG0rLXE49nbWZi85hSu3csDALzUJRhv9mkGR7vqf/aM8ch4tCaMR+uIR3PLiBWLTk6Gr6xidJI2ZswY9O3bFz4+Pjrbc3JyMGbMGIsmaYbasWOHwWUdHR3h6Oiot10qlep8SBKJRG9bWebutwVinIOl2zT3eKbUN6aOoWXFjEeVWsDX+69h/p8XUaIS4OvmiPnDItGlmZfRxzIH45HxaE0Yj9YRj+aWEeNzNKYto5M0QRAgkejPafTPP/9ALpeXU8N0Xl5ekMlkSEtL09melpYGPz8/i7ZFRPruZBYg/uckHL6WAQCIbemLTwa3RoN6DiL3jIio9jM4SWvTpg0kEgkkEgl69eoFO7v/VVWpVEhJSUHfvn0t2jkHBwe0bdsWO3fuxKBBgwCU3p7cuXMnJkyYYNG2OHBAFx+MtY4HYw0pV13xuOXMXfzf+rPILlTC2V6G6QNCMbxt42obHFAZxiPj0ZowHq0jHjlwoAxNkpSUlITY2FjUr19fu8/BwQFBQUEYMmSI4b38V25uLq5cuaJ9nZKSgqSkJHh4eKBJkyaIj4/H6NGjERUVhfbt22PRokXIy8vTjvY0VWJiIhITE6FSqQBw4MDD+GCsdTwYa0g5S8djXrEK83ffwpbz9wEAYb4umNE3GE0aOFbr4IDKMB7rbjxaI8ajdcQjBw6UkZCQAAAICgrCiBEjjHrwrTLHjh1Djx49tK81D/WPHj0aK1aswIgRI6BQKDB9+nSkpqYiMjIS27Zt0xtMYKy4uDjExcUhOzsbcrmcAwcewgdjrePBWEPKWTIeT9x8gPifk3EzowBSCfB696aY2CsE9jJx45jxWDfj0VoxHq0jHjlwoByjR4/W/v/48eMxa9YseHmZ/gBxdHQ0BEGotMyECRMs/vXmwzhwQB8fjLWOB2MNKWfufqVKjcTdV7Fk12Wo1AIauTtj4YhItA/2qPIcagrjse7Eoy1gPFpHPNb2gQNm9WrVqlXayWCJyDbdvJ+PEcsOY+GOS1CpBTwV2RBbJnW1qgSNiKguMvpOWllV3QEjIuslCAJ+PXEbCb+fQ26REq6Odpg9KByD2jQSu2tERAQzk7TahKM7dXH0knWMXjKknCn7swpK8MGGc9h05i4AICqwARYMb43GDVysMm4Zj7U7Hm0N49E64pGjO6tgzAgFa8PRnZXj6CXrGL1kSDlj95/4Jwcz/0hBWk4JZBLgpU4NMSrKD7KSXKSn5xp0njWN8Vh749EWMR6tIx45uvNfxjx3phkhae04urNyHL1kHaOXDCln6H55A08s2X0VX+27BkEAAj1csHBEBCID3A06NzExHmtfPPL6KG6btSEeObrzX+7u7uWuMlCWZiUCzZ0pW8PRnfo4esk6Ri8ZUq6q/TcfFGH2L3/j7O3SP7hGRAVg+oAw1HO0nSceGI+1Jx55fbSONmtDPNb20Z0GXaF3795tcmeISDyCIOCnIzcxe9N5FCrVkDvb45PBrdCvlb/YXSMioioYlKR17969uvtBRBaWkVeMd9edxvbk0rVvH2/qiQXDI+Ent8xE1EREVL0MStJOnz5t8AFbt25tcmeIyDL2XVLgzbWnoMgpgr1Mgtceb4RJseGws5OJ3TUiIjKQQUlaZGQkJBJJlfOi2fIzaZyCQxeHmFvHEHNDypXdX1Siwtw/L2H5X9cBACHe9bBgeGt42xUBsN2YZDzaZjya2ydrxXi0jnjkFBz/SklJMbkz1opTcFSOQ8ytY4i5IeU0+68o8jHjzxu4eq8AADA0whsTujSGg6wQmZmMR7HbrGvxyOujdbdZG+KRU3D8KzAw0OTOWCtOwVE5DjG3jiHmhpRTqVRYe0qBxL+uo1iphmc9B3w6pBV6tvAx+TysDePRduKR10fbaLM2xCOn4KhEcnIybt68ieLiYp3tAwcONPWQouIUHPo4xNw6hphrygmQ4O+UB0jPKYSPqxPaB3vgfl4R3l57Cnsv3QMA9GjujblDI+Dt6mj2eVgbxqN1xSOvj4xHa4hHTsHxkGvXruHpp5/GmTNndJ5T08yjZqvPpBFZs91XHmDxt+eQml2o3dbAxR4lKgG5RUo4yiSY9kQoRj8eVOWchkREZBuMTh0nTZqE4OBgpKenw8XFBefOncO+ffsQFRWFPXv2VEMXieq2bWdTMW3TNZ0EDQAe5Jcgt0iJRu5OWPFsKEZ1CmSCRkRUixh9J+3QoUPYtWsXvLy8tLcIu3Tpgjlz5mDixIk4efJkdfSTqE5SqQXM2nS+yjJNGnDuMyKi2sboO2kqlQqurq4AAC8vL9y5cwdA6eCCixcvWrZ3RHXckZQMvTtoD0vNLkLSbetcGJ2IiExn9J208PBwnDp1CsHBwejQoQPmzp0LBwcHLFu2DI888kh19LFGcJ40XZwHyDrmAUrLLjCo3Xt5xYxHK2+zNsSjIeV4fbSNNmtDPHKetHK8//77yMvLAwDMmjULTz75JLp27QpPT0+sWbPG2MOJhvOkVY7zAFnHPEBnrqcb1K6TUIT09HTGoxW3WRvi0ZByvD7aRpu1IR45T1o5YmNjtf8fEhKCCxcuICMjAw0aNLCph5Y5T1rlOA+QuPMA5RYpMXNjMtaduFvpMSQA/ORO6PyoH3x8fBiPVtymLcejMeV4fbSNNmtDPHKeNAN5eHhY4jCi4jxp+jgPkDjzAB2/8QBT1iThZkY+pBKgT5gvtp1LgwRA2YXZNH8SfdA/FHYyKePRBtq0xXg0pRyvj7bRZm2IR86TRkQ1QqlS4z+7r+DzXVegUgto5O6MhSMiERXojjUHL2Lxvjs6gwj85E5IGBCGPmG+SE837GtRIiKyHUzSiKzAzfv5mLzmJE7czAQAPBXZELMHhcPNyR5qtRo9QhpgaMdHcexGps6KAzKpxKYfwCYioooxSSMSkSAI+PXEbczYmIzcIiVcHe0we1A4BrVppFdWJpWgU1NPEXpJRERiYJJGJJKsghJ8sDUFOy49AAC0C2qABcMjEeDhInLPiIjIGjBJIxLB4Wv3MWVNEu5mFUImlWBKTDO8Hh0CmdR2RkgTEVH1YpL2L05mq4uTNVbPZI3FSjUW7byMr/ZdgyAAjeSOWDwyEo8FegAQoFYLenUYj4xHU+tzMtvqwXi0jnjkZLa1GCezrRwna7T8ZI03MgqRsC0FF9LzAQADwjwxJtIVfo7FlY7OZDwyHk2tz8lsqwfj0TrikZPZ1mKczLZynKzRcpM1CoKA1Udv4cPNF1BQooLc2R4fPx2O2DAfKBQKTh5qAMajdUweakg5xqNttFkb4pGT2dYhnMxWHydrNH+yxoy8Yry77jS2J6cBADqHeGL+sEj4yZ20FwhOHmoYxqN1TB5qSDnGo220WRvikZPZEpFJ9l1S4M21p6DIKYK9TIJ3YltgXJdgSDk4gIiIDMAkjcjCipRqzN58Hsv/ug4ACPGpj8UjI9GyoVzcjhERkU1hkkZkQRdTczBh9QVcvVcAABjVKRDT+oXC2UEmcs+IiMjWMEkjsgBBELDi4HXM2XoBxUo1POs5YO7Q1ugV6it214iIyEYxSSMyU3pOId5eexp7LykAAI8HuWHhM1HwlTuL3DMiIrJlTNKIzLA9OQ3vrjuNjLxiONpJMa1fC8Q+4gRvV0exu0ZERDaOSRqRCQqKVfhwczJ++PsmACDU3w1LRkaiqXe9SiemJSIiMhSTtH9xWShdXPak4vpnb2dh8ppTuHYvDwDwUpdgvNmnGRztZFyGp5owHq1jGR5DyjEebaPN2hCPXBaqFuOyUJXjsif69VVqAT+eSMNXB+9AqRbgXc8eH8QGoX0TN2Rl3De6TS7DYzjGo3Usw2NIOcajbbRZG+KRy0LVYlwWqnJc9kS3fmp2Ed5aexqHUzIAAH3CfDFncDgauDiY3CaX4TEc49E6luExpBzj0TbarA3xyGWh6hAuC6WPy56U1t92Lg3/t/4ssguVcLaXYcbAMAyPCoBEUv7KAVyGp3owHq1jGR5DyjEebaPN2hCPXBaKqI7KLVJi1h/XseV86VeZEY3lWDSyDYK96oncMyIiqguYpBGV4/iNB5iy5iRuZhRAKgHGR4dgUkwz2Mts9y9/IiKyLUzSiMpQqtRI3H0VS3ZdhkotwM/VAYtGtkHHpl5id42IiOoYJmlE/7p5Px9Tfk7C8RsPAAADI/zxRicfNG3iIXLPiIioLmKSRnWeIAj49cRtJPx+DrlFSrg62mH2oHAMjPDnxLRERCQaJmlUp2Xll+C9DWew6fRdAEBUYAMsHBGJAA8Xm55sk4iIbB+TNKoTVGoBR1IykJ5TCB9XJ7QP9sDR6xmIX5OEO1mFkEklmNyrGV6Pbgo7Dg4gIiIrwCSNar1tZ1Mxe/N53M0q1G6r5yhDXlHpahOBni5YNCISbZo0EKuLREREepikUa22+8oD/N+maxAe2q5J0Do39cSyUVGo58gfBSIisi78XodqLZVawMI9t/QStLKu3cuDk72sxvpERERkKCZpVGsdvZ6B9NySSsvczSrEkX/X4yQiIrIm/I7nX2q1WjuaT61WQxCECkf3mbvfFohxDpZuM63MM2iVlssuKLdNU/pjTB1DyzIea0c8mns8xqP1YDxaRzyaW0asWDSmvTqbpCUmJiIxMREqVemzSQqFAoWFpb/U1Wo1srKyIAhCuQuhmrvfFohxDpZss0ipxu8nbhhU1l5ZUO58aKb0x5g6hpZlPNp+PFrieIxH68F4tI54NLeMWLGYk5NjcNk6m6TFxcUhLi4O2dnZkMvl8Pb2hpubG4DSD04ikcDb27vCi4w5+22BGOdgqTYvpuZg8i+ncDG18h8ECQA/uRP6tHkEMqnEIv0xpo6hZRmPth2Pljoe49F6MB6tIx7NLSNWLDo5ORlcts4maQ+TSqU6H5JEItHbVpa5+22BGOdgTpuCIGDFweuYs/UCipVqeNZzwJOhDfD9sbTS/WXb+fe/CQPCYG9X8cABU/pjTB1DyzIebS8eq+N4jEfrwXi0jng0t4wYn6MxbTFJo1ohPacQb689jb2XFACAHs298cngVhAKstDh0YZ686T5yZ2QMCAMfcP9xeoyERFRpZikkc3bnpyGd9edRkZeMRztpHivfyhe6BgIQRCQXgD0DfdDbLi/3ooD5X3FSUREZC2YpJHNKihW4cPNyfjh75sAgBZ+rljyTBs86usKoPTrTw2ZVIJOTT1F6ScREZEpmKSRTTp7OwsTV5/ENUUeAODlrsF4K7Y5HCt5voyIiMiWMEkjm6JWC1i2/xrm/3kRJSoBvm6OmD8sEl2aeYndNSIiIotikkY2405mAd78+RQOXbsPAIht6YtPBrdGg3oOIveMiIjI8pikkU3YfPoupv16GtmFSjjbyzBjYBiGRwVAIuHD/0REVDsxSSOrllukRMJv57DuxD8AgIjGciwa2QbBXvVE7hkREVH1YpJGVuvEzQeYvDoJNzPyIZUA46NDMCmmGexltjsBJhERkaGYpJHVUarU+HLXVSzZdRkqtYBG7s5YOCIS7YM9xO4aERFRjWGSRlbldlYR4n79G8dvZgIAnopsiFlPhUPubC9ux4iIiGoYkzSyCoIgYP3J2/jgt2TkF6vh6miH2YPCMahNI7G7RkREJAomaSS6rIISvL/hLDaeugMAiApsgIUjIhHg4SJyz4iIiMTDJI1EdfjafcSvScKdrELIpBK81NEfbz3RGvZcOYCIiOo4JmkkimKlGgt3XMLSvVchCECgpwsWDo9AQ8diLnxOREQEJmkkgquKXExenYQzt7MAACOiAjB9QBic7aVIT08XuXdERETWgUnav9RqNdRqtfb/BUHQvi6vrDn7bUF1nIMgCFhz7B/M3nQeBSUqyJ3t8fHT4egX7lctbZp7PFPqG1PH0LKMR3HOgfFoWjnGo220WRvi0dwyYsWiMe3V2SQtMTERiYmJUKlUAACFQoHCwkIApW9gVlYWBEGAVKo/caq5+22Bpc8hs0CJj7dfx75rpXfPogJcMT02CD71/3f3zNJtmns8U+obU8fQsoxHcc6B8WhaOcajbbRZG+LR3DJixWJOTo7BZetskhYXF4e4uDhkZ2dDLpfD29sbbm5uAEo/OIlEAm9v7wovMubstwWWPIf9lxV465cLUOQUwV4mwdt9mmNs5yBIH3r2zNLvm7nHM6W+MXUMLct4FOccGI+mlWM82kabtSEezS0jViw6OTkZXLbOJmkPk0qlOh+SRCLR21aWufttgbnnUFiiwtxtF/HtXykAgBCf+lg8MhItG8qrrU1LH8+U+sbUMbQs41Gcc2A8mlaO8WgbbdaGeDS3jBifozFtMUmjanExNQeTVp/EhdTS27qjOgViWr9QODtwag0iIiJDMEkjixIEASsOXsecrRdQrFTDq74D5g5tjZ4tfMXuGhERkU1hkkYWk55TiLfXnsbeSwoAQI/m3pg7NALero4i94yIiMj2MEkji9iRnIZ31p1GRl4xHO2keK9/KF7oGAiJhBPTEhERmYJJGpmloFiFDzcn44e/bwIAQv3dsGRkJJr5uorcMyIiItvGJI1MdvZ2FiauPolrijwAwMtdg/FWbHM4ct1NIiIiszFJoyqp1AKOpGQgPacQPq5OiApsgP/+lYL5f15EiUqAr5sj5g+LRJdmXmJ3lYiIqNZgkkaV2nY2FbM3n8fdrELtNgeZFMWq0mUt+rb0w5zBrdCgnoNYXSQiIqqVmKRRhXZfeYD/23QNwkPbNQnaCx0DMeuplhwcQEREVA1sd7pnqlYqtYCFe27pJWhl7TifBnVlBYiIiMhkTNKoXEevZyA9t6TSMnezCnEkJaOGekRERFS3MEmjcqWWeQatMuk5hpUjIiIi4zBJIz23MvKxdO9Vg8r6uDpVc2+IiIjqJg4cIC1BELD+5G1M/+0ccouUkAAVPpMmAeAnd0L7YI8a7CEREVHdwSSNAABZBSV4f8NZbDx1BwAQFdgAPR+pj3m7bwHQTdY0YzkTBoRBJuXITiIiourAJI1w+Np9xK9Jwp2sQsikEkyJaYZXuz2C+/cUCPL30psnzU/uhIQBYegb7i9ir4mIiGo3Jml1WLFSjUU7LuHLvVchCECgpwsWj2yDyAB3qNX/TlYb7ofYcH+dFQfaB3vwDhoREVE1Y5JWR11V5GLy6iScuZ0FABgRFYDpA8JQz1E/JGRSCTo19azpLhIREdVpTNLqGEEQsProLczamIyCEhXkzvb4ZHAr9GvFry6JiIisCZO0OiQjrxjvrjuN7clpAIDOIZ6YPywSfnJOo0FERGRtmKTVEfsuKfDm2lNQ5BTBXibBO7EtMK5LMKR8toyIiMgqMUmr5QpLVJi77SK+/SsFABDiUx+LR0aiZUO5yD0jIiKiyjBJq8UupuZg0uqTuJCaAwAY1SkQ0/qFwtlBJnLPiIiIqCpM0mohQRCw4uB1zNl6AcVKNbzqO2Du0Nbo2cJX7K4RERGRgZik1TLpOYV4e+1p7L2kAAD0aO6NuUMj4O3qKHLPiIiIyBhM0mqRHclpeGfdaWTkFcPRTor3+ofihY6BkEg4OICIiMjWSMXugCVs2rQJzZs3R7NmzfDNN9+I3Z0aV1Cswnvrz+Cl748hI68Yof5u2PRGF4zqFMQEjYiIyEbZ/J00pVKJ+Ph47N69G3K5HG3btsXTTz8NT8+6MUP+2dtZmLj6JK4p8gAAr3R7BG/2eRSOdhwcQEREZMts/k7akSNH0LJlSzRq1Aj169dHv3798Oeff4rdrWqnVgtYuvcqnv7iL1xT5MHXzRGrxnXA/z0RygSNiIioFhA9Sdu3bx8GDBiAhg0bQiKRYMOGDXplEhMTERQUBCcnJ3To0AFHjhzR7rtz5w4aNWqkfd2oUSPcvn27JroumjuZBXjum7/xydYLKFEJ6NvSD9smdUOXZl5id42IiIgsRPQkLS8vDxEREUhMTCx3/5o1axAfH4+EhAScOHECERERiI2NRXp6eg331DpsPn0XfRftw6Fr9+HiIMPcIa3x5fOPoUE9B7G7RkRERBYk+jNp/fr1Q79+/Srcv2DBArz88ssYM2YMAGDp0qXYvHkzvv32W0ydOhUNGzbUuXN2+/ZttG/fvsLjFRUVoaioSPs6OzsbAKBWq6FWq7X/LwiC9vXDzN1vitwiJWZuTMa6E6XnGtFYjgXDIxDsVQ+CIEAQBIu1BVTPOdR0m+Yez5T6xtQxtKw1xmNNYzwyHq0J49E64tHcMmLFojHtiZ6kVaa4uBjHjx/HtGnTtNukUiliYmJw6NAhAED79u1x9uxZ3L59G3K5HFu3bsUHH3xQ4THnzJmDmTNn6m1XKBQoLCwEUPoGZmVlQRAESKX6NxvN3W+ss3dzkbAtBbeziiGVAKPb+WFch4awU+chPT3P7OOXx9LnIEab5h7PlPrG1DG0rLXFoxgYj4xHa8J4tI54NLeMWLGYk5NjcFmrTtLu3bsHlUoFX1/dmfJ9fX1x4cIFAICdnR3mz5+PHj16QK1W45133ql0ZOe0adMQHx+vfZ2dnY2AgAB4e3vDzc0NQOkHJ5FI4O3tXeFFxpz9hlKq1Phiz1V8vvsqVGoBjdydMX9Ya7QP9jD5mIay1DmI2aa5xzOlvjF1DC1rLfEoJsYj49GaMB6tIx7NLSNWLDo5ORlc1qqTNEMNHDgQAwcONKiso6MjHB31Z9+XSqU6H5JEItHbVpa5+6ty834+pvychOM3HgAABkU2xKxB4XBzsjfpeKYw9xysoU1zj2dKfWPqGFpW7Hi0BoxHxqM1YTxaRzyaW0aMz9GYtqw6SfPy8oJMJkNaWprO9rS0NPj5+YnUq+olCAJ+PXEbCb+fQ26REq6Odvjw6XA8Fdmo6spERERUa1h1kubg4IC2bdti586dGDRoEIDS25M7d+7EhAkTLNqWNQwcyCoowQcbzmHTmbsAgKjABlgwvDUaN3AR5cFGPhgr/oOxhpTjg9q20Sbj0fg+WSvGo3XEIwcO1IDc3FxcuXJF+zolJQVJSUnw8PBAkyZNEB8fj9GjRyMqKgrt27fHokWLkJeXpx3taarExEQkJiZCpVIBEH/gwIl/cjDzjxSk5ZRAJgFe6tQQo6L8ICvJRXp6rhlnaho+GGsdD8YaUo4PattGm4xH08/D2jAerSMeOXCgBhw7dgw9evTQvtY81D969GisWLECI0aMgEKhwPTp05GamorIyEhs27ZNbzCBseLi4hAXF4fs7GzI5XLRBg4UK9VYtPMyvtp3DYIABHq6YNHwCEQEuJt1fubig7HW8WCsIeX4oLZttMl4NP08rA3j0TrikQMHakB0dHSVc3xNmDDB4l9vPkyMgQNXFbmYvDoJZ25nAQBGRAVg+oAw1HMU/WMBwAdjTa3PB7WrB+OR8WhNGI/WEY8cOEAWJwgCVh+9hVkbk1FQooK7iz0+GdwKfcP9xe4aERERWQkmadVIpRbwd8p9pOcUwsfVCe2DPZBVUIJ3153G9uTSEaudQzwxf1gk/OSG3/4kIiKi2o9J2r8sPbpz9+UMLP72LFKz/7cEVQMXe6jUArILlXCQSfBWn+YY2zkIUqnE6kY6cfSSdYxeMqQcR9PZRpuMR+P7ZK0Yj9YRjxzdWYtV5+jOXZcy8H9bUvS2P8gvAQB417fH/IEheNTHBffuKSx5WhbD0UvWMXrJkHIcTWcbbTIeTT8Pa8N4tI545OjOWqy6Rneq1AKWfHu20rZlUik6hQVCJpVY7oQsjKOXrGP0kiHlOJrONtpkPJp+HtaG8Wgd8cjRnXWIpUZ3/p1yX+crzvKkZhfh2I1MdGpa8Rqj1oCjl6xj9JIh5TiazjbaZDwa3ydrxXi0jnis7aM7bfcnxEql5xRatBwRERHVTUzSLMzH1bDbmIaWIyIiorqJSZqFtQ/2gJ9bxQmYBIC/vHQ6DiIiIqKK8Jm0f1lqCg4JgA/6t0DcT0mQABAe2gcAH/QPhQQC1OrKV1oQE4eYW8cQc0PKccoD22iT8Wh8n6wV49E64pFTcNRi1TkFRxtvCd6P9sWyYxlIzy3Rbvepb4/J0QF4zEeK9PT0ajozy+AQc+sYYm5IOU55YBttMh5NPw9rw3i0jnjkFBy1WHUvsN4/QoIXekbg+M1MpOcUwcfVEe2CPKx62o2yOMTcOoaYG1KOUx7YRpuMR9PPw9owHq0jHjkFRx1iqSk4yu63t5Ph8RDvaulvTeAQc+sYYm5IOU55YBttMh6N75O1YjxaRzxyCg4iIiIiqnFM0oiIiIisEJM0IiIiIivEJI2IiIjICnHgwL8sNU+aIfttAecBso55gAwpx3i0jTYZj8b3yVoxHq0jHjlPWi1WnfOkcR4g62izNswDZEg5xqNttMl4NP08rA3j0TrikfOk1WLVPU8a5wESv83aMA+QIeUYj7bRJuPR9POwNoxH64hHzpNWh1THPGmcB0j8NmvDPECGlGM82kabjEfj+2StGI/WEY+cJ42IiIiIalydv5MmCKWLnGdnZ2u3qdVq5OTkwMnJqcLb9ebstwVinIOl2zT3eKbUN6aOoWUZj4xHU+szHqsH49E64tHcMmLFoibf0OQflanzSZrmAb6AgACRe0JERER1RU5ODuRyeaVlJIIhqVwtplarcefOHbi6ukIi+d/i5+3atcPRo0crrFfZ/uzsbAQEBODWrVvawQi2qKr3wBbaNPd4ptQ3po6hZRmPjEdT6zMeqwfj0Tri0ZwyYsWiIAjIyclBw4YNq7yDV+fvpEmlUjRu3Fhvu0wmq/RDq2o/ALi5udn0RciQc7T2Ns09nin1jaljaFnGI+PR1PqMx+rBeLSOeLREGTFisao7aBq2+UBADYiLizNrf20gxjlauk1zj2dKfWPqGFqW8ch4NLU+47F6MB6tIx4tVcZa1fmvO6uDZu61rKwsm/5LkWoHxiNZE8YjWQtbiEXeSasGjo6OSEhIgKOjo9hdIWI8klVhPJK1sIVY5J00IiIiIivEO2lEREREVohJGhEREZEVYpJGREREZIWYpBERERFZISZpRERERFaISVoNS0lJQY8ePRAWFoZWrVohLy9P7C5RHXXx4kVERkZq/zk7O2PDhg1id4vqsIULF6Jly5YICwvDxIkTDVqAmqi6fPbZZ2jZsiXCw8OxatUqUfrAKThqWPfu3fHhhx+ia9euyMjIgJubG+zs6vzqXCSy3NxcBAUF4caNG6hXr57Y3aE6SKFQoGPHjjh37hzs7e3RrVs3fPbZZ+jUqZPYXaM66MyZMxg9ejQOHjwIQRDQo0cPbNu2De7u7jXaD95Jq0Gai0/Xrl0BAB4eHkzQyCr8/vvv6NWrFxM0EpVSqURhYSFKSkpQUlICHx8fsbtEddT58+fRqVMnODk5wdnZGREREdi2bVuN94NJmhH27duHAQMGoGHDhpBIJOV+NZSYmIigoCA4OTmhQ4cOOHLkiHbf5cuXUb9+fQwYMACPPfYYPv744xrsPdU25sZjWT///DNGjBhRzT2m2szcePT29sZbb72FJk2aoGHDhoiJiUHTpk1r8AyoNjE3HsPDw7Fnzx5kZmbiwYMH2LNnD27fvl2DZ1CKSZoR8vLyEBERgcTExHL3r1mzBvHx8UhISMCJEycQERGB2NhYpKenAyj9K3H//v344osvcOjQIWzfvh3bt2+vyVOgWsTceNTIzs7GwYMH8cQTT9REt6mWMjceHzx4gE2bNuH69eu4ffs2Dh48iH379tXkKVAtYm48ap6L7NmzJwYPHoyOHTtCJpPV5CmUEsgkAIT169frbGvfvr0QFxenfa1SqYSGDRsKc+bMEQRBEA4ePCj06dNHu3/u3LnC3Llza6S/VLuZEo8a33//vfDcc8/VRDepjjAlHn/++Wdh/Pjx2v1z584VPv300xrpL9Vu5lwfNcaNGyds2rSpOrtZLt5Js5Di4mIcP34cMTEx2m1SqRQxMTE4dOgQAKBdu3ZIT0/HgwcPoFarsW/fPoSGhorVZarFDIlHDX7VSdXNkHgMCAjAwYMHUVhYCJVKhT179qB58+ZidZlqMUOvj5q7ahcvXsSRI0cQGxtb433lU+sWcu/ePahUKvj6+ups9/X1xYULFwAAdnZ2+Pjjj9GtWzcIgoA+ffrgySefFKO7VMsZEo8AkJWVhSNHjmDdunU13UWqQwyJx44dO+KJJ55AmzZtIJVK0atXLwwcOFCM7lItZ+j18amnnkJWVhbq1auH5cuXizLQj0laDevXrx/69esndjeIAAByuRxpaWlid4MIAPDRRx/ho48+ErsbRACg962DGPh1p4V4eXlBJpPp/cJLS0uDn5+fSL2iuorxSNaE8UjWxJbikUmahTg4OKBt27bYuXOndptarcbOnTs5GSPVOMYjWRPGI1kTW4pHft1phNzcXFy5ckX7OiUlBUlJSfDw8ECTJk0QHx+P0aNHIyoqCu3bt8eiRYuQl5eHMWPGiNhrqq0Yj2RNGI9kTWpNPNb4eFIbtnv3bgGA3r/Ro0dry3z++edCkyZNBAcHB6F9+/bC4cOHxesw1WqMR7ImjEeyJrUlHrl2JxEREZEV4jNpRERERFaISRoRERGRFWKSRkRERGSFmKQRERERWSEmaURERERWiEkaERERkRVikkZERERkhZikEREREVkhJmlEZJIXX3wRgwYNqrbj79mzBxKJBJmZmQCAFStWwN3dvdra0wgKCsKiRYuqvZ2KREdHY/LkydXaRnFxMUJCQnDw4EEAwPXr1yGRSJCUlFSt7ZqiY8eOWLdundjdIBIFkzSiWsLUX+41kRRYo4qSvqNHj+KVV16p+Q7969dff8Xs2bOrtY2lS5ciODgYjz/+uEWPK5FIsGHDBose8/3338fUqVOhVqstelwiW8AkjYioDG9vb7i4uIjWvoeHB1xdXavt+IIg4D//+Q/GjRtXbW1YUr9+/ZCTk4OtW7eK3RWiGsckjagWePHFF7F3714sXrwYEokEEokE169fBwDs3bsX7du3h6OjI/z9/TF16lQolcpK66lUKowbNw7BwcFwdnZG8+bNsXjxYqP79ddffyE6OhouLi5o0KABYmNj8eDBAwBAUVERJk6cCB8fHzg5OaFLly44evSoUcf/7bff8Nhjj8HJyQmPPPIIZs6cqT03AMjMzMSrr74KX19fODk5ITw8HJs2bcKePXswZswYZGVlac97xowZAHS/7nz22WcxYsQInTZLSkrg5eWF77//HgCgVqsxZ84c7XsVERGBX375pdJ+f/HFF2jWrBmcnJzg6+uLoUOHaveVvbOp+cr34X8vvviiwe/Bw44fP46rV6+if//+evsuXLiAxx9/XPte7d27F0BpYhcSEoLPPvtMp3xSUhIkEgmuXLmCoKAgAMDTTz8NiUSifV1VHwVBwIwZM9CkSRM4OjqiYcOGmDhxorauTCbDE088gdWrV1f6nhLVSqIu705EFpGZmSl06tRJePnll4W7d+8Kd+/eFZRKpfDPP/8ILi4uwvjx44Xz588L69evF7y8vISEhIRK6xUXFwvTp0////buPybq+o8D+JMfd8cBd4Bcvwg5NBAkwx8rBK7BSklmsVZ0OmsOS/+Q89dqltofgpu21Am0xmhuCfPXsnJOIULkl7Jz4S3DRO1cJyY0G2msuoDjunt+/3B+8uLwaMuvjF6PjT94v3m936/P+267196fe3+gzWbjlStXuH//foaHh/PQoUPKnEVFRXzxxRdHzembb76hRqNhcXExOzs72dXVxQ8//JA///wzSXLt2rWMi4tjfX09L1y4wKKiIsbExPDmzZskydbWVgJgf38/SbK6uppRUVHK+KdOnaJer2dNTQ0dDgcbGxuZmJjI0tJSkqTH42FmZiYff/xxNjY20uFwsLa2lvX19XS5XKyoqKBer1eu+/fffydJGo1GlpeXkyTr6uqo1WqVPpKsra2lVqvlb7/9RpLcunUrU1NT2dDQQIfDwerqamo0Gra1tfldF5vNxpCQEB48eJBXr17l2bNn+cEHHyj9ubm5XLduHUnS5XIp+V2/fp0tLS0MCwvjxx9/PKY18KesrIypqak+bd3d3QTA+Ph4fv7557x48SJXrFhBnU7HGzdukCS3bdvGtLQ0n7i1a9cyJyeHJNnX10cArK6u5vXr19nX1zemHD/77DPq9XrW19fzhx9+YEdHB3fv3u0zT1VVFY1G46jXJMREJUWaEBPEnR/ut7377rtMSUmh1+tV2iorKxkZGUmPxzNqnD+rVq1iYWGh8nugIm3JkiU0mUx++5xOJ1UqFQ8cOKC0DQ8PMy4ujjt27CAZuEibN28e33vvPZ9x9+3bx0ceeYQkefz4cQYHB9Nut/vN4e/j3XZnkeZ2u2kwGLh3716f61q8eDFJcmhoiOHh4Tx9+rTPGMuXL+eSJUv8znv48GHq9XqlyPu70V6PGzducOrUqbRYLEpboDXwZ926dXz22Wd92m4Xae+//77S5na7GR8fz+3bt5Mkf/zxR4aEhLCjo4PkrdfLYDCwpqZGiQHAI0eO+IwdKMddu3Zx2rRpHB4eHjXno0ePMjg4WHnPCvFfEXo/d/GEEPfWpUuXkJWVhaCgIKXNZDLB6XSit7cXCQkJo8ZWVlZiz549uHbtGgYHBzE8PIxZs2aNee7Ozk6YzWa/fQ6HA263GyaTSWlTqVTIyMjApUuXxjT+uXPnYLVasW3bNqXN4/FgaGgIAwMD6OzsRHx8PKZNmzbmnP8uNDQUixYtwoEDB7B06VL88ccfOHr0qHLr7fvvv8fAwADy8vJ84oaHhzF79my/Y+bl5cFoNGLq1KnIz89Hfn4+Xnrppbt+D87tdqOwsBBGo9HntnOgNfA35uDgIMLCwvzOk5WV5XPtTz75pPJ6xMXF4fnnn8eePXuQkZGB2tpauFyuUV/jseZoNptRUVGhrMfChQtRUFCA0NC/Pp60Wi28Xi9cLhe0Wu1d5xNiIpEiTQgxwieffIL169dj165dyMrKgk6nw86dO9HR0THmMe71h6nT6cSWLVvw8ssvj+gLCwv71+Z/7bXXkJubi76+Ppw4cQJarRb5+flKDgDwxRdf4NFHH/WJ02g0fsfT6XQ4e/Ys2tra0NjYiM2bN6O0tBQ2m23UR4wUFxejp6cHZ86c8SleAq2BPwaDAefPnw943f6sWLECS5cuRXl5Oaqrq7F48eKAhywC5Th58mTY7XY0NTXhxIkTsFgs2LlzJ06ePAmVSgUA+OWXXxARESEFmvjPkSJNiAlCrVbD4/H4tE2fPh2HDx8GSWU3zWq1QqfTIT4+ftQ4q9WK7OxsWCwWpc3hcPyjfNLT09Hc3IwtW7aM6HvsscegVqthtVphNBoB3NotstlsY34cyJw5c2C325GUlDTq/L29vbh8+bLf3TR/1+1PdnY2Jk+ejEOHDuHLL7+E2WxWioe0tDRoNBpcu3YNubm5Y8obuLVLNX/+fMyfPx8lJSWIjo5GS0uL30KmrKwMn376KU6fPo3Y2FifvkBr4M/s2bNRVVXl85647auvvkJOTg4A4M8//8TXX3+N1atXK/0LFy5EREQEqqqq0NDQgFOnTvnEq1SqEWs6lhy1Wi0KCgpQUFCAVatWITU1FefPn8ecOXMAAF1dXaPuTAoxkUmRJsQEkZiYiI6ODly9ehWRkZGYNGkSLBYLKioqsGbNGqxevRp2ux0lJSV46623EBwcPGpccnIy9u7di+PHj2PKlCnYt28fbDYbpkyZMuZ8Nm3ahCeeeAIWiwUrV66EWq1Ga2srzGYzDAYDiouL8fbbb2PSpElISEjAjh07MDAwMOZHQ2zevBkvvPACEhIS8MorryA4OBjnzp1DV1cXtm7ditzcXOTk5KCwsBBlZWVISkrCd999h6CgIOTn5yMxMRFOpxPNzc2YOXMmwsPDR90VevXVV/HRRx/h8uXLaG1tVdp1Oh3Wr1+PN998E16vF08//TR+/fVXWK1W6PV6FBUVjRirrq4OV65cQU5ODmJiYlBfXw+v14uUlJQRf9vU1IR33nkHlZWVMBgM+OmnnwDcKmqioqICroE/zzzzDJxOJy5cuIAZM2b49FVWViI5ORnTp09HeXk5+vv78cYbbyj9ISEhWLZsGTZt2oTk5GSf26PArfdSc3MzTCYTNBoNYmJiAuZYU1MDj8eDuXPnIjw8HPv374dWq1WKdwBob2/Hc8895/d6hJjQ7veX4oQQ/w673c7MzExqtVoCYHd3N0myra2NTz31FNVqNR9++GFu2LCBbrf7rnFDQ0NctmwZo6KiGB0dzeLiYm7cuJEzZ85U4gIdHLg9d3Z2NjUaDaOjo7lgwQLlIMDg4CDXrFlDg8FAjUZDk8nEM2fOKLGBDg6QZENDA7Ozs6nVaqnX65mRkeFzMvDmzZt8/fXXGRsby7CwMM6YMYN1dXVK/8qVKxkbG0sAyonXOw8O3Hbx4kUCoNFo9DmEQZJer5cVFRVMSUmhSqXiAw88wAULFvDkyZN+16S9vZ25ubmMiYmhVqtlenq6z6nZOw8OlJSUEMCIn6KiojGvgT+LFi3ixo0bld9vHxw4ePAgMzIyqFarmZaWxpaWlhGxDoeDAJQDHnc6duwYk5KSGBoa6nMa8245HjlyhHPnzqVer2dERAQzMzPZ1NSkxPb29lKlUrGnp+eu1yTERBREkvenPBRCCHE/fPvtt8jLy4PD4UBkZOQ/im1vb8e8efPQ09ODhx566B5l+JcNGzagv78fu3fvvudzCTHeyMNshRDiPyY9PR3bt29Hd3f3mGNcLhd6e3tRWloKs9n8fynQAODBBx+85/8mS4jxSnbShBBCBFRTU4Ply5dj1qxZOHbs2IjTrEKIf58UaUIIIYQQ45Dc7hRCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIekSBNCCCGEGIf+B2YYuW17ct0CAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(7, 5))\n", + "ax.loglog(sweep[\"collective_size_bytes\"], sweep[\"model_latency_s\"] * 1e6, \"o-\",\n", + " label=f\"ISL model ({LINK_BW_GBPS:g} GB/s links)\")\n", + "ax.set_xlabel(\"total collective size (bytes)\")\n", + "ax.set_ylabel(\"all-to-all latency (µs)\")\n", + "ax.set_title(f\"{n}-GPU fully-connected NVLink all-to-all (ISL network model)\")\n", + "ax.grid(True, which=\"both\", alpha=0.3)\n", + "ax.legend() # ASTRA-sim 2.0 and EC2 nccl-tests series overlay here later\n", + "plt.show()" + ] } ], "metadata": { @@ -37,8 +590,16 @@ "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.13.5" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" } }, "nbformat": 4, From de426ae0a6cbb3ddb996e92dad4650ba02bcb555 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Tue, 30 Jun 2026 13:23:11 -0400 Subject: [PATCH 09/19] first pass at xy and fc --- .../isl/distributed/distributed_buffers.py | 454 ++++++++++++++++++ .../astrasim2_correlation/correlation.ipynb | 21 +- 2 files changed, 471 insertions(+), 4 deletions(-) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index 35d87cb0..3d1e8a6c 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -18,6 +18,7 @@ TransferModel, ) +from typing import Optional def identify_mesh_casts( src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map @@ -259,3 +260,456 @@ def _cost_mesh_cast_hypercube(self, mcns: isl.Map) -> int: # Return the hypercube cost as a piecewise polynomial. return hypercube_costs.sum() + + +class FullyConnectedMulticastModel(TransferModel): + """ + Multicast cost model for a fully-connected fabric (e.g. an NVSwitch-style + all-to-all interconnect). + + On a fully-connected fabric every cross-node delivery costs exactly one hop + regardless of distance, and a self-delivery (source node == destination node) + costs zero. The cost of a mapping is therefore the number of deliveries that + actually traverse the fabric: + + cost = | { (data, dst, src) in mcs : dist_fn(dst, src) >= 1 } | + + This is distance-independent in magnitude (one hop per crossing); ``dist_fn`` + is used only to tell self-deliveries (0 hops) apart from fabric-crossing ones. + + See Also + -------- + HypercubeMulticastModel : + Worst-case convex-hypercube cost. On a one-hot fully-connected encoding it + overestimates all-to-all traffic ~3x relative to this model (e.g. 168 vs + 56 hops for an 8-GPU all-to-all), because each unicast is costed as a + (1 + 1)(1 + 1) - 1 = 3 hypercube instead of a single crossing. + """ + + def __init__(self, dist_fn: isl.Map): + """ + Initializes the model with the distance function over the metric space. + + Parameters + ---------- + dist_fn: + A distance function { [dst -> src] -> [hops] }. Only used to classify a + delivery as self (0 hops) vs. fabric-crossing (>= 1 hop); the hop + magnitude does not enter the cost. + """ + self.dist_fn = dist_fn + + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + """ + Given a buffer, its fills across time, and its occupancies across time, + calculate the spatial transfers on a fully-connected fabric. + + Parameters + ---------- + buff: + The buffer whose spatial analysis is being considered. Unused; the + topology is captured entirely by ``dist_fn``. + fills: + The fill of `buffer` across time from parents. + occs: + The occupancy of `buffer` across time. + + Returns + ------- + A TransferInfo whose `hops` is the number of fabric-crossing deliveries. + """ + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + result: isl.PwQPolynomial = self._cost_fully_connected(mcs) + + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), + hops=result, + link_transfer=True, + ) + + def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: + """ + Count the deliveries in `mcns` that traverse the fabric. + + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. + + Returns + ------- + The number of (data, dst, src) deliveries with dist_fn(dst, src) >= 1, as a + piecewise quasi-polynomial (constant when there are no parameters). + """ + # [dst -> src] pairs that actually traverse the fabric (>= 1 hop). + crossing_hops: isl.Set = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" + ) + crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() + crossing: isl.Map = mcns.intersect_range(crossing_pairs) + return crossing.wrap().card() + + +class XYRoutingMulticastModel(TransferModel): + """ + Multicast cost model for XY (dimension-order) routing on a 2-D mesh. + + XY routing constrains every packet to travel along the X dimension first and + only then along the Y dimension, so a multicast from one source forms a rigid + tree: + + 1. an X segment along the source's row, reaching every column that holds a + destination, and + 2. an independent Y segment down each of those columns, starting from the + source's row. + + The hop cost of one such tree (source ``s = (xs, ys)`` with destination set + ``D``) is therefore:: + + x_extent({xs} u {xd : (xd, yd) in D}) + + sum over destination columns xd of + y_extent({ys} u {yd : (xd, yd) in D}) + + Because the Y segments restart from the source row in every column rather than + sharing a trunk, this is an upper bound on free (any-monotone-path) routing and + a lower bound on the hypercube model, giving the ordering:: + + extent_DOR (floor) <= XY routing <= hypercube + + For example, source ``(1, 0)`` casting to ``(0, 2)`` and ``(2, 2)`` costs 4 + (floor), 6 (XY), and 8 (hypercube) respectively. + + Source selection is per destination: ``identify_mesh_casts`` pairs each + destination with its nearest source (devolving ties), and destinations that + share a source form one tree; the cost sums over all such trees and all data. + + Preconditions + ------------- + The NoC is two-dimensional, ``noc[x, y]`` (no temporal dimensions in the + spacetime), with ``x`` routed before ``y``. N-dimensional dimension-order + routing is a future extension. The returned cost is a parameter-free constant + (the validated regime); parametric spacetimes are not yet supported. + + See Also + -------- + HypercubeMulticastModel : + Reaches every node in the bounding box -- an upper bound on XY routing. + """ + + def __init__(self, dist_fn: isl.Map): + """ + Parameters + ---------- + dist_fn: + A distance function { [dst -> src] -> [hops] } used to pick each + destination's nearest source (Manhattan, like the hypercube model). + """ + self.dist_fn = dist_fn + + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + """ + Given a buffer, its fills across time, and its occupancies across time, + calculate the XY-routing spatial transfers. + + Parameters + ---------- + buff: + The buffer whose spatial analysis is being considered. Unused; the + topology is captured by ``dist_fn`` and the ``noc[x, y]`` coordinates. + fills: + The fill of `buffer` across time from parents. + occs: + The occupancy of `buffer` across time. + + Returns + ------- + A TransferInfo whose `hops` is the total XY-routing link count. + """ + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + result: isl.PwQPolynomial = self._cost_xy(mcs) + + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), + hops=result, + link_transfer=True, + ) + + def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: + """ + Total XY-routing link count for the multicast networks `mcns`. + + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`, + grouped per destination by nearest source. + + Returns + ------- + The X-phase links (per source row) plus Y-phase links (per destination + column), as a constant piecewise quasi-polynomial. + """ + ctx = isl.DEFAULT_CONTEXT + + # X-phase: horizontal links along each source row. The x-extent of + # {dsts} u {src} per (data, src) is exactly calculate_extents_per_dim()'s + # first (x) dimension; summing it counts every X link. + x_extent: isl.PwAff = calculate_extents_per_dim(mcns)[0] + x_links: isl.Val = self._eval_const( + isl.PwQPolynomial.from_pw_aff(x_extent).sum() + ) + + # Y-phase: vertical links per (data, src, destination column), each column + # spanning from the source row ys to the destinations in that column. + # { [data -> src] -> [dst noc[x, y]] } + per_src: isl.Map = mcns.range_reverse().uncurry() + # Split each destination into its column and its y-coordinate, keying y by + # column: { [data -> src -> col[x]] -> [yv[y]] }. + split_col: isl.Map = isl.Map.read_from_str( + ctx, "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }" + ) + dst_y: isl.Map = per_src.apply_range(split_col).uncurry() + # Inject the source row ys into every destination column so each column's + # Y segment starts from the source. + src_y: isl.Map = per_src.domain().unwrap().range_map().apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }") + ) + src_row: isl.Map = ( + dst_y.domain().unwrap().range_product(src_y).uncurry() + ) + # { [data -> src -> col] -> [yv[y]] }: all y-positions touched in a column. + col_ys: isl.Map = dst_y.union(src_row) + + # Count the links {ymin <= p < ymax} in each column via cardinality (robust + # where summing a min/max polynomial is not). + ge_min: isl.Map = isl.Map.read_from_str( + ctx, "{ yv[ymin] -> p[t] : t >= ymin }" + ) + lt_max: isl.Map = isl.Map.read_from_str( + ctx, "{ yv[ymax] -> p[t] : t < ymax }" + ) + links: isl.Map = col_ys.lexmin().apply_range(ge_min).intersect( + col_ys.lexmax().apply_range(lt_max) + ) + y_links: isl.Val = self._eval_const(links.wrap().card()) + + # Total links as a parameter-free constant. + total: isl.Val = x_links.add(y_links) + zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) + return isl.PwQPolynomial.from_qpolynomial( + isl.QPolynomial.val_on_domain(zero_dim, total) + ) + + @staticmethod + def _eval_const(pwq: isl.PwQPolynomial) -> isl.Val: + """Evaluate a parameter-free piecewise quasi-polynomial to its value.""" + return pwq.eval(isl.Point.zero(pwq.domain().get_space())) + + +class StarMulticastModel(TransferModel): + """ + Does distributed multicasting assuming all nodes are connected to a central node. + """ + + def __init__(self, reindexer: Optional[isl.Map] = None): + """ + No distance function as hops for a star model are assumed to be 1 to and from center to any node, and + all data must route through the center. + + Parameters + ---------- + reindexer: + flattens an input so that 0 (or the lexmin across all dimensions) is the assumed center everything + connects to. + """ + self.reindexer: Optional[isl.Map] = reindexer + + + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + """ + Given a buffer, its fills across time, and its occupancies across time, + calculate the spatial transfers." + + Parameters + ---------- + buff: + The buffer whose spatial analysis is being considered. Currently, + we rely on dist_fn to deal with this rather than buffer. + fills: + The fill of `buffer` across time from parents. + occs: + The occupancy of `buffer` across time. + + Returns + ------- + Fills that were fulfilled, Fills that were unfilled, and parent reads per + position in spacetime. Then, gets hops per timestep. + """ + if self.reindexer: + occs_map = isl.apply_domain(self.relabeler) + fills_map = isl.apply_domain(self.relabeler) + else: + occs_map = occs.map + fills_map = fills_map + + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + result: isl.PwQPolynomial = self._cost_star_multicast(mcs) + + # TODO: Read once from all buffers, assert that + # card(mcs) == tensor_size * duplication factor + n_meshcasts: int = mcs.card() + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), + hops=result, + link_transfer=True, + ) + + def classify_src_dst(rel: isl.Map | isl.Set): + """ + Build a quasi-affine classifier over a wrapped relation [src -> dst]. + + f([src -> dst]) = 0 if src == dst + = 1 elif src is the lexmin OR dst is the lexmin + = 2 otherwise + + Works for any dimensionality: `src`/`dst` may be scalars (1-D) or + tuples (n-D). The only requirement is that the src space and the dst + space match. + + Parameters + ---------- + rel: + isl.Map ({ src -> dst }, with src-space == dst-space) + or isl.Set ({ [src -> dst] }, i.e. already wrapped). + The lexmin of src-space is the center of the star. + + Returns + ------- + isl.PwAff defined on the wrapped space [src -> dst]. + + Preconditions + ------------- + lexmin is unique. + """ + # Accept either a relation or an already-wrapped set. + m = rel.unwrap() if isinstance(rel, isl.Set) else rel + + assert m.dim(isl.dim_type.in_) == m.dim(isl.dim_type.out), \ + "src and dst must share the same space" + + space_set = m.domain() + + # Position-wise identity src -> dst. Used to (a) select the diagonal + # src == dst, and (b) carry the lexmin point from the domain (src) + # space into the range (dst) space, so tuple names need not match. + ident = isl.Map.identity(m.get_space()) + + lex_src = space_set.lexmin() # lexmin point in the src space + lex_dst = lex_src.apply(ident) # same point, in the dst space + + # The three regions, as relations src -> dst (all subsets of m): + eq = m.intersect(ident) # src == dst + is_lexmin = m.intersect_domain(lex_src).union( # src == lexmin + m.intersect_range(lex_dst)) # or dst == lexmin + + # Move everything into the wrapped [src -> dst] set space: + W = m.wrap() + R0 = eq.wrap() + R1 = is_lexmin.wrap().subtract(R0) # the "elif": drop src==dst + R2 = W.subtract(R0).subtract(R1) # everything else + + # A constant quasi-affine piece with value `c` on the given domain. + def const_on(domain, c): + ls = isl.LocalSpace.from_space(domain.get_space()) + val = isl.Val.int_from_si(domain.get_ctx(), c) + aff = isl.Aff.zero_on_domain(ls).set_constant_val(val) + return isl.PwAff.from_aff(aff).intersect_domain(domain) + + # Disjoint domains, so union_add is just a disjoint union of pieces. + return (const_on(R0, 0) + .union_add(const_on(R1, 1)) + .union_add(const_on(R2, 2))) + + + def _pairing(src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map): + """ + Given srcs with data, fills to destinations, and a distance function, identify per data + the srcs delivering that data to dsts. + + Parameters + ---------- + src_occupancy: + An isl.Map of the form { [src] -> [data] } corresponding to the data held + at the buffer at space `src`. + dst_fill: + An isl.Map of the form { [dst] -> [data] } corresponding to the data requested + at the element at space `dst`. + + Returns + ------- + { [data] -> [dst -> src] } where { [dst] -> [data] } and { [src] -> [data] } are in + `src_occupancy` and `dst_fill` respectively, and where `[dst -> src]` is the infimum of + `dst_fn(src, dst), ∀ src, dst s.t. { [src] -> [data] } ∈ `src_occupancy` and + `{ [dst] -> [data] }` ∈ `dst_fill`. + + Preconditions: + No duplication of data. + """ + # Makes { [dst -> data] -> [dst -> data] } + fill_to_fill: isl.Map = dst_fill.wrap().identity() + if DUMP_ISL_IR: + logging.info(f"fill_to_fill: {fill_to_fill}") + + # Inverts src_occupancy s.t. data -> src. + # i.e. { [xs, ys] -> [d0, d1] } to { [d0, d1] -> [xs, ys] } + data_presence: isl.Map = src_occupancy.reverse() + + # { [dst -> data] -> [dst -> src] } where src contains data. + fills_to_matches: isl.Map = ( + fill_to_fill.uncurry() # { [[dst -> data] -> dst] -> data } + .apply_range(data_presence) # { [[dst -> data] -> dst] -> src } + .curry() + ) # { [[dst -> data] -> [dst -> src] } + if DUMP_ISL_IR: + logging.info(f"fills_to_matches: {fills_to_matches}") + + # Calculates the distance of a fill to the nearest src satisfying the fill. + # { [dst -> data] -> [dist] } + fill_min_dist: isl.Map = fills_to_matches.apply_range(dist_fn).lexmin() + # Isolates the relevant minimal pairs. + # { [dst -> data] -> [dst -> src] :.dst -> src is minimized distance } + minimal_pairs: isl.Map = ( + fill_min_dist.apply_range( + # Note: Need to match fill -> min_dist with min_dist -> [fill -> match] as lexmin over + # fill and match will minimize distance over the tuple (src, dst, data), but that + # overconstrains the optimization as we want to minimize over distance (dst, data) + # only for all src. + fills_to_matches.range_map() + .apply_range(dist_fn) + .reverse() + ) + .range() + .unwrap() + ) + if DUMP_ISL_IR: + logging.info(f"minimal_pairs: {minimal_pairs}") + + # Isolates the multicast networks. + # { [data] -> [dst -> src] : dst -> src is minimized distance } + multicast_networks: isl.Map = minimal_pairs.curry().range().unwrap() + # Devolves to a single source if multiple sources per domain point. + multicast_networks = multicast_networks.uncurry().lexmin().curry() + + return multicast_networks + + + def _cost_star_multicast(self, mcs): + """ + """ + raise NotImplementedError("WIP: star multicast cost not yet implemented") \ No newline at end of file diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index 047f3531..06aa15c2 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "a2a2df5f", "metadata": { "execution": { @@ -263,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "6d5e39c9", "metadata": { "execution": { @@ -513,10 +513,23 @@ "\n", "sweep = pd.DataFrame(rows, columns=[\n", " \"nodes\", \"collective_size_bytes\", \"per_rank_bytes\", \"per_gpu_wire_bytes\",\n", - " \"hops_per_element\", \"fabric_chunks_per_element\",\n", + " \"hops_per_tile_coordinate\", \"fabric_chunks_per_tile_coordinate\",\n", " \"model_latency_s\", \"algbw_GBps\", \"busbw_GBps\",\n", "])\n", - "sweep" + "sweep\n", + "\n", + "# TODO: Needs to account for links sharing the same \\\n", + "# Currently: doing an ALL GATHER, not all to all.\n", + "# All to all: reshufflling tile shards to be on 1 gpu.\n", + "# Hops per element * per rank byte (document somewhere)\n", + "\n", + "# NOTE: Can think of nvlink as one hop. 0,0 as router. Every time you want to send something, have to go through 0,0.\n", + "# NOTE: How do we route for star routing? Instead of figuring out all routings, should we have a set of routing choices?\n", + "# NOTE: We support various different topologie. However, you have to specify the routing chosen for multicast?\n", + "# NOTE: It is a simple multicast.\n", + " # What if that's the contribution? For regular workloads, superimposing multicast trees would balance traffic.\n", + " # Don't need fancy things (cycle accurate simulation), and still account for congestion (bw pressure on links).\n", + " # Hard to frame contribution in terms of coordinates" ] }, { From 27c1afd6d38cbf3484904a66bc3f025af4597082 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Tue, 30 Jun 2026 13:23:30 -0400 Subject: [PATCH 10/19] draft new files --- .../_looptree/reuse/isl/distributed/README.md | 448 ++++++++++++++++++ .../fully_connected/test_cases.yaml | 103 ++++ .../distribuffers/test_fully_connected.py | 90 ++++ .../distribuffers/test_xy_routing.py | 93 ++++ .../distribuffers/xy_routing/test_cases.yaml | 83 ++++ 5 files changed, 817 insertions(+) create mode 100644 accelforge/model/_looptree/reuse/isl/distributed/README.md create mode 100644 tests/not_working/distribuffers/fully_connected/test_cases.yaml create mode 100644 tests/not_working/distribuffers/test_fully_connected.py create mode 100644 tests/not_working/distribuffers/test_xy_routing.py create mode 100644 tests/not_working/distribuffers/xy_routing/test_cases.yaml diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md new file mode 100644 index 00000000..fe39d5cd --- /dev/null +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -0,0 +1,448 @@ +# Transfer models: `HypercubeMulticastModel` and how to write a new one + +This directory holds **distributed / network cost models** for spatial reuse analysis. This +document explains how `HypercubeMulticastModel` works and gives a step-by-step recipe (plus a fully +worked example) for adding your own model. + +--- + +## 1. Overview + +A **transfer model** estimates the on-chip data-movement cost of a mapping: given where data +*lives* (occupancy) and where it is *needed* (fills), it computes how many network *hops* the +delivery costs and which transfers are fulfilled peer-to-peer vs. read from a parent. + +All transfer models implement one abstract interface, `TransferModel`. The concrete +implementations: + +| Model | File | Shape | +|-------|------|-------| +| `SimpleLinkTransferModel` | [`../spatial.py`](../spatial.py) (line ~86) | Neighbor-to-neighbor mesh; no constructor state | +| `HypercubeMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) (line ~157) | Distance-aware worst-case multicast; takes a `dist_fn` | +| `FullyConnectedMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) (line ~265) | Fully-connected fabric; 1 hop per fabric crossing (see §6) | +| `XYRoutingMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | XY / dimension-order routing on a 2-D mesh; X-then-Y multicast tree (see §7) | +| `StarMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | **WIP / incomplete** — not yet usable | + +> **There is no registry or factory.** Models are constructed directly +> (`HypercubeMulticastModel(dist_fn)`) and applied via `.apply(...)`. The only current usage is the +> test suite — [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py). +> The `not_working/` location signals these models are work-in-progress. + +--- + +## 2. The `TransferModel` contract + +Defined in [`../spatial.py`](../spatial.py). You implement exactly one abstract method: + +```python +class TransferModel(ABC): + @abstractmethod + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + ... +``` + +### Inputs + +- **`buff: MappingNode`** — the buffer being analyzed. (`HypercubeMulticastModel` ignores it and + relies on its `dist_fn` instead; `SimpleLinkTransferModel` uses it to find spatial dims.) +- **`fills: Fill`** — a *tagged* `isl.Map` `{ [spacetime] -> [data] }` describing what each element + needs from its parent over time. +- **`occs: Occupancy`** — a *tagged* `isl.Map` `{ [spacetime] -> [data] }` describing what each + element holds over time. + +`Fill` and `Occupancy` are `TaggedMap`s (see +[`../mapping_to_isl/types.py`](../mapping_to_isl/types.py)): the raw relation is `.map_`, and +`.tags` is a list of `Tag`s labeling each **input** dimension. Both constructors assert +`len(tags) == map_.dim(isl.dim_type.in_)`. The relevant tags are: + +- `TemporalTag()` — that dimension spreads over time. +- `SpatialTag(spatial_dim, buffer)` — that dimension spreads over space, in `buffer`. + +### Output: `TransferInfo` + +A frozen dataclass (in `../spatial.py`) you must fully populate: + +| Field | Type | Meaning | +|-------|------|---------| +| `fulfilled_fill` | `Transfers` | Fills satisfied by peer-to-peer transfers (a tagged map). | +| `unfulfilled_fill` | `Fill` | Fills *not* satisfied (must come from higher in the hierarchy). | +| `parent_reads` | `Reads` | Fills satisfied by parent-to-child reads. | +| `hops` | `isl.PwQPolynomial` | The transfer cost metric across spacetime. | +| `link_transfer` | `bool` | Metadata flag — whether this used link transfers. | + +`Transfers` and `Reads` are thin `TaggedMap` subclasses; construct them as +`Transfers(tags, map_)` / `Reads(tags, map_)`. + +--- + +## 3. How `HypercubeMulticastModel` works + +> **Worst-case multicast.** It assumes every multicast broadcasts to the *convex hypercube* that +> encloses all of its sources and destinations — an upper bound on the real cost. + +### Constructor + +```python +HypercubeMulticastModel(dist_fn: isl.Map) +``` + +`dist_fn` is a distance function `{ [src -> dst] -> [hops] }`. Two assumptions are baked in: + +1. **Orthogonal dimensions / Manhattan distance** — each unit move along a dimension costs 1 hop, + and dimensions are orthogonal in the metric space. +2. **Translational invariance** — distance depends only on the displacement: if + `|src − dst| = |src' − dst'|` then `dist_fn(src, dst) = dist_fn(src', dst')`. + +These come from `calculate_extents_per_dim`, which the model relies on. + +### `apply(...)` data flow + +``` +occs.map_ { [spacetime] -> [data] } fills.map_ { [spacetime] -> [data] } + \ / + \ / + identify_mesh_casts(occs, fills, dist_fn) + | + v + mcs : { [data] -> [dst -> src] } (each datum's nearest-source multicast network) + | + _cost_mesh_cast_hypercube(mcs) + | + v + hops : isl.PwQPolynomial (total upper-bound hop count) +``` + +**Step A — `identify_mesh_casts(src_occupancy, dst_fill, dist_fn)`** +([`distributed_buffers.py`](distributed_buffers.py), line ~22). + +For every datum, it pairs the destinations that request it with the *nearest* source that holds it. +Conceptually: + +1. Reverse occupancy to `{ [data] -> [src] }` (which elements hold each datum). +2. Match each fill `{ [dst] -> [data] }` against the sources holding that datum → + `{ [dst -> data] -> [dst -> src] }`. +3. Apply `dist_fn` and take `lexmin` over distance to keep only the closest source per + `(dst, data)`. +4. Regroup as `{ [data] -> [dst -> src] }` and `lexmin` again to *devolve to a single source* when + several are equidistant. + +The result is the set of **multicast networks** (`mcns`): per datum, the destinations grouped with +their chosen source. + +**Step B — extents per dimension: `calculate_extents_per_dim(mcns)`** +([`distributed_buffers.py`](distributed_buffers.py), line ~98). + +For each multicast network it unions the sources with the destinations, then for each NoC dimension +projects away the others and takes `dim_max − dim_min`. That difference is the **extent** (the side +length of the bounding box) along that dimension, returned as one `isl.PwAff` per dimension. + +**Step C — hypercube cost: `_cost_mesh_cast_hypercube(mcns)`** +([`distributed_buffers.py`](distributed_buffers.py), line ~213). + +It folds the per-dimension extents into a single cost, then `.sum()`s over all networks. + +> ⚠️ **Known discrepancy — this is the bug currently being tracked.** +> The docstring/comment says the cost is +> `(∏_i extent_i) − 1` (the number of *interior* points of the bounding box minus one — the +> standard hypercube broadcast cost). **But the code actually computes** +> +> ``` +> cost = (∏_i (extent_i + 1)) − 1 +> ``` +> +> because the loop multiplies by `dim_extent.add(one)` (`extent_i + 1`), not `extent_i`. Since +> `extent_i = max − min` is already *one less* than the number of points along a dimension, adding +> 1 back double-counts the span. This is consistent with the observed **~3× overestimate of hops in +> all-to-all** topologies noted in recent work. Treat the formula in the code as *not yet correct*; +> a new template should decide deliberately whether it wants `extent_i` or `extent_i + 1`. + +### Return value + +```python +return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), # everything treated as fulfilled + parent_reads=Reads(occs.tags, mcs), # the multicast map + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty (map minus itself) + hops=result, # the PwQPolynomial cost + link_transfer=True, +) +``` + +Note the idiom `fills.map_.subtract(fills.map_)` — an **empty map over the right space**. This is +how you build a typed empty relation without hardcoding dimensions. + +--- + +## 4. Contrast: `SimpleLinkTransferModel` + +For comparison (the *other* valid shape — no constructor state), `SimpleLinkTransferModel` +([`../spatial.py`](../spatial.py), line ~86): + +- Takes **no `dist_fn`**; constructed as `SimpleLinkTransferModel()`. +- Asserts `fills.tags == occs.tags`. +- Builds a **neighbor-to-neighbor mesh** via `make_mesh_connectivity` (only 1 or 2 spatial dims + supported — raises otherwise). +- Data reachable from a neighbor is `fulfilled_fill`; the rest is `unfulfilled_fill`. +- `hops` is **1 per neighbor-filled element** (`PwQPolynomial.one_on_domain(...)`), not a + distance-weighted sum. +- Has an early-out: if there is no temporal dimension or no spatial dimension, nothing moves, so it + returns an empty/zero `TransferInfo`. + +So the two existing models bracket the design space: **stateful + distance-aware** (hypercube) vs. +**stateless + fixed-topology** (simple link). + +--- + +## 5. How to create a new template + +1. **Pick a home.** Add your class under `reuse/isl/`. Put distributed / NoC / mesh models in this + `distributed/` directory next to `distributed_buffers.py`. +2. **Subclass `TransferModel`** and decide your constructor state — a `dist_fn`, topology + parameters, bandwidth, or nothing (like `SimpleLinkTransferModel`). +3. **Implement `apply(self, buff, fills, occs) -> TransferInfo`.** If your model is mesh/multicast + shaped, reuse the existing kernels: + - `identify_mesh_casts(occs.map_, fills.map_, self.dist_fn)` to get `{ [data] -> [dst -> src] }`. + - `calculate_extents_per_dim(mcns)` if you want per-dimension bounding-box extents. + Otherwise write your own cost kernel over the ISL maps. +4. **Honor the invariants.** Assert `fills.tags == occs.tags` if your model needs aligned tags. + Build `hops` as an `isl.PwQPolynomial` over the correct domain, and build empty maps with the + `map_.subtract(map_)` idiom rather than hardcoding spaces. +5. **Add a test** mirroring + [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py): + a YAML-driven gamut of `(dims, fill, occ, dist_fn, expected_hops)` cases. + +### Copy-paste skeleton + +```python +import islpy as isl + +from accelforge.frontend.mapping import MappingNode +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import Fill, Occupancy +from accelforge.model._looptree.reuse.isl.spatial import ( + Reads, + Transfers, + TransferInfo, + TransferModel, +) +# Reuse these if your model is mesh/multicast-shaped: +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + identify_mesh_casts, + calculate_extents_per_dim, +) + + +class MyTransferModel(TransferModel): + """One-line description of the topology/assumptions this model encodes.""" + + def __init__(self, dist_fn: isl.Map): + # TODO: store whatever state your cost kernel needs (or drop the arg entirely). + self.dist_fn = dist_fn + + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + # TODO (optional): assert fills.tags == occs.tags + + # 1. Group destinations with their nearest source per datum. + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + + # 2. TODO: compute your cost as an isl.PwQPolynomial. + hops: isl.PwQPolynomial = self._cost(mcs) + + # 3. Assemble the result. + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty + hops=hops, + link_transfer=True, + ) + + def _cost(self, mcns: isl.Map) -> isl.PwQPolynomial: + # TODO: your cost kernel. + raise NotImplementedError +``` + +**Checklist** + +- [ ] Subclasses `TransferModel`, implements `apply`. +- [ ] Constructor state matches what the cost kernel needs. +- [ ] `hops` is an `isl.PwQPolynomial` over the right domain. +- [ ] `fulfilled_fill` + `parent_reads` + `unfulfilled_fill` partition the fills correctly for your + model's semantics. +- [ ] Empty maps built with `map_.subtract(map_)`, not hardcoded. +- [ ] A YAML-driven gamut test exists. + +--- + +## 6. Worked example: `FullyConnectedMulticastModel` + +A real, tested model for a **fully-connected fabric** (e.g. an NVSwitch-style all-to-all). It lives +next to the hypercube model in [`distributed_buffers.py`](distributed_buffers.py). + +On a fully-connected fabric every cross-node delivery costs **one hop regardless of distance**, and a +self-delivery (source node == destination node) costs **zero**. So the cost of a mapping is just the +number of deliveries that actually traverse the fabric: + +``` +cost = | { (data, dst, src) ∈ mcs : dist_fn(dst, src) ≥ 1 } | +``` + +`dist_fn` is used *only* to tell self-deliveries (0 hops) apart from fabric-crossing ones — its hop +*magnitude* never enters the cost. This sidesteps the hypercube extent overestimate (see §3) entirely. + +```python +class FullyConnectedMulticastModel(TransferModel): + """Multicast cost on a fully-connected fabric: 1 hop per fabric crossing.""" + + def __init__(self, dist_fn: isl.Map): + self.dist_fn = dist_fn + + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + result: isl.PwQPolynomial = self._cost_fully_connected(mcs) + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty + hops=result, + link_transfer=True, + ) + + def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: + """Count the deliveries in ``mcns`` that traverse the fabric (dist >= 1).""" + # [dst -> src] pairs that actually traverse the fabric (>= 1 hop). + crossing_hops: isl.Set = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" + ) + crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() + crossing: isl.Map = mcns.intersect_range(crossing_pairs) + return crossing.wrap().card() +``` + +How the kernel works: `dist_fn.intersect_range({ hops[h] : h ≥ 1 }).domain()` is the set of +`[dst -> src]` pairs that cross the fabric; intersecting `mcns`' range with it keeps only crossing +deliveries; `.wrap().card()` counts the `(data, dst, src)` points as an `isl.PwQPolynomial` +(constant when there are no parameters). `intersect_range`, `domain`, `wrap`, and `card` are all +standard `islpy`/barvinok operations already used elsewhere in this subsystem. + +**Verified numbers** (8-GPU one-hot encoding, from the test below): + +| Case | `FullyConnectedMulticastModel` | `HypercubeMulticastModel` | +|------|-------------------------------|---------------------------| +| all-to-all (64 chunks, 8 self) | **56** | 168 | +| single unicast GPU0 → GPU3 | **1** | 3 | +| self chunk GPU5 → GPU5 | **0** | 0 | + +The all-to-all column is the headline: **56 vs 168 is exactly the ~3× overestimate** the hypercube +model incurs on a fully-connected fabric (each unicast costed as a `(1+1)(1+1) − 1 = 3` hypercube +instead of a single crossing — see the discrepancy in §3). + +This example is exercised by a real test: +[`tests/not_working/distribuffers/test_fully_connected.py`](../../../../../../tests/not_working/distribuffers/test_fully_connected.py) +with cases in +[`tests/not_working/distribuffers/fully_connected/test_cases.yaml`](../../../../../../tests/not_working/distribuffers/fully_connected/test_cases.yaml). +Run it (m4 on `PATH` per the islpy-barvinok setup): + +```bash +PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ + tests/not_working/distribuffers/test_fully_connected.py -q +``` + +--- + +## 7. XY (dimension-order) routing: `XYRoutingMulticastModel` + +`XYRoutingMulticastModel` ([`distributed_buffers.py`](distributed_buffers.py)) models **XY routing** +on a 2-D mesh: every packet travels **along X first, then Y**. A multicast from one source is +therefore a rigid tree: + +1. an **X segment** along the source's row, reaching every column that holds a destination, then +2. an independent **Y segment** down each of those columns, *starting from the source's row*. + +The cost of one tree (source `s = (xs, ys)`, destinations `D`) is: + +``` +x_extent({xs} ∪ {dst columns}) + + Σ over destination columns xd of y_extent({ys} ∪ {dst y's in column xd}) +``` + +Source selection is **per destination** (`identify_mesh_casts` pairs each destination with its +nearest source, devolving ties); destinations sharing a source form one tree, and the model sums +over all trees and all data. + +### Where it sits: `extent_DOR` (floor) ≤ XY ≤ hypercube + +Because each column's Y segment restarts from the source row instead of sharing a trunk, XY is an +**upper bound on free routing** and a **lower bound on the hypercube** (which reaches every node in +the bounding box). Note `extent_DOR_hops` in the test yamls is the **free-routing floor**, *not* the +XY cost — it assumes routing can move freely between dimensions. Worked example (verified by the +model): source `(1,0)` casting to `(0,2)` and `(2,2)`: + +``` + y=2 D . D + y=1 | . | + y=0 +-S-+ S=(1,0); X covers cols 0..2 (2 links) + x=0 1 2 then Y down col 0 and col 2 from row 0 (2 + 2 links) + + floor (extent_DOR) = 4 ≤ XY = 6 ≤ hypercube = 8 +``` + +### Cost kernel + +```python +def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: + # X-phase: horizontal links along each source row (x-extent of {dsts} ∪ {src}). + x_extent = calculate_extents_per_dim(mcns)[0] + x_links = self._eval_const(isl.PwQPolynomial.from_pw_aff(x_extent).sum()) + + # Y-phase: vertical links per (data, src, destination column), each column + # spanning from the source row ys to the destinations in that column. + per_src = mcns.range_reverse().uncurry() # {[data->src] -> dst} + split_col = isl.Map.read_from_str(ctx, + "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }") + dst_y = per_src.apply_range(split_col).uncurry() # {[data->src->col] -> yv[y]} + src_y = per_src.domain().unwrap().range_map().apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }")) + src_row = dst_y.domain().unwrap().range_product(src_y).uncurry() + col_ys = dst_y.union(src_row) # source row ∪ dst y's per column + + # Count {ymin <= p < ymax} links per column via cardinality (robust where + # summing a min/max polynomial is not). + ge_min = isl.Map.read_from_str(ctx, "{ yv[ymin] -> p[t] : t >= ymin }") + lt_max = isl.Map.read_from_str(ctx, "{ yv[ymax] -> p[t] : t < ymax }") + links = col_ys.lexmin().apply_range(ge_min).intersect( + col_ys.lexmax().apply_range(lt_max)) + y_links = self._eval_const(links.wrap().card()) + ... # total = x_links + y_links, returned as a constant PwQPolynomial +``` + +Notes / limitations: +- **2-D `noc[x, y]` only** (no temporal dims); N-D dimension-order routing is a TODO. The helper maps + hardcode the `noc[x, y]` shape. +- The Y-term is counted via `card()` of the per-column link set rather than summing a min/max + polynomial — the latter trips a barvinok `summate` assertion at scale (e.g. the 8×8 case). +- Returns a **parameter-free constant** (the validated regime); parametric spacetimes are future work. + +### Tested + +[`tests/not_working/distribuffers/test_xy_routing.py`](../../../../../../tests/not_working/distribuffers/test_xy_routing.py) +with hand-derived cases in +[`tests/not_working/distribuffers/xy_routing/test_cases.yaml`](../../../../../../tests/not_working/distribuffers/xy_routing/test_cases.yaml) +(there is **no XY oracle in the repo**, so the expected values are hand-derived and each case carries +its geometry). Cases: unicast `4`, the `(1,0)` discriminator `6`, three-corner `6`, 1-D column `3`, +replicated-source `4`, and an 8×8 scale case `448`. Run: + +```bash +PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ + tests/not_working/distribuffers/test_xy_routing.py -q +``` + +--- + +## 8. References + +- Interface + `SimpleLinkTransferModel` + `TransferInfo`: [`../spatial.py`](../spatial.py) +- `HypercubeMulticastModel`, `identify_mesh_casts`, `calculate_extents_per_dim`: + [`distributed_buffers.py`](distributed_buffers.py) +- Tagged-map / tag types: [`../mapping_to_isl/types.py`](../mapping_to_isl/types.py) +- Example test harness: + [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py) diff --git a/tests/not_working/distribuffers/fully_connected/test_cases.yaml b/tests/not_working/distribuffers/fully_connected/test_cases.yaml new file mode 100644 index 00000000..90672f94 --- /dev/null +++ b/tests/not_working/distribuffers/fully_connected/test_cases.yaml @@ -0,0 +1,103 @@ +################################################### +# 8-GPU fully-connected (NVLink/NVSwitch-style) # +# all-to-all, one-hot GPU encoding. # +# # +# GPU i sits at one-hot coordinate e_i. A # +# delivery costs 1 hop iff it crosses the fabric # +# (src != dst) and 0 otherwise, so: # +# all-to-all -> 56 crossings (64 - 8 self) # +# one unicast -> 1 # +# one self -> 0 # +# Compare the hypercube model: 168 / 3 / 0 (3x). # +# dist_fn is unit-cost (matching only). # +################################################### + +# All-to-all over 8 GPUs: each GPU holds chunks data[self, d] and requests data[s, self]. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and 0 <= d < 8 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and 0 <= s < 8 + } + dims: &8d_onehot_spatial + - type: Spatial + spatial_dim: 0 + target: 0 + - type: Spatial + spatial_dim: 1 + target: 0 + - type: Spatial + spatial_dim: 2 + target: 0 + - type: Spatial + spatial_dim: 3 + target: 0 + - type: Spatial + spatial_dim: 4 + target: 0 + - type: Spatial + spatial_dim: 5 + target: 0 + - type: Spatial + spatial_dim: 6 + target: 0 + - type: Spatial + spatial_dim: 7 + target: 0 + dist_fn: &fully_connected_unit | + { + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[0] : + xd0 = xs0 and xd1 = xs1 and xd2 = xs2 and xd3 = xs3 and xd4 = xs4 and xd5 = xs5 and xd6 = xs6 and xd7 = xs7; + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[1] : + (xd0 < xs0) or (xd0 > xs0) or (xd1 < xs1) or (xd1 > xs1) or (xd2 < xs2) or (xd2 > xs2) or (xd3 < xs3) or (xd3 > xs3) or (xd4 < xs4) or (xd4 > xs4) or (xd5 < xs5) or (xd5 > xs5) or (xd6 < xs6) or (xd6 > xs6) or (xd7 < xs7) or (xd7 > xs7) + } + expected: + fully_connected_hops: 56 + +# Single chunk GPU0 -> GPU3: one unicast crossing, cost 1. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 0 and d = 3 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 0 and d = 3 + } + dims: *8d_onehot_spatial + dist_fn: *fully_connected_unit + expected: + fully_connected_hops: 1 + +# Self chunk GPU5 -> GPU5: never crosses the fabric, cost 0. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 5 and d = 5 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 5 and d = 5 + } + dims: *8d_onehot_spatial + dist_fn: *fully_connected_unit + expected: + fully_connected_hops: 0 diff --git a/tests/not_working/distribuffers/test_fully_connected.py b/tests/not_working/distribuffers/test_fully_connected.py new file mode 100644 index 00000000..e2a895db --- /dev/null +++ b/tests/not_working/distribuffers/test_fully_connected.py @@ -0,0 +1,90 @@ +""" +Tests for the FullyConnectedMulticastModel. + +The model costs each delivery that crosses the fabric at one hop and self- +deliveries at zero, so an 8-GPU one-hot all-to-all costs 56 hops (64 chunks - 8 +self-deliveries) -- a third of the HypercubeMulticastModel's 168 on the same +input. See accelforge/model/_looptree/reuse/isl/distributed/README.md. +""" + +import unittest +from pathlib import Path + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( + # Data movement descriptors. + Fill, + Occupancy, + # Tags + Tag, + SpatialTag, + TemporalTag, +) +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + FullyConnectedMulticastModel, +) +from accelforge.model._looptree.reuse.isl.spatial import TransferInfo +from .util import load_solutions + + +def construct_spacetime(dims: list) -> list[Tag]: + """ + Given a list of dimension tags as strings, convert them into the proper `Tag` + objects. + + Parameters + ---------- + dims: + The list of dim tags as strings. + + Returns + ------- + list[Tag] where list[i] is the tag corresponding to dims[i]. + """ + spacetime: list[Tag] = [] + for dim in dims: + if dim["type"] == "Temporal": + spacetime.append(TemporalTag()) + elif dim["type"] == "Spatial": + spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) + + return spacetime + + +class TestFullyConnectedMulticastModel(unittest.TestCase): + """ + Tests the FullyConnectedMulticastModel with a series of premade test cases. + """ + + TEST_CASES_FILE: str = Path(__file__).parent / "fully_connected" / "test_cases.yaml" + testcases: dict = load_solutions(TEST_CASES_FILE) + + def test_gamut(self): + """ + Tests the entire gamut of test cases we have specified in the yaml. + """ + for test in self.testcases: + # Reads test case parameters and constructs the necessary objects. + dim_tags: list[Tag] = construct_spacetime(test["dims"]) + fill: Fill = Fill(dim_tags, test["fill"]) + occ: Occupancy = Occupancy(dim_tags, test["occ"]) + dist_fn: isl.Map = test["dist_fn"] + model: FullyConnectedMulticastModel = FullyConnectedMulticastModel(dist_fn) + + # Applies the model. + info: TransferInfo = model.apply(0, fill, occ) + # Checks the results. + sum_extract: int = info.hops.eval( + isl.Point.zero(info.hops.domain().get_space()) + ) + + # The block is used for debugging test cases not yet implemented. + if test["expected"]["fully_connected_hops"] is None: + print("~~~Test case in progress:~~~") + print(f"Fill: {fill}") + print(f"Occ: {occ}") + print(f"Dist Fn: {dist_fn}") + print(f"Returned: {sum_extract}") + else: + assert sum_extract == test["expected"]["fully_connected_hops"] diff --git a/tests/not_working/distribuffers/test_xy_routing.py b/tests/not_working/distribuffers/test_xy_routing.py new file mode 100644 index 00000000..7e3662f3 --- /dev/null +++ b/tests/not_working/distribuffers/test_xy_routing.py @@ -0,0 +1,93 @@ +""" +Tests for the XYRoutingMulticastModel (dimension-order routing on a 2-D mesh). + +XY routing sends each packet along X then Y, so a multicast tree is an X-segment +along the source row plus an independent Y-segment down each destination column. +The expected hop counts here are HAND-DERIVED (the repo has no XY oracle -- the +yaml's extent_DOR_hops is a free-routing floor, not the XY cost); each case's +geometry is documented in fully_connected/../xy_routing/test_cases.yaml so the +numbers can be re-checked. See +accelforge/model/_looptree/reuse/isl/distributed/README.md. +""" + +import unittest +from pathlib import Path + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( + # Data movement descriptors. + Fill, + Occupancy, + # Tags + Tag, + SpatialTag, + TemporalTag, +) +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + XYRoutingMulticastModel, +) +from accelforge.model._looptree.reuse.isl.spatial import TransferInfo +from .util import load_solutions + + +def construct_spacetime(dims: list) -> list[Tag]: + """ + Given a list of dimension tags as strings, convert them into the proper `Tag` + objects. + + Parameters + ---------- + dims: + The list of dim tags as strings. + + Returns + ------- + list[Tag] where list[i] is the tag corresponding to dims[i]. + """ + spacetime: list[Tag] = [] + for dim in dims: + if dim["type"] == "Temporal": + spacetime.append(TemporalTag()) + elif dim["type"] == "Spatial": + spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) + + return spacetime + + +class TestXYRoutingMulticastModel(unittest.TestCase): + """ + Tests the XYRoutingMulticastModel with a series of premade test cases. + """ + + TEST_CASES_FILE: str = Path(__file__).parent / "xy_routing" / "test_cases.yaml" + testcases: dict = load_solutions(TEST_CASES_FILE) + + def test_gamut(self): + """ + Tests the entire gamut of test cases we have specified in the yaml. + """ + for test in self.testcases: + # Reads test case parameters and constructs the necessary objects. + dim_tags: list[Tag] = construct_spacetime(test["dims"]) + fill: Fill = Fill(dim_tags, test["fill"]) + occ: Occupancy = Occupancy(dim_tags, test["occ"]) + dist_fn: isl.Map = test["dist_fn"] + model: XYRoutingMulticastModel = XYRoutingMulticastModel(dist_fn) + + # Applies the model. + info: TransferInfo = model.apply(0, fill, occ) + # Checks the results. + sum_extract: int = info.hops.eval( + isl.Point.zero(info.hops.domain().get_space()) + ) + + # The block is used for debugging test cases not yet implemented. + if test["expected"]["xy_routing_hops"] is None: + print("~~~Test case in progress:~~~") + print(f"Fill: {fill}") + print(f"Occ: {occ}") + print(f"Dist Fn: {dist_fn}") + print(f"Returned: {sum_extract}") + else: + assert sum_extract == test["expected"]["xy_routing_hops"] diff --git a/tests/not_working/distribuffers/xy_routing/test_cases.yaml b/tests/not_working/distribuffers/xy_routing/test_cases.yaml new file mode 100644 index 00000000..ff192655 --- /dev/null +++ b/tests/not_working/distribuffers/xy_routing/test_cases.yaml @@ -0,0 +1,83 @@ +################################################### +# XY (dimension-order) routing test cases. # +# # +# Packets route along X first, then Y, so a # +# multicast tree = X-segment along the source row # +# + an independent Y-segment down each destination # +# column from the source row. # +# cost = x_extent({xs} u dst columns) # +# + sum_col y_extent({ys} u dsts in col) # +# # +# Expected xy_routing_hops are HAND-DERIVED (no # +# oracle exists in the repo); each case lists the # +# geometry so the number can be re-checked. # +# # +# Ordering: extent_DOR (floor) <= XY <= hypercube. # +################################################### + +# A. Unicast (0,0) -> (2,2): X=2 + col2 Y={0,2}=2 -> 4 (== floor). +- occ: "{ noc[x, y] -> data[c] : c = 0 and x = 0 and y = 0 }" + fill: "{ noc[x, y] -> data[c] : c = 0 and x = 2 and y = 2 }" + dims: &2d_spatial + - type: Spatial + spatial_dim: 0 + target: 0 + - type: Spatial + spatial_dim: 1 + target: 0 + dist_fn: &2d_manhattan | + { + [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + (yd - ys)] : + xd >= xs and yd >= ys; + [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] : + xd < xs and yd < ys; + [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + (yd - ys)] : + xd < xs and yd >= ys; + [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + -(yd - ys)] : + xd >= xs and yd < ys + } + expected: + xy_routing_hops: 4 + +# B. Source (1,0) -> (0,2),(2,2): X={0,1,2}=2 + col0 {0,2}=2 + col2 {0,2}=2 -> 6. +# Discriminator: floor 4 < XY 6 < hypercube 8. +- occ: "{ noc[x, y] -> data[c] : c = 0 and x = 1 and y = 0 }" + fill: "{ noc[x, y] -> data[c] : c = 0 and ((x = 0 and y = 2) or (x = 2 and y = 2)) }" + dims: *2d_spatial + dist_fn: *2d_manhattan + expected: + xy_routing_hops: 6 + +# C. Three corners from (0,0) -> (2,0),(0,2),(2,2): X={0,2}=2 + col0 {0,2}=2 + col2 {0,2}=2 -> 6. +- occ: "{ noc[x, y] -> data[c] : c = 0 and x = 0 and y = 0 }" + fill: "{ noc[x, y] -> data[c] : c = 0 and ((x = 2 and y = 0) or (x = 0 and y = 2) or (x = 2 and y = 2)) }" + dims: *2d_spatial + dist_fn: *2d_manhattan + expected: + xy_routing_hops: 6 + +# D. 1-D column (0,1) -> column x=0, y=0..3: X=0 + col0 {0..3}=3 -> 3 (== floor). +- occ: "{ noc[x, y] -> data[c] : c = 0 and x = 0 and y = 1 }" + fill: "{ noc[x, y] -> data[c] : c = 0 and x = 0 and 0 <= y < 4 }" + dims: *2d_spatial + dist_fn: *2d_manhattan + expected: + xy_routing_hops: 3 + +# E. Replicated source (0,0)&(2,0) -> (0,2),(2,2): per-dst nearest splits into two +# self-column trees, each 2 -> 4 (a single-root tree would cost 6). +- occ: "{ noc[x, y] -> data[c] : c = 0 and ((x = 0 and y = 0) or (x = 2 and y = 0)) }" + fill: "{ noc[x, y] -> data[c] : c = 0 and ((x = 0 and y = 2) or (x = 2 and y = 2)) }" + dims: *2d_spatial + dist_fn: *2d_manhattan + expected: + xy_routing_hops: 4 + +# F. Scale: 8x8, each datum data[d0,d1] held at (d0,d1), requested by column x=d0. +# Per datum: X=0 + Y=7 -> 7; 64 data -> 448. +- occ: "{ noc[xs, ys] -> data[d0, d1] : d0 = xs and d1 = ys and 0 <= xs < 8 and 0 <= ys < 8 }" + fill: "{ noc[xd, yd] -> data[d0, d1] : d0 = xd and 0 <= d1 < 8 and 0 <= xd < 8 and 0 <= yd < 8 }" + dims: *2d_spatial + dist_fn: *2d_manhattan + expected: + xy_routing_hops: 448 From 142722f5381d4e841348458b02b99d6b2ad32a53 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Tue, 30 Jun 2026 15:28:15 -0400 Subject: [PATCH 11/19] LLM implemented code from pseudocode but it needs verification --- .../_looptree/reuse/isl/distributed/README.md | 69 +- .../isl/distributed/distributed_buffers.py | 596 ++++++++++++------ 2 files changed, 470 insertions(+), 195 deletions(-) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index fe39d5cd..03e6135a 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -438,7 +438,74 @@ PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ --- -## 8. References +## 8. Per-edge memory pressure (link load): `EdgePressure` + +`hops` collapses a whole routing to one scalar, which hides a real constraint: **each physical edge +has a finite bandwidth**, so the *busiest* link is what saturates first. `EdgePressure` keeps the +load broken out per directed physical edge — `{ edge -> number-of-trees-crossing-it }` — which is +exactly the quantity the production symbolic path calls `max_traffic` and feeds into the `Network` +latency formula `max(max_hops·latency, max_link_traffic / throughput)` +([`frontend/arch/components.py`](../../../../frontend/arch/components.py)). Two models expose it +today via an `edge_pressure(fills, occs) -> EdgePressure` method. + +### The primitive + +After `identify_mesh_casts` fixes the `[src] -> [dst]` pairs, build, per directed edge type, a map +`M = { [data -> src] -> edge }` associating each multicast **tree** with every edge its route +crosses. Then + +```python +load = M.reverse().card() # { edge -> #trees crossing it } +``` + +The key is the tree `(data, src)`, **not** the destination: within one tree a link is traversed once +regardless of how many leaves hang off it, so this counts *pressure* (distinct flows on a link), not +summed hops. `EdgePressure.bottleneck()` returns the max load (enumerating pieces; parameter-free +regime, like `_cost_xy`); `eval_edge(name, coords)` looks up one edge. + +### XY routing → directed mesh edges + +`XYRoutingMulticastModel._directed_mesh_links` decomposes each tree onto four directed link types: +`xedge_r`/`xedge_l` along the source row (split at the source column `xs`) and `yedge_u`/`yedge_d` +down each destination column (split at the source row `ys`). It reuses the §7 link-set construction +but **keeps the edge identity** instead of collapsing to a count (and builds the X links explicitly — +`calculate_extents_per_dim` only yields a scalar length and discards *which* links are used). + +Decisive cross-check (no oracle needed): **`Σ_edges load == total XY hops`**. The per-edge loads must +sum back to the already-trusted A–F totals (4/6/6/3/4/448), so a wrong decomposition fails. Worked F +geometry (8×8, datum `(d0,d1)` held at node `(d0,d1)`, requested by all of column `x=d0`): every +datum of column `c` floods all 7 vertical links of column `c`, so `yedge_u[c,t]` load = `t+1` and +`yedge_d[c,t]` = `7−t`; the busiest directed link is `7` (the physical link total is `8`). Cases A–E +are single trees, so every edge load is `1`. + +### Star / fully-connected → spokes + +`StarMulticastModel` **is** the spokes realization of the fully-connected fabric (NVSwitch-style: each +node has one link to a central switch). A delivery routes `src → switch → dst`, so each datum loads +its source's egress spoke once (multicast fans out *at the switch*) and each destination's ingress +spoke once; self-deliveries (0 hops) load nothing. `_spoke_loads` returns `spoke_in[n]` (ingress) and +`spoke_out[n]` (egress) loads. For an N-way all-to-all every node receives `N−1` and sources `1`, so +the **ingress spokes are hottest at `N−1`** — this is where bandwidth actually bites, whereas +`FullyConnectedMulticastModel`'s contention-free full-mesh view has no hotspot. The two are tied by +**`Σ_nodes ingress == FullyConnected crossing count`** (every crossing delivery is one node's +ingress; e.g. 8-GPU all-to-all: `56`). The star scalar `hops` = `Σ egress + Σ ingress` (injections + +deliveries). + +### Tested + +[`tests/not_working/distribuffers/test_edge_pressure.py`](../../../../../../tests/not_working/distribuffers/test_edge_pressure.py) +— the XY `Σ load == hops` invariant over A–F, F's bottleneck/edge loads (`7`, `yedge_u[0,6]=7`, +`yedge_d[0,1]=6`), single-tree unit bottlenecks, and the star spoke loads / `Σ ingress == FC count` +invariant for 4- and 8-GPU all-to-all. Run: + +```bash +PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ + tests/not_working/distribuffers/test_edge_pressure.py -q +``` + +--- + +## 9. References - Interface + `SimpleLinkTransferModel` + `TransferInfo`: [`../spatial.py`](../spatial.py) - `HypercubeMulticastModel`, `identify_mesh_casts`, `calculate_extents_per_dim`: diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index 3d1e8a6c..b1dd416f 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -5,6 +5,8 @@ import logging +from dataclasses import dataclass + import islpy as isl from accelforge.frontend.mapping import MappingNode @@ -20,6 +22,146 @@ from typing import Optional + +@dataclass(frozen=True) +class EdgePressure: + """ + Per-edge memory pressure (link load) for a spatial transfer. + + Where a ``TransferModel``'s ``hops`` collapses a whole routing to a single + scalar, ``EdgePressure`` keeps the load broken out *per physical edge*: how + many multicast trees cross each individual link. This is what a per-link + bandwidth limit acts on -- the busiest edge (``bottleneck``) is what saturates + first, exactly the quantity the symbolic network model calls + ``max_traffic`` and divides by an edge's throughput to get a bandwidth-bound + latency. + + "Edge" is a *directed* physical link. Its identity is encoded in the tuple + name and coordinates of ``load``'s domain, e.g. ``yedge_u[x, t]`` (the upward + vertical link in column ``x`` between rows ``t`` and ``t + 1``) for the XY + mesh model, or ``spoke_in[n]`` / ``spoke_out[n]`` (a node's ingress / egress + link to the central switch) for the star/fully-connected spokes model. + + The load is keyed on the multicast *tree* ``(data, src)``, not on individual + destinations: within one tree a link is traversed once regardless of how many + leaves hang off it, so ``load`` measures pressure (distinct flows over a link) + rather than summed hops. Consequently the per-direction sums satisfy useful + invariants -- e.g. for XY routing ``sum over edges of load == total hops``. + + Attributes + ---------- + load: + An ``isl.UnionPwQPolynomial`` ``{ edge -> number-of-trees }`` spanning + every directed edge type the model emits. + + Notes + ----- + ``bottleneck`` and ``eval_edge`` assume the load is piecewise constant (the + parameter-free regime the distributed models are validated in); there is no + clean ISL "max of a quasi-polynomial over its domain" primitive, so the + bottleneck is obtained by enumerating pieces. + """ + + load: isl.UnionPwQPolynomial + + def total(self) -> int: + """ + Sum the load over every edge. + + For a single-direction (or single edge-type) pressure this is the total + traffic; for the full mesh pressure it equals the model's total ``hops`` + (the ``sum over edges of load == total hops`` invariant), which is the + primary cross-check that the per-edge decomposition is correct. + """ + pieces: list[isl.PwQPolynomial] = [] + self.load.foreach_pw_qpolynomial(pieces.append) + total = 0 + for pwq in pieces: + summed: isl.PwQPolynomial = pwq.sum() + total += int( + str(summed.eval(isl.Point.zero(summed.domain().get_space()))) + ) + return total + + def bottleneck(self) -> int: + """ + Return the load on the single most-pressured edge. + + This is the bandwidth-binding quantity: with a uniform per-link + bandwidth, the most-congested edge saturates first, so its load sets the + transfer's bandwidth-bound latency. + + The per-edge load is generally *not* constant across an edge type (e.g. a + flooded column's upward link ``yedge_u[x, t]`` carries ``t + 1`` trees), so + the maximum is found by enumerating the (finite, parameter-free) edge + domain and evaluating the load at each edge -- sampling one point per + piece would under-report a monotone load. + """ + best = 0 + pieces: list[isl.PwQPolynomial] = [] + self.load.foreach_pw_qpolynomial(pieces.append) + for pwq in pieces: + edges: list[isl.Point] = [] + pwq.domain().foreach_point(edges.append) + for edge in edges: + best = max(best, int(str(pwq.eval(edge)))) + return best + + def eval_edge(self, name: str, coords: list[int]) -> int: + """ + Look up the load on one named edge, e.g. ``eval_edge("yedge_u", [0, 6])``. + + Returns 0 if no flow crosses that edge (it is outside the load's support). + + Parameters + ---------- + name: + The edge tuple name (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d`` + for the mesh model, ``spoke_in``/``spoke_out`` for the spokes model). + coords: + The integer edge coordinates within that tuple. + """ + pieces: list[isl.PwQPolynomial] = [] + self.load.foreach_pw_qpolynomial(pieces.append) + for pwq in pieces: + if pwq.domain().get_space().get_tuple_name(isl.dim_type.set) != name: + continue + point: isl.Point = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, + "{ %s[%s] }" % (name, ", ".join(str(c) for c in coords)), + ).sample_point() + return int(str(pwq.eval(point))) + return 0 + + +def _edge_pressure_from_links(edge_maps: list[isl.Map]) -> EdgePressure: + """ + Turn directed flow maps into an ``EdgePressure``. + + Parameters + ---------- + edge_maps: + A list of ``{ [data -> src] -> edge }`` maps, one per directed edge type, + each associating a multicast tree with every edge its route traverses. + + Returns + ------- + An ``EdgePressure`` whose ``load`` is ``{ edge -> number-of-trees }``: for + each map we reverse it and take the cardinality (``reverse().card()`` counts, + per edge, how many distinct ``(data, src)`` trees cross it), then union the + per-type results into one ``UnionPwQPolynomial``. + """ + acc: Optional[isl.UnionPwQPolynomial] = None + for edge_map in edge_maps: + # { edge -> #trees crossing it } + per_edge: isl.PwQPolynomial = edge_map.reverse().card() + contribution = isl.UnionPwQPolynomial.from_pw_qpolynomial(per_edge) + acc = contribution if acc is None else acc.add(contribution) + if acc is None: + acc = isl.UnionPwQPolynomial.read_from_str(isl.DEFAULT_CONTEXT, "{ }") + return EdgePressure(acc) + + def identify_mesh_casts( src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map ) -> isl.Map: @@ -438,67 +580,164 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo link_transfer=True, ) - def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: + def edge_pressure(self, fills: Fill, occs: Occupancy) -> EdgePressure: """ - Total XY-routing link count for the multicast networks `mcns`. + Per-edge memory pressure (link load) of the XY-routing transfer. + + After ``identify_mesh_casts`` fixes the ``[src] -> [dst]`` pairs, this + decomposes each multicast tree onto the directed mesh links it traverses + and counts, per link, how many trees cross it. The result is the load a + per-link bandwidth limit acts on; ``EdgePressure.bottleneck`` is the + busiest link. + + Parameters + ---------- + fills: + The fill of `buffer` across time from parents. + occs: + The occupancy of `buffer` across time. + + Returns + ------- + An ``EdgePressure`` over the four directed mesh-edge types + (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). Its ``total`` equals + this model's ``hops`` -- the cross-check that the decomposition is exact. + """ + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + return _edge_pressure_from_links(self._directed_mesh_links(mcs)) + + def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: + """ + Decompose every multicast tree in `mcns` onto the directed mesh links it + traverses under XY routing. + + Each returned map is ``{ [data -> src] -> edge }`` for one directed edge + type, associating a tree with every link of that type on its route. The + X phase runs along the source row out to every destination column; the Y + phase runs down each destination column from the source row. Directions + split at the source: rightward/leftward in X (at the source column ``xs``) + and upward/downward in Y (at the source row ``ys``). Parameters ---------- mcns: - Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`, - grouped per destination by nearest source. + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. Returns ------- - The X-phase links (per source row) plus Y-phase links (per destination - column), as a constant piecewise quasi-polynomial. + ``[xedge_r, xedge_l, yedge_u, yedge_d]`` maps. An edge ``xedge_r[t, ys]`` + is the rightward link in row ``ys`` between columns ``t`` and ``t + 1``; + ``yedge_u[x, t]`` is the upward link in column ``x`` between rows ``t`` and + ``t + 1`` (and ``_l`` / ``_d`` the opposite directions). """ ctx = isl.DEFAULT_CONTEXT + # { [data -> src] -> dst noc[x, y] } and a handle on the source per tree. + per_src: isl.Map = mcns.range_reverse().uncurry() + keymap: isl.Map = per_src.domain().unwrap().range_map() # [data->src] -> src - # X-phase: horizontal links along each source row. The x-extent of - # {dsts} u {src} per (data, src) is exactly calculate_extents_per_dim()'s - # first (x) dimension; summing it counts every X link. - x_extent: isl.PwAff = calculate_extents_per_dim(mcns)[0] - x_links: isl.Val = self._eval_const( - isl.PwQPolynomial.from_pw_aff(x_extent).sum() + # --- Y phase: vertical links per (tree, destination column). --- + # Key each destination's y by its column: { [data->src->col] -> yv[y] }. + dst_y: isl.Map = per_src.apply_range( + isl.Map.read_from_str( + ctx, "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }" + ) + ).uncurry() + # Inject the source row ys into every destination column so each Y segment + # starts from the source. + src_y: isl.Map = keymap.apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }") ) - - # Y-phase: vertical links per (data, src, destination column), each column - # spanning from the source row ys to the destinations in that column. - # { [data -> src] -> [dst noc[x, y]] } - per_src: isl.Map = mcns.range_reverse().uncurry() - # Split each destination into its column and its y-coordinate, keying y by - # column: { [data -> src -> col[x]] -> [yv[y]] }. - split_col: isl.Map = isl.Map.read_from_str( - ctx, "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }" + src_row: isl.Map = dst_y.domain().unwrap().range_product(src_y).uncurry() + col_ys: isl.Map = dst_y.union(src_row) + # Every link {ymin <= t < ymax} touched in a column, as a relation keyed by + # the column (card of an explicit link set, robust where a min/max sum is + # not). { [data->src->col] -> p[t] }. + ylinks: isl.Map = col_ys.lexmin().apply_range( + isl.Map.read_from_str(ctx, "{ yv[ymin] -> p[t] : t >= ymin }") + ).intersect( + col_ys.lexmax().apply_range( + isl.Map.read_from_str(ctx, "{ yv[ymax] -> p[t] : t < ymax }") + ) ) - dst_y: isl.Map = per_src.apply_range(split_col).uncurry() - # Inject the source row ys into every destination column so each column's - # Y segment starts from the source. - src_y: isl.Map = per_src.domain().unwrap().range_map().apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }") + # Re-key links by the tree and carry ys so direction splits at the source: + # { [data->src] -> [[col[x] -> p[t]] -> ysv[ys]] }. + y_with_ys: isl.Map = ylinks.curry().range_product( + keymap.apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> ysv[ys] }") + ) + ) + yedge_u: isl.Map = y_with_ys.apply_range( + isl.Map.read_from_str( + ctx, "{ [[col[x] -> p[t]] -> ysv[ys]] -> yedge_u[x, t] : t >= ys }" + ) ) - src_row: isl.Map = ( - dst_y.domain().unwrap().range_product(src_y).uncurry() + yedge_d: isl.Map = y_with_ys.apply_range( + isl.Map.read_from_str( + ctx, "{ [[col[x] -> p[t]] -> ysv[ys]] -> yedge_d[x, t] : t < ys }" + ) ) - # { [data -> src -> col] -> [yv[y]] }: all y-positions touched in a column. - col_ys: isl.Map = dst_y.union(src_row) - # Count the links {ymin <= p < ymax} in each column via cardinality (robust - # where summing a min/max polynomial is not). - ge_min: isl.Map = isl.Map.read_from_str( - ctx, "{ yv[ymin] -> p[t] : t >= ymin }" + # --- X phase: horizontal links along the source row. --- + # Columns spanned per tree = {src column} u {destination columns}. (Built + # explicitly, not from calculate_extents_per_dim, which keeps only the + # extent length and discards which links are used.) + col_x: isl.Map = per_src.apply_range( + isl.Map.read_from_str(ctx, "{ noc[x, y] -> cx[x] }") + ).union( + keymap.apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> cx[xs] }") + ) ) - lt_max: isl.Map = isl.Map.read_from_str( - ctx, "{ yv[ymax] -> p[t] : t < ymax }" + xlinks: isl.Map = col_x.lexmin().apply_range( + isl.Map.read_from_str(ctx, "{ cx[xmin] -> ex[t] : t >= xmin }") + ).intersect( + col_x.lexmax().apply_range( + isl.Map.read_from_str(ctx, "{ cx[xmax] -> ex[t] : t < xmax }") + ) + ) + # Carry the source (xs, ys) so direction splits at xs and the row ys is + # part of the edge identity: { [data->src] -> [ex[t] -> xy[xs, ys]] }. + x_with_src: isl.Map = xlinks.range_product( + keymap.apply_range( + isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> xy[xs, ys] }") + ) + ) + xedge_r: isl.Map = x_with_src.apply_range( + isl.Map.read_from_str( + ctx, "{ [ex[t] -> xy[xs, ys]] -> xedge_r[t, ys] : t >= xs }" + ) ) - links: isl.Map = col_ys.lexmin().apply_range(ge_min).intersect( - col_ys.lexmax().apply_range(lt_max) + xedge_l: isl.Map = x_with_src.apply_range( + isl.Map.read_from_str( + ctx, "{ [ex[t] -> xy[xs, ys]] -> xedge_l[t, ys] : t < xs }" + ) ) - y_links: isl.Val = self._eval_const(links.wrap().card()) - # Total links as a parameter-free constant. - total: isl.Val = x_links.add(y_links) + return [xedge_r, xedge_l, yedge_u, yedge_d] + + def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: + """ + Total XY-routing link count for the multicast networks `mcns`. + + Equals the sum of the per-edge loads from ``_directed_mesh_links`` (every + directed link of every tree counted once), returned as a parameter-free + constant. This is the X-phase links (per source row) plus the Y-phase + links (per destination column). + + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`, + grouped per destination by nearest source. + + Returns + ------- + The total link count as a constant piecewise quasi-polynomial. + """ + ctx = isl.DEFAULT_CONTEXT + total: isl.Val = isl.Val.zero(ctx) + for edge_map in self._directed_mesh_links(mcns): + total = total.add(self._eval_const(edge_map.wrap().card())) zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) return isl.PwQPolynomial.from_qpolynomial( isl.QPolynomial.val_on_domain(zero_dim, total) @@ -512,33 +751,53 @@ def _eval_const(pwq: isl.PwQPolynomial) -> isl.Val: class StarMulticastModel(TransferModel): """ - Does distributed multicasting assuming all nodes are connected to a central node. + Multicast cost model for a star / central-switch fabric -- the spokes + realization of a fully-connected interconnect (e.g. an NVSwitch, where every + GPU connects to a shared switch rather than to a full mesh of peers). + + Every node has exactly one spoke (its bidirectional link to the switch). A + delivery routes ``src -> switch -> dst``: the source injects each datum once + up its egress spoke (multicast fan-out happens at the switch, so one copy per + datum regardless of how many destinations want it), and every destination + receives its datum down its ingress spoke. Self-deliveries (a node already + holding the datum) never cross the fabric and so load no spoke. + + Where ``FullyConnectedMulticastModel`` treats the fabric as a contention-free + full mesh (one dedicated link per pair, no hotspot) and only counts crossings, + this model exposes *where the contention is*: the per-spoke load. The two are + tied by the invariant ``sum over nodes of ingress == FullyConnected crossing + count`` (every crossing delivery is exactly one node's ingress). + + See Also + -------- + FullyConnectedMulticastModel : + The same all-to-all traffic costed as crossings on a full mesh; its hop + count equals this model's total ingress. """ - def __init__(self, reindexer: Optional[isl.Map] = None): + def __init__(self, dist_fn: isl.Map): """ - No distance function as hops for a star model are assumed to be 1 to and from center to any node, and - all data must route through the center. - Parameters ---------- - reindexer: - flattens an input so that 0 (or the lexmin across all dimensions) is the assumed center everything - connects to. + dist_fn: + A distance function { [dst -> src] -> [hops] }, used both to pick each + destination's nearest source and to tell self-deliveries (0 hops, no + spoke load) apart from fabric-crossing ones (>= 1 hop). The hop + magnitude does not enter the spoke load -- on a star every crossing is + one switch hop each way. """ - self.reindexer: Optional[isl.Map] = reindexer - - + self.dist_fn = dist_fn + def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: """ Given a buffer, its fills across time, and its occupancies across time, - calculate the spatial transfers." + calculate the spatial transfers on a star / central-switch fabric. Parameters ---------- buff: - The buffer whose spatial analysis is being considered. Currently, - we rely on dist_fn to deal with this rather than buffer. + The buffer whose spatial analysis is being considered. Unused; the + topology is captured entirely by ``dist_fn``. fills: The fill of `buffer` across time from parents. occs: @@ -546,22 +805,12 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- - Fills that were fulfilled, Fills that were unfilled, and parent reads per - position in spacetime. Then, gets hops per timestep. + A TransferInfo whose `hops` is the total spoke traversals (injections plus + deliveries). Per-spoke load is available via ``edge_pressure``. """ - if self.reindexer: - occs_map = isl.apply_domain(self.relabeler) - fills_map = isl.apply_domain(self.relabeler) - else: - occs_map = occs.map - fills_map = fills_map - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_star_multicast(mcs) - # TODO: Read once from all buffers, assert that - # card(mcs) == tensor_size * duplication factor - n_meshcasts: int = mcs.card() return TransferInfo( fulfilled_fill=Transfers(fills.tags, fills.map_), parent_reads=Reads(occs.tags, mcs), @@ -570,146 +819,105 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo link_transfer=True, ) - def classify_src_dst(rel: isl.Map | isl.Set): - """ - Build a quasi-affine classifier over a wrapped relation [src -> dst]. - - f([src -> dst]) = 0 if src == dst - = 1 elif src is the lexmin OR dst is the lexmin - = 2 otherwise - - Works for any dimensionality: `src`/`dst` may be scalars (1-D) or - tuples (n-D). The only requirement is that the src space and the dst - space match. - + def edge_pressure(self, fills: Fill, occs: Occupancy) -> EdgePressure: + """ + Per-edge memory pressure (spoke load) of the star transfer. + + After ``identify_mesh_casts`` fixes the ``[src] -> [dst]`` pairs, this + reports, per node spoke, how much data crosses it -- egress (data the node + sources into the switch) and ingress (data it receives) as two directed + edges. ``EdgePressure.bottleneck`` is the busiest spoke, the link a + per-spoke bandwidth limit binds on first (e.g. for an N-way all-to-all the + ingress spokes are hottest at ``N - 1``). + Parameters ---------- - rel: - isl.Map ({ src -> dst }, with src-space == dst-space) - or isl.Set ({ [src -> dst] }, i.e. already wrapped). - The lexmin of src-space is the center of the star. - + fills: + The fill of `buffer` across time from parents. + occs: + The occupancy of `buffer` across time. + Returns ------- - isl.PwAff defined on the wrapped space [src -> dst]. - - Preconditions - ------------- - lexmin is unique. - """ - # Accept either a relation or an already-wrapped set. - m = rel.unwrap() if isinstance(rel, isl.Set) else rel - - assert m.dim(isl.dim_type.in_) == m.dim(isl.dim_type.out), \ - "src and dst must share the same space" - - space_set = m.domain() - - # Position-wise identity src -> dst. Used to (a) select the diagonal - # src == dst, and (b) carry the lexmin point from the domain (src) - # space into the range (dst) space, so tuple names need not match. - ident = isl.Map.identity(m.get_space()) - - lex_src = space_set.lexmin() # lexmin point in the src space - lex_dst = lex_src.apply(ident) # same point, in the dst space - - # The three regions, as relations src -> dst (all subsets of m): - eq = m.intersect(ident) # src == dst - is_lexmin = m.intersect_domain(lex_src).union( # src == lexmin - m.intersect_range(lex_dst)) # or dst == lexmin - - # Move everything into the wrapped [src -> dst] set space: - W = m.wrap() - R0 = eq.wrap() - R1 = is_lexmin.wrap().subtract(R0) # the "elif": drop src==dst - R2 = W.subtract(R0).subtract(R1) # everything else - - # A constant quasi-affine piece with value `c` on the given domain. - def const_on(domain, c): - ls = isl.LocalSpace.from_space(domain.get_space()) - val = isl.Val.int_from_si(domain.get_ctx(), c) - aff = isl.Aff.zero_on_domain(ls).set_constant_val(val) - return isl.PwAff.from_aff(aff).intersect_domain(domain) - - # Disjoint domains, so union_add is just a disjoint union of pieces. - return (const_on(R0, 0) - .union_add(const_on(R1, 1)) - .union_add(const_on(R2, 2))) - - - def _pairing(src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map): - """ - Given srcs with data, fills to destinations, and a distance function, identify per data - the srcs delivering that data to dsts. + An ``EdgePressure`` over ``spoke_in[n]`` (ingress) and ``spoke_out[n]`` + (egress) edges. + """ + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + ingress, egress = self._spoke_loads(mcs) + acc = isl.UnionPwQPolynomial.from_pw_qpolynomial(ingress) + acc = acc.add(isl.UnionPwQPolynomial.from_pw_qpolynomial(egress)) + return EdgePressure(acc) + + def _spoke_loads( + self, mcns: isl.Map + ) -> tuple[isl.PwQPolynomial, isl.PwQPolynomial]: + """ + Per-spoke ingress and egress load for the multicast networks `mcns`. Parameters ---------- - src_occupancy: - An isl.Map of the form { [src] -> [data] } corresponding to the data held - at the buffer at space `src`. - dst_fill: - An isl.Map of the form { [dst] -> [data] } corresponding to the data requested - at the element at space `dst`. + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. Returns ------- - { [data] -> [dst -> src] } where { [dst] -> [data] } and { [src] -> [data] } are in - `src_occupancy` and `dst_fill` respectively, and where `[dst -> src]` is the infimum of - `dst_fn(src, dst), ∀ src, dst s.t. { [src] -> [data] } ∈ `src_occupancy` and - `{ [dst] -> [data] }` ∈ `dst_fill`. - - Preconditions: - No duplication of data. - """ - # Makes { [dst -> data] -> [dst -> data] } - fill_to_fill: isl.Map = dst_fill.wrap().identity() - if DUMP_ISL_IR: - logging.info(f"fill_to_fill: {fill_to_fill}") - - # Inverts src_occupancy s.t. data -> src. - # i.e. { [xs, ys] -> [d0, d1] } to { [d0, d1] -> [xs, ys] } - data_presence: isl.Map = src_occupancy.reverse() - - # { [dst -> data] -> [dst -> src] } where src contains data. - fills_to_matches: isl.Map = ( - fill_to_fill.uncurry() # { [[dst -> data] -> dst] -> data } - .apply_range(data_presence) # { [[dst -> data] -> dst] -> src } - .curry() - ) # { [[dst -> data] -> [dst -> src] } - if DUMP_ISL_IR: - logging.info(f"fills_to_matches: {fills_to_matches}") - - # Calculates the distance of a fill to the nearest src satisfying the fill. - # { [dst -> data] -> [dist] } - fill_min_dist: isl.Map = fills_to_matches.apply_range(dist_fn).lexmin() - # Isolates the relevant minimal pairs. - # { [dst -> data] -> [dst -> src] :.dst -> src is minimized distance } - minimal_pairs: isl.Map = ( - fill_min_dist.apply_range( - # Note: Need to match fill -> min_dist with min_dist -> [fill -> match] as lexmin over - # fill and match will minimize distance over the tuple (src, dst, data), but that - # overconstrains the optimization as we want to minimize over distance (dst, data) - # only for all src. - fills_to_matches.range_map() - .apply_range(dist_fn) - .reverse() - ) - .range() - .unwrap() + ``(ingress, egress)`` where ``ingress`` is ``{ spoke_in[n] -> #data the + node receives }`` and ``egress`` is ``{ spoke_out[n] -> #data the node + sources }``, counting only fabric-crossing (>= 1 hop) deliveries. + """ + # Keep only deliveries that actually cross the fabric (>= 1 hop); a node + # already holding its datum loads no spoke. + crossing_hops: isl.Set = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" ) - if DUMP_ISL_IR: - logging.info(f"minimal_pairs: {minimal_pairs}") + crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() + crossing: isl.Map = mcns.intersect_range(crossing_pairs) - # Isolates the multicast networks. - # { [data] -> [dst -> src] : dst -> src is minimized distance } - multicast_networks: isl.Map = minimal_pairs.curry().range().unwrap() - # Devolves to a single source if multiple sources per domain point. - multicast_networks = multicast_networks.uncurry().lexmin().curry() + # { dst -> [src -> data] }: regroup so each delivery is keyed by destination. + cur: isl.Map = crossing.reverse().curry() + # Distinct (node, data) pairs per direction. A source injects each datum + # once (multicast fans out at the switch); a destination receives each once. + ingress_nodes: isl.Map = cur.range_factor_range() # { noc[dst] -> data } + egress_nodes: isl.Map = cur.range().unwrap() # { noc[src] -> data } + + # Relabel the node tuple to the directed spoke edge, then count data per + # spoke. { spoke_in[n] -> #data } and { spoke_out[n] -> #data }. + dims: int = ingress_nodes.dim(isl.dim_type.in_) + idx: str = ", ".join(f"i{k}" for k in range(dims)) + ingress: isl.PwQPolynomial = ingress_nodes.apply_domain( + isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, "{ noc[%s] -> spoke_in[%s] }" % (idx, idx) + ) + ).card() + egress: isl.PwQPolynomial = egress_nodes.apply_domain( + isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, "{ noc[%s] -> spoke_out[%s] }" % (idx, idx) + ) + ).card() + return ingress, egress - return multicast_networks + def _cost_star_multicast(self, mcns: isl.Map) -> isl.PwQPolynomial: + """ + Total spoke traversals for the multicast networks `mcns`. + Each datum costs one egress-spoke hop per injecting source plus one + ingress-spoke hop per receiving destination, so the total is + ``sum(egress) + sum(ingress)``. Returned as a parameter-free constant. - def _cost_star_multicast(self, mcs): - """ + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. """ - raise NotImplementedError("WIP: star multicast cost not yet implemented") \ No newline at end of file + ctx = isl.DEFAULT_CONTEXT + ingress, egress = self._spoke_loads(mcns) + total: isl.Val = isl.Val.zero(ctx) + for load in (ingress, egress): + summed: isl.PwQPolynomial = load.sum() + total = total.add( + summed.eval(isl.Point.zero(summed.domain().get_space())) + ) + zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) + return isl.PwQPolynomial.from_qpolynomial( + isl.QPolynomial.val_on_domain(zero_dim, total) + ) From c8a56a513417e16aa2ef38f4c17723a35be728f0 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Wed, 1 Jul 2026 16:04:43 -0400 Subject: [PATCH 12/19] edge pressure test --- .../distribuffers/test_edge_pressure.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 tests/not_working/distribuffers/test_edge_pressure.py diff --git a/tests/not_working/distribuffers/test_edge_pressure.py b/tests/not_working/distribuffers/test_edge_pressure.py new file mode 100644 index 00000000..0f272bc9 --- /dev/null +++ b/tests/not_working/distribuffers/test_edge_pressure.py @@ -0,0 +1,200 @@ +""" +Tests for per-edge memory pressure (link load) of the distributed transfer models. + +Where a model's ``hops`` is a single scalar, ``EdgePressure`` breaks the load out +per *physical* directed edge: how many multicast trees cross each link. Two models +are covered: + +- ``XYRoutingMulticastModel.edge_pressure`` -- directed mesh links + (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). The decisive, oracle-free + check is the invariant ``sum over edges of load == total XY hops``: the per-edge + loads must sum back to the already-validated A-F hop totals (4/6/6/3/4/448), so + a wrong decomposition fails here. Bottlenecks and a couple of individual edge + loads (hand-derived, geometry documented inline) pin the shape. + +- ``StarMulticastModel.edge_pressure`` -- the spokes realization of a + fully-connected fabric (``spoke_in[n]`` ingress, ``spoke_out[n]`` egress). For + an N-way all-to-all each node receives N-1 and sources 1, so the ingress spokes + are hottest at N-1. Tied to ``FullyConnectedMulticastModel`` by + ``sum over nodes of ingress == FullyConnected crossing count``. + +See accelforge/model/_looptree/reuse/isl/distributed/README.md. +""" + +import unittest +from pathlib import Path + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( + Fill, + Occupancy, + Tag, + SpatialTag, + TemporalTag, +) +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + XYRoutingMulticastModel, + StarMulticastModel, + FullyConnectedMulticastModel, +) +from .util import load_solutions + + +def construct_spacetime(dims: list) -> list[Tag]: + """Convert a list of dim-tag dicts (from yaml) into ``Tag`` objects.""" + spacetime: list[Tag] = [] + for dim in dims: + if dim["type"] == "Temporal": + spacetime.append(TemporalTag()) + elif dim["type"] == "Spatial": + spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) + return spacetime + + +# 1-D Manhattan distance over a line of GPUs, for the star/all-to-all cases. +_MANHATTAN_1D: str = ( + "{ [noc[gd] -> noc[gs]] -> hops[gd - gs] : gd >= gs;" + " [noc[gd] -> noc[gs]] -> hops[gs - gd] : gd < gs }" +) + + +def _eval_const(pwq: isl.PwQPolynomial) -> int: + """Evaluate a parameter-free piecewise quasi-polynomial to an int.""" + return int(str(pwq.eval(isl.Point.zero(pwq.domain().get_space())))) + + +class TestXYRoutingEdgePressure(unittest.TestCase): + """Per-mesh-edge pressure for XY routing, reusing the XY hop test geometries.""" + + TEST_CASES_FILE: str = Path(__file__).parent / "xy_routing" / "test_cases.yaml" + testcases: dict = load_solutions(TEST_CASES_FILE) + + def test_load_sums_to_hops(self): + """ + Invariant: summed per-edge load == total XY hops (the cross-check that the + edge decomposition is exact). Validated against the trusted A-F totals. + """ + for test in self.testcases: + expected = test["expected"]["xy_routing_hops"] + if expected is None: + continue + dim_tags = construct_spacetime(test["dims"]) + fill = Fill(dim_tags, test["fill"]) + occ = Occupancy(dim_tags, test["occ"]) + model = XYRoutingMulticastModel(test["dist_fn"]) + pressure = model.edge_pressure(fill, occ) + assert pressure.total() == expected, ( + f"Σ edge load {pressure.total()} != hops {expected}" + ) + + def test_case_F_bottleneck_and_edges(self): + """ + Case F (8x8, datum (d0,d1) at node (d0,d1), requested by all of column + x=d0): every datum of column c floods all of column c's 7 vertical links. + The upward link yedge_u[c,t] carries data with d1<=t (load t+1), the + downward yedge_d[c,t] carries data with d1>t (load 7-t); the busiest + directed link is 7 (the physical link total is 8). + """ + f = next(t for t in self.testcases if t["expected"]["xy_routing_hops"] == 448) + dim_tags = construct_spacetime(f["dims"]) + model = XYRoutingMulticastModel(f["dist_fn"]) + pressure = model.edge_pressure(Fill(dim_tags, f["fill"]), + Occupancy(dim_tags, f["occ"])) + assert pressure.bottleneck() == 7 + assert pressure.eval_edge("yedge_u", [0, 6]) == 7 # top link, all 7 below + assert pressure.eval_edge("yedge_d", [0, 1]) == 6 # links below row 1 + assert pressure.eval_edge("xedge_r", [0, 0]) == 0 # no horizontal traffic + + def test_monotone_overlap_bottleneck(self): + """ + Regression: the directed load is non-constant (a flooded up-link carries + ``t + 1`` trees), so the bottleneck must enumerate the edge domain, not + sample one point per piece. Three sources stacked up column 0 (rows 2, 1, + 0) all casting to (0, 3): the top up-link (rows 2->3) carries all three, so + the bottleneck is 3 even though lower links carry 1 and 2. + """ + tags = [SpatialTag(0, 0), SpatialTag(1, 0)] + manhattan = isl.Map.read_from_str(isl.DEFAULT_CONTEXT, ( + "{ [noc[xd,yd]->noc[xs,ys]]->hops[(xd-xs)+(yd-ys)] : xd>=xs and yd>=ys;" + " [noc[xd,yd]->noc[xs,ys]]->hops[-(xd-xs)+-(yd-ys)] : xdnoc[xs,ys]]->hops[-(xd-xs)+(yd-ys)] : xd=ys;" + " [noc[xd,yd]->noc[xs,ys]]->hops[(xd-xs)+-(yd-ys)] : xd>=xs and yddata[c] : (c=0 and x=0 and y=2) or (c=1 and x=0 and y=1)" + " or (c=2 and x=0 and y=0) }"))) + fill = Fill(tags, isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, "{ noc[x,y]->data[c] : x=0 and y=3 and 0<=c<3 }")) + pressure = XYRoutingMulticastModel(manhattan).edge_pressure(fill, occ) + assert pressure.eval_edge("yedge_u", [0, 2]) == 3 # top link, all three + assert pressure.bottleneck() == 3 # NOT 1 or 2 + + def test_single_tree_cases_have_unit_bottleneck(self): + """ + Cases A-E are single multicast trees, so no link is shared by two trees and + every edge carries load exactly 1. + """ + for test in self.testcases: + exp = test["expected"]["xy_routing_hops"] + if exp is None or exp == 448: # F has overlapping trees + continue + dim_tags = construct_spacetime(test["dims"]) + model = XYRoutingMulticastModel(test["dist_fn"]) + pressure = model.edge_pressure(Fill(dim_tags, test["fill"]), + Occupancy(dim_tags, test["occ"])) + assert pressure.bottleneck() == 1 + + +class TestStarSpokePressure(unittest.TestCase): + """Per-spoke pressure for the star (FC spokes) model on N-way all-to-all.""" + + @staticmethod + def _all_to_all(n: int): + """Build (fill, occ, dist_fn) for an N-GPU all-to-all on a line.""" + tags = [SpatialTag(0, 0)] + occ = Occupancy(tags, isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + f"{{ noc[g] -> data[d] : 0 <= g < {n} and d = g }}")) + fill = Fill(tags, isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + f"{{ noc[g] -> data[d] : 0 <= g < {n} and 0 <= d < {n} and d != g }}")) + dist_fn = isl.Map.read_from_str(isl.DEFAULT_CONTEXT, _MANHATTAN_1D) + return fill, occ, dist_fn + + def test_all_to_all_spoke_loads(self): + """ + N-way all-to-all: every node receives N-1 (ingress) and sources 1 (egress); + the bottleneck spoke is the ingress at N-1. + """ + for n in (4, 8): + fill, occ, dist_fn = self._all_to_all(n) + pressure = StarMulticastModel(dist_fn).edge_pressure(fill, occ) + for node in range(n): + assert pressure.eval_edge("spoke_in", [node]) == n - 1 + assert pressure.eval_edge("spoke_out", [node]) == 1 + assert pressure.bottleneck() == n - 1 + + def test_ingress_sum_equals_fully_connected_count(self): + """ + Cross-model invariant: Σ ingress over spokes == FullyConnected crossing + count (every crossing delivery is exactly one node's ingress). + """ + for n in (4, 8): + fill, occ, dist_fn = self._all_to_all(n) + pressure = StarMulticastModel(dist_fn).edge_pressure(fill, occ) + total_ingress = sum(pressure.eval_edge("spoke_in", [g]) for g in range(n)) + fc_hops = _eval_const( + FullyConnectedMulticastModel(dist_fn).apply(0, fill, occ).hops + ) + assert total_ingress == fc_hops == n * (n - 1) + + def test_star_hops_is_injections_plus_deliveries(self): + """Star scalar hops == Σ egress + Σ ingress == N + N(N-1).""" + for n in (4, 8): + fill, occ, dist_fn = self._all_to_all(n) + hops = _eval_const(StarMulticastModel(dist_fn).apply(0, fill, occ).hops) + assert hops == n + n * (n - 1) + + +if __name__ == "__main__": + unittest.main() From 81be62a4d3b89aeddc292fcd44b65c8154e46d05 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Wed, 1 Jul 2026 17:34:29 -0400 Subject: [PATCH 13/19] isl distribuffers refactor --- .../_looptree/reuse/isl/distributed/README.md | 214 +++++--- .../isl/distributed/distributed_buffers.py | 469 ++++++++++++------ .../model/_looptree/reuse/isl/spatial.py | 47 +- tests/not_working/distribuffers/helpers.py | 158 ++++++ .../spec/binding/valid_bindings.yaml | 114 ++--- .../distribuffers/test_contract.py | 367 ++++++++++++++ .../distribuffers/test_edge_pressure.py | 91 ++-- .../distribuffers/test_fully_connected.py | 70 +-- .../distribuffers/test_multicast.py | 66 +-- .../distribuffers/test_xy_routing.py | 66 +-- tests/not_working/distribuffers/util.py | 56 --- 11 files changed, 1151 insertions(+), 567 deletions(-) create mode 100644 tests/not_working/distribuffers/helpers.py create mode 100644 tests/not_working/distribuffers/test_contract.py delete mode 100644 tests/not_working/distribuffers/util.py diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index 03e6135a..cad8a6a5 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -17,11 +17,11 @@ implementations: | Model | File | Shape | |-------|------|-------| -| `SimpleLinkTransferModel` | [`../spatial.py`](../spatial.py) (line ~86) | Neighbor-to-neighbor mesh; no constructor state | -| `HypercubeMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) (line ~157) | Distance-aware worst-case multicast; takes a `dist_fn` | -| `FullyConnectedMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) (line ~265) | Fully-connected fabric; 1 hop per fabric crossing (see §6) | -| `XYRoutingMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | XY / dimension-order routing on a 2-D mesh; X-then-Y multicast tree (see §7) | -| `StarMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | **WIP / incomplete** — not yet usable | +| `SimpleLinkTransferModel` | [`../spatial.py`](../spatial.py) | Neighbor-to-neighbor mesh; no constructor state | +| `HypercubeMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | Distance-aware worst-case multicast; takes a `dist_fn` | +| `FullyConnectedMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | Fully-connected fabric; 1 hop per fabric crossing (see §6) | +| `XYRoutingMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | XY / dimension-order routing on a 2-D mesh; X-then-Y multicast tree; per-link `EdgePressure` (see §7–§8) | +| `StarMulticastModel` | [`distributed_buffers.py`](distributed_buffers.py) | Star / central-switch — the spokes realization of a fully-connected fabric; per-spoke `EdgePressure` (see §8). Parameter-free constant regime only, like XY | > **There is no registry or factory.** Models are constructed directly > (`HypercubeMulticastModel(dist_fn)`) and applied via `.apply(...)`. The only current usage is the @@ -64,11 +64,25 @@ A frozen dataclass (in `../spatial.py`) you must fully populate: | Field | Type | Meaning | |-------|------|---------| -| `fulfilled_fill` | `Transfers` | Fills satisfied by peer-to-peer transfers (a tagged map). | -| `unfulfilled_fill` | `Fill` | Fills *not* satisfied (must come from higher in the hierarchy). | +| `fulfilled_fill` | `Transfers` | Fills satisfied by peer-to-peer transfers (a tagged map) — the fills *covered* by a matched multicast source. | +| `unfulfilled_fill` | `Fill` | Fills *not* satisfied — no source held the datum, so it must come from higher in the hierarchy. | | `parent_reads` | `Reads` | Fills satisfied by parent-to-child reads. | | `hops` | `isl.PwQPolynomial` | The transfer cost metric across spacetime. | | `link_transfer` | `bool` | Metadata flag — whether this used link transfers. | +| `edge_pressure` | `Optional[EdgePressure]` | Per-directed-edge load backing `hops` (see §8), for models that define a per-link topology. Defaults to `None`. | + +`fulfilled_fill` and `unfulfilled_fill` **partition the fills exactly**: with +`covered = _covered_fills(mcs)` (the `{ dst -> data }` pairs `identify_mesh_casts` matched to a +source), `fulfilled = fills ∩ covered` and `unfulfilled = fills − covered`. A destination requesting +a datum that *no* source holds simply never appears in the multicast networks, so it lands in +`unfulfilled_fill` — it is never silently treated as fulfilled. + +`edge_pressure` is populated by the models with an explicit per-link topology +(`XYRoutingMulticastModel`: mesh links; `StarMulticastModel`: spokes) inside `apply()`, from the +same `identify_mesh_casts` result as `hops`. `HypercubeMulticastModel` and +`FullyConnectedMulticastModel` leave it `None` — a convex bounding box has no notion of individual +links, and a contention-free full mesh (one dedicated link per pair) has no shared-link pressure to +report. `Transfers` and `Reads` are thin `TaggedMap` subclasses; construct them as `Transfers(tags, map_)` / `Reads(tags, map_)`. @@ -86,12 +100,18 @@ A frozen dataclass (in `../spatial.py`) you must fully populate: HypercubeMulticastModel(dist_fn: isl.Map) ``` -`dist_fn` is a distance function `{ [src -> dst] -> [hops] }`. Two assumptions are baked in: +`dist_fn` is a distance function `{ [dst -> src] -> [hops] }` — note the orientation: its domain is +a **`[dst -> src]` pair** (destination first), because `identify_mesh_casts` composes it as +`fills_to_matches.apply_range(dist_fn)` onto `{ ... -> [dst -> src] }` maps. Two further caller +contracts: the tuple names in `dist_fn`'s domain must match the spacetime tuple names of +`fills`/`occs` (ISL raises on a name mismatch when `dist_fn` is applied), and its range tuple must +be named `hops` (the fully-connected and star models filter on `{ hops[h] : h >= 1 }` to tell +self-deliveries apart from fabric crossings). Two assumptions are baked in: 1. **Orthogonal dimensions / Manhattan distance** — each unit move along a dimension costs 1 hop, and dimensions are orthogonal in the metric space. 2. **Translational invariance** — distance depends only on the displacement: if - `|src − dst| = |src' − dst'|` then `dist_fn(src, dst) = dist_fn(src', dst')`. + `|src − dst| = |src' − dst'|` then `dist_fn(dst, src) = dist_fn(dst', src')`. These come from `calculate_extents_per_dim`, which the model relies on. @@ -113,7 +133,7 @@ occs.map_ { [spacetime] -> [data] } fills.map_ { [spacetime] -> [data] } ``` **Step A — `identify_mesh_casts(src_occupancy, dst_fill, dist_fn)`** -([`distributed_buffers.py`](distributed_buffers.py), line ~22). +([`distributed_buffers.py`](distributed_buffers.py)). For every datum, it pairs the destinations that request it with the *nearest* source that holds it. Conceptually: @@ -130,14 +150,14 @@ The result is the set of **multicast networks** (`mcns`): per datum, the destina their chosen source. **Step B — extents per dimension: `calculate_extents_per_dim(mcns)`** -([`distributed_buffers.py`](distributed_buffers.py), line ~98). +([`distributed_buffers.py`](distributed_buffers.py)). For each multicast network it unions the sources with the destinations, then for each NoC dimension projects away the others and takes `dim_max − dim_min`. That difference is the **extent** (the side length of the bounding box) along that dimension, returned as one `isl.PwAff` per dimension. **Step C — hypercube cost: `_cost_mesh_cast_hypercube(mcns)`** -([`distributed_buffers.py`](distributed_buffers.py), line ~213). +([`distributed_buffers.py`](distributed_buffers.py)). It folds the per-dimension extents into a single cost, then `.sum()`s over all networks. @@ -159,24 +179,34 @@ It folds the per-dimension extents into a single cost, then `.sum()`s over all n ### Return value ```python +# { dst -> data } fills actually covered by a matched source (D2). +covered: isl.Map = _covered_fills(mcs) + return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), # everything treated as fulfilled - parent_reads=Reads(occs.tags, mcs), # the multicast map - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty (map minus itself) - hops=result, # the PwQPolynomial cost + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), + parent_reads=Reads(occs.tags, mcs), # the multicast map + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), + hops=result, # the PwQPolynomial cost link_transfer=True, + # No per-link decomposition defined for the hypercube abstraction. + edge_pressure=None, ) ``` -Note the idiom `fills.map_.subtract(fills.map_)` — an **empty map over the right space**. This is -how you build a typed empty relation without hardcoding dimensions. +Note the partition idiom: `_covered_fills(mcs)` reshapes the multicast networks +`{ [data] -> [dst -> src] }` into `{ dst -> data }` (via +`range_reverse().uncurry().domain_factor_domain().reverse()` — "forget which source served it, keep +dst and data"), which lines up with `fills.map_` so `intersect`/`subtract` split the fills into the +covered and uncovered halves. Earlier revisions returned the *entire* fill map as `fulfilled_fill` +and an always-empty `unfulfilled_fill` (`fills.map_.subtract(fills.map_)`); that silently +mis-reported fills whose datum no source holds, and was fixed in D2 — do not copy that idiom. --- ## 4. Contrast: `SimpleLinkTransferModel` For comparison (the *other* valid shape — no constructor state), `SimpleLinkTransferModel` -([`../spatial.py`](../spatial.py), line ~86): +([`../spatial.py`](../spatial.py)): - Takes **no `dist_fn`**; constructed as `SimpleLinkTransferModel()`. - Asserts `fills.tags == occs.tags`. @@ -188,7 +218,7 @@ For comparison (the *other* valid shape — no constructor state), `SimpleLinkTr - Has an early-out: if there is no temporal dimension or no spatial dimension, nothing moves, so it returns an empty/zero `TransferInfo`. -So the two existing models bracket the design space: **stateful + distance-aware** (hypercube) vs. +So these two models bracket the design space: **stateful + distance-aware** (hypercube) vs. **stateless + fixed-topology** (simple link). --- @@ -202,11 +232,18 @@ So the two existing models bracket the design space: **stateful + distance-aware 3. **Implement `apply(self, buff, fills, occs) -> TransferInfo`.** If your model is mesh/multicast shaped, reuse the existing kernels: - `identify_mesh_casts(occs.map_, fills.map_, self.dist_fn)` to get `{ [data] -> [dst -> src] }`. + - `_covered_fills(mcs)` to partition the fills into fulfilled/unfulfilled. - `calculate_extents_per_dim(mcns)` if you want per-dimension bounding-box extents. - Otherwise write your own cost kernel over the ISL maps. + - `_edge_pressure_from_links(...)` / `_const_pwq(...)` / `_eval_const(...)` if you report a + per-link `EdgePressure` (see §8) and derive `hops` from it. + Otherwise write your own cost kernel over the ISL maps. Call `identify_mesh_casts` **once** per + `apply` and derive everything (`hops`, `edge_pressure`, the fill partition) from that single + result, so the outputs can never disagree. 4. **Honor the invariants.** Assert `fills.tags == occs.tags` if your model needs aligned tags. - Build `hops` as an `isl.PwQPolynomial` over the correct domain, and build empty maps with the - `map_.subtract(map_)` idiom rather than hardcoding spaces. + Build `hops` as an `isl.PwQPolynomial` over the correct domain, and partition the fills with + `_covered_fills` (`fulfilled = fills ∩ covered`, `unfulfilled = fills − covered`) rather than + declaring everything fulfilled. Decide deliberately whether your topology defines an + `edge_pressure` (per-link decomposition) or leaves it `None`. 5. **Add a test** mirroring [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py): a YAML-driven gamut of `(dims, fill, occ, dist_fn, expected_hops)` cases. @@ -228,6 +265,7 @@ from accelforge.model._looptree.reuse.isl.spatial import ( from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( identify_mesh_casts, calculate_extents_per_dim, + _covered_fills, ) @@ -241,19 +279,26 @@ class MyTransferModel(TransferModel): def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: # TODO (optional): assert fills.tags == occs.tags - # 1. Group destinations with their nearest source per datum. + # 1. Group destinations with their nearest source per datum. Call this + # ONCE and derive every output from the same `mcs`. mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) # 2. TODO: compute your cost as an isl.PwQPolynomial. hops: isl.PwQPolynomial = self._cost(mcs) - # 3. Assemble the result. + # 3. Partition the fills by whether a source was matched (D2). + covered: isl.Map = _covered_fills(mcs) # { dst -> data } + + # 4. Assemble the result. return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), hops=hops, link_transfer=True, + # TODO: an EdgePressure if your topology defines per-link loads + # (see §8); None if it has no per-link decomposition. + edge_pressure=None, ) def _cost(self, mcns: isl.Map) -> isl.PwQPolynomial: @@ -266,9 +311,12 @@ class MyTransferModel(TransferModel): - [ ] Subclasses `TransferModel`, implements `apply`. - [ ] Constructor state matches what the cost kernel needs. - [ ] `hops` is an `isl.PwQPolynomial` over the right domain. -- [ ] `fulfilled_fill` + `parent_reads` + `unfulfilled_fill` partition the fills correctly for your - model's semantics. -- [ ] Empty maps built with `map_.subtract(map_)`, not hardcoded. +- [ ] `fulfilled_fill` + `unfulfilled_fill` partition the fills via `_covered_fills(mcs)` + (`fills ∩ covered` / `fills − covered`), not "everything fulfilled / empty unfulfilled". +- [ ] `identify_mesh_casts` is called once per `apply`; `hops`, `edge_pressure`, and the fill + partition all derive from the same result. +- [ ] `edge_pressure` is a deliberate choice: an `EdgePressure` if the topology has per-link loads, + `None` (with a comment saying why) if not. - [ ] A YAML-driven gamut test exists. --- @@ -299,12 +347,17 @@ class FullyConnectedMulticastModel(TransferModel): def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_fully_connected(mcs) + # { dst -> data } fills actually covered by a matched source (D2). + covered: isl.Map = _covered_fills(mcs) + return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), # empty + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), hops=result, link_transfer=True, + # No per-link decomposition defined for the full-mesh abstraction. + edge_pressure=None, ) def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: @@ -388,38 +441,47 @@ model): source `(1,0)` casting to `(0,2)` and `(2,2)`: ### Cost kernel +There is no standalone cost method: the cost *is* the per-edge decomposition, aggregated. `apply()` +decomposes every tree onto directed mesh links, wraps the per-link loads as an `EdgePressure` +(see §8), and derives `hops` from its total — one aggregation path, so the scalar and the per-edge +view can never disagree: + ```python -def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: - # X-phase: horizontal links along each source row (x-extent of {dsts} ∪ {src}). - x_extent = calculate_extents_per_dim(mcns)[0] - x_links = self._eval_const(isl.PwQPolynomial.from_pw_aff(x_extent).sum()) - - # Y-phase: vertical links per (data, src, destination column), each column - # spanning from the source row ys to the destinations in that column. - per_src = mcns.range_reverse().uncurry() # {[data->src] -> dst} - split_col = isl.Map.read_from_str(ctx, - "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }") - dst_y = per_src.apply_range(split_col).uncurry() # {[data->src->col] -> yv[y]} - src_y = per_src.domain().unwrap().range_map().apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }")) - src_row = dst_y.domain().unwrap().range_product(src_y).uncurry() - col_ys = dst_y.union(src_row) # source row ∪ dst y's per column - - # Count {ymin <= p < ymax} links per column via cardinality (robust where - # summing a min/max polynomial is not). - ge_min = isl.Map.read_from_str(ctx, "{ yv[ymin] -> p[t] : t >= ymin }") - lt_max = isl.Map.read_from_str(ctx, "{ yv[ymax] -> p[t] : t < ymax }") - links = col_ys.lexmin().apply_range(ge_min).intersect( - col_ys.lexmax().apply_range(lt_max)) - y_links = self._eval_const(links.wrap().card()) - ... # total = x_links + y_links, returned as a constant PwQPolynomial +def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: + # `identify_mesh_casts` is called exactly once; `hops`, `edge_pressure`, + # and the fill partition all derive from this single `mcs` (D4/D5). + mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) + links: list[isl.Map] = self._directed_mesh_links(mcs) # see §8 + pressure: EdgePressure = _edge_pressure_from_links(links) + # `hops` == the sum of per-edge loads (Σ_edges load == total link count), + # wrapped as the constant PwQPolynomial `TransferInfo.hops` expects. + hops: isl.PwQPolynomial = _const_pwq(pressure.total()) + covered: isl.Map = _covered_fills(mcs) # { dst -> data } + + return TransferInfo( + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), + parent_reads=Reads(occs.tags, mcs), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), + hops=hops, + link_transfer=True, + edge_pressure=pressure, + ) ``` +`_directed_mesh_links` builds the X segments explicitly along each source row (out to every +destination column, split rightward/leftward at the source column `xs`) and the Y segments down +each destination column from the source row (split upward/downward at `ys`), keeping every link's +identity — see §8 for the edge naming. + Notes / limitations: -- **2-D `noc[x, y]` only** (no temporal dims); N-D dimension-order routing is a TODO. The helper maps - hardcode the `noc[x, y]` shape. -- The Y-term is counted via `card()` of the per-column link set rather than summing a min/max - polynomial — the latter trips a barvinok `summate` assertion at scale (e.g. the 8×8 case). +- **2-D node tuples only** (no temporal dims); `apply` raises a `ValueError` ("XYRoutingMulticastModel + requires a 2-D node tuple (X then Y)...") for any other arity — N-D dimension-order routing is a + TODO. The tuple *name* is generic: it is read off the maps at `apply` time (via + `_mesh_node_tuple`), so `noc[x, y]`, `pe[x, y]`, etc. all work as long as `fills`, `occs`, and + `dist_fn` agree on it (see the caller contract in §3). +- Each per-column/per-row link set is built explicitly and counted via `card()` rather than summing + a min/max polynomial — the latter trips a barvinok `summate` assertion at scale (e.g. the 8×8 + case). - Returns a **parameter-free constant** (the validated regime); parametric spacetimes are future work. ### Tested @@ -445,14 +507,18 @@ has a finite bandwidth**, so the *busiest* link is what saturates first. `EdgePr load broken out per directed physical edge — `{ edge -> number-of-trees-crossing-it }` — which is exactly the quantity the production symbolic path calls `max_traffic` and feeds into the `Network` latency formula `max(max_hops·latency, max_link_traffic / throughput)` -([`frontend/arch/components.py`](../../../../frontend/arch/components.py)). Two models expose it -today via an `edge_pressure(fills, occs) -> EdgePressure` method. +([`frontend/arch/components.py`](../../../../frontend/arch/components.py)). It is exposed as the +`TransferInfo.edge_pressure` field: `model.apply(buff, fills, occs).edge_pressure`, an +`Optional[EdgePressure]` populated by `XYRoutingMulticastModel` and `StarMulticastModel` inside +`apply()` (from the same `identify_mesh_casts` result as `hops`, so the two are always consistent) +and `None` for `HypercubeMulticastModel` / `FullyConnectedMulticastModel` (see §2). There is no +standalone `edge_pressure(fills, occs)` method. ### The primitive -After `identify_mesh_casts` fixes the `[src] -> [dst]` pairs, build, per directed edge type, a map +After `identify_mesh_casts` fixes the `[dst -> src]` pairs, build, per directed edge type, a map `M = { [data -> src] -> edge }` associating each multicast **tree** with every edge its route -crosses. Then +crosses. Then (`_edge_pressure_from_links` does exactly this, unioning the per-type results) ```python load = M.reverse().card() # { edge -> #trees crossing it } @@ -460,16 +526,18 @@ load = M.reverse().card() # { edge -> #trees crossing it } The key is the tree `(data, src)`, **not** the destination: within one tree a link is traversed once regardless of how many leaves hang off it, so this counts *pressure* (distinct flows on a link), not -summed hops. `EdgePressure.bottleneck()` returns the max load (enumerating pieces; parameter-free -regime, like `_cost_xy`); `eval_edge(name, coords)` looks up one edge. +summed hops. `EdgePressure.total()` sums the load over every edge (this is what XY/Star `hops` is +built from, via `_const_pwq`); `bottleneck()` returns the max load (enumerating the finite, +parameter-free edge domain); `eval_edge(name, coords)` looks up one edge. ### XY routing → directed mesh edges `XYRoutingMulticastModel._directed_mesh_links` decomposes each tree onto four directed link types: `xedge_r`/`xedge_l` along the source row (split at the source column `xs`) and `yedge_u`/`yedge_d` -down each destination column (split at the source row `ys`). It reuses the §7 link-set construction -but **keeps the edge identity** instead of collapsing to a count (and builds the X links explicitly — -`calculate_extents_per_dim` only yields a scalar length and discards *which* links are used). +down each destination column (split at the source row `ys`). This **is** the §7 cost kernel — every +link is built explicitly with its identity kept (not via `calculate_extents_per_dim`, which only +yields a scalar length and discards *which* links are used), and the scalar `hops` is just +`EdgePressure.total()` over these maps. Decisive cross-check (no oracle needed): **`Σ_edges load == total XY hops`**. The per-edge loads must sum back to the already-trusted A–F totals (4/6/6/3/4/448), so a wrong decomposition fails. Worked F @@ -494,9 +562,11 @@ deliveries). ### Tested [`tests/not_working/distribuffers/test_edge_pressure.py`](../../../../../../tests/not_working/distribuffers/test_edge_pressure.py) -— the XY `Σ load == hops` invariant over A–F, F's bottleneck/edge loads (`7`, `yedge_u[0,6]=7`, -`yedge_d[0,1]=6`), single-tree unit bottlenecks, and the star spoke loads / `Σ ingress == FC count` -invariant for 4- and 8-GPU all-to-all. Run: +— the XY `Σ load == hops` invariant over A–F (with the pressure taken from +`apply(...).edge_pressure`), F's bottleneck/edge loads (`7`, `yedge_u[0,6]=7`, `yedge_d[0,1]=6`), +single-tree unit bottlenecks, and the star spoke loads / `Σ ingress == FC count` invariant for 4- +and 8-GPU all-to-all (star `hops` = injections + deliveries: `16`/`64`, ingress bottleneck +`3`/`7`). Run: ```bash PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index b1dd416f..ce2b6447 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -77,10 +77,10 @@ def total(self) -> int: self.load.foreach_pw_qpolynomial(pieces.append) total = 0 for pwq in pieces: - summed: isl.PwQPolynomial = pwq.sum() - total += int( - str(summed.eval(isl.Point.zero(summed.domain().get_space()))) - ) + # `.sum()` collapses the edge-indexed domain away (summing over every + # edge of this piece), leaving a 0-set-dim, parameter-free + # polynomial -- exactly `_eval_const`'s precondition. + total += _eval_const(pwq.sum()) return total def bottleneck(self) -> int: @@ -134,6 +134,169 @@ def eval_edge(self, name: str, coords: list[int]) -> int: return 0 +def _eval_const(pwq: isl.PwQPolynomial) -> int: + """ + Evaluate a parameter-free, already-reduced piecewise quasi-polynomial to its + scalar value. + + # Design (D6): this idiom -- evaluate at the space's zero point to pull a + # single int out of an ``isl.PwQPolynomial`` -- previously existed four times + # with drifting return types: ``EdgePressure.total`` did it inline and + # returned ``int``, ``XYRoutingMulticastModel._eval_const`` returned the raw + # ``isl.Val`` (forcing every caller to convert), ``_cost_star_multicast`` + # inlined a variant that accumulated ``isl.Val``s, and the test suite had its + # own copy. Consolidating to one module-level, ``int``-returning helper means + # every call site agrees on both the operation and its return type, and the + # test suite can import this directly instead of re-deriving it. + + Parameters + ---------- + pwq: + A piecewise quasi-polynomial with **no free set dimensions** (its domain + is a point, up to parameters) and **no parameters** -- e.g. the output of + ``.card()`` on a parameter-free map/set, ``.sum()`` on a + parameter-free polynomial (which sums away every set dimension), or a + constant built by ``_const_pwq``. This is the "parameter-free regime" + every distributed model in this module is validated in; a piecewise + quasi-polynomial that still varies over real domain points or + parameters is evaluated only at the space's zero point, which is + **not** meaningful for such inputs -- callers are responsible for + reducing to a true constant first (via ``.sum()``/``.card()``). + + Returns + ------- + The polynomial's constant value as a Python ``int``. + + Notes + ----- + ``isl.Val`` has no direct ``int()`` conversion in this islpy build (it + raises ``TypeError``); round-tripping through ``str()`` is the working + idiom already used throughout this module and ``EdgePressure``. + """ + return int(str(pwq.eval(isl.Point.zero(pwq.domain().get_space())))) + + +def _const_pwq(value: int) -> isl.PwQPolynomial: + """ + Build a 0-dimensional, parameter-free constant ``isl.PwQPolynomial`` equal to + ``value``. + + # Design (D5): ``_cost_xy`` and ``_cost_star_multicast`` each independently + # built this "wrap an int as a constant PwQPolynomial" idiom (zero-dim space + # + ``QPolynomial.val_on_domain`` + ``PwQPolynomial.from_qpolynomial``) right + # before returning. Both models now compute their cost by aggregating an + # ``EdgePressure`` (``pressure.total()``) and need to hand that scalar back + # as a ``TransferInfo.hops`` polynomial, so this is factored out once rather + # than duplicated in ``XYRoutingMulticastModel.apply`` and + # ``StarMulticastModel.apply``. + + Parameters + ---------- + value: + The integer the returned polynomial evaluates to everywhere (it has no + domain dimensions or parameters to vary over). + + Returns + ------- + An ``isl.PwQPolynomial`` over the empty (0-dim, 0-param) space, suitable + wherever a parameter-free ``hops`` cost is expected. Round-trips through + ``_eval_const`` back to ``value``. + """ + ctx = isl.DEFAULT_CONTEXT + zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) + return isl.PwQPolynomial.from_qpolynomial( + isl.QPolynomial.val_on_domain(zero_dim, isl.Val(value, ctx)) + ) + + +def _covered_fills(mcs: isl.Map) -> isl.Map: + """ + Recover the ``{ dst -> data }`` fills actually covered by a matched + multicast source, from ``identify_mesh_casts``'s result. + + # Design (D2): all four models used to return + # ``fulfilled_fill = fills`` (the *entire* fill map, unconditionally) and + # ``unfulfilled_fill = fills.map_.subtract(fills.map_)`` (always empty -- + # subtracting a map from itself). That silently dropped the case + # ``identify_mesh_casts`` is explicitly built to detect: a destination + # requesting a datum that *no* source holds simply never appears in ``mcs``, + # so it should show up as unfulfilled, not as (falsely) fulfilled. + # + # ``mcs`` is ``{ [data] -> [dst -> src] }`` (one entry per *matched* + # dst/datum pair -- see ``identify_mesh_casts``). To turn that into the + # ``{ dst -> data }`` shape that lines up with ``fills.map_`` (also + # ``{ dst -> data }``, see ``Fill``/``Occupancy`` in + # ``mapping_to_isl/types.py``) for a direct ``intersect``/``subtract``: + # 1. ``range_reverse()`` { data -> [dst -> src] } -> { data -> [src -> dst] } + # 2. ``uncurry()`` { data -> [src -> dst] } -> { [data -> src] -> dst } + # 3. ``domain_factor_domain()`` { [data -> src] -> dst } -> { data -> dst } + # (keeps the *domain*'s domain factor -- i.e. drops ``src`` -- while + # preserving the ``-> dst`` range; this is exactly "forget which + # source served it, keep dst and data".) + # 4. ``reverse()`` { data -> dst } -> { dst -> data } + # Empirically verified against a hand-built example (see the scratchpad + # snippet referenced in the PR/commit that introduced this function) before + # being wired in here. + + Parameters + ---------- + mcs: + ``{ [data] -> [dst -> src] }``, the multicast-network map returned by + ``identify_mesh_casts``. + + Returns + ------- + ``{ dst -> data }`` -- every (destination, datum) pair that has a matched + source in ``mcs``. A model's fill partition is then + ``fulfilled = fills.map_.intersect(covered)`` and + ``unfulfilled = fills.map_.subtract(covered)``. + """ + return mcs.range_reverse().uncurry().domain_factor_domain().reverse() + + +def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: + """ + Read the spacetime/node tuple's name and dimensionality off a multicast + network map, instead of assuming a hardcoded name. + + # Design (D1): ``XYRoutingMulticastModel._directed_mesh_links`` and + # ``StarMulticastModel._spoke_loads`` used to hardcode the ISL tuple name + # ``'noc'`` directly into every map string they built + # (e.g. ``"{ noc[x, y] -> ... }"``). Any caller using a different spacetime + # tuple name (real pipelines derive names from occupancy maps -- see + # ``SimpleLinkTransferModel.apply`` in ``spatial.py``, which reads + # ``occs.map_.get_tuple_name(isl.dim_type.in_)`` rather than assuming a + # literal) would hit a raw ISL assertion deep inside these helpers instead + # of a clear error, because the name in the caller's maps would silently + # fail to match the literal ``'noc'`` baked into the helper's map strings. + # Reading the name back off ``mcns`` (already threaded through + # ``identify_mesh_casts`` from the caller's ``occs``/``fills``) and + # interpolating it into every map string generalizes both helpers to any + # tuple name, following the same precedent. + + Parameters + ---------- + mcns: + ``{ [data] -> [dst -> src] }``, the multicast-network map returned by + ``identify_mesh_casts``. Both ``dst`` and ``src`` share the same node + tuple (``identify_mesh_casts``'s caller contract requires + ``src_occupancy``'s and ``dst_fill``'s domains to name the same + spacetime tuple), so it suffices to read the name/dims off one side + (``dst``, via ``.range().unwrap()``'s domain). + + Returns + ------- + ``(name, dims)``: the node tuple's ISL tuple name (e.g. ``"noc"`` or + ``"pe"``) and its dimensionality. Correct even when ``mcns`` is empty -- + ISL preserves space/tuple-name information on empty relations (verified + empirically), so this does not require any data to be present. + """ + node_space: isl.Map = mcns.range().unwrap() + name: str = node_space.get_tuple_name(isl.dim_type.in_) + dims: int = node_space.dim(isl.dim_type.in_) + return name, dims + + def _edge_pressure_from_links(edge_maps: list[isl.Map]) -> EdgePressure: """ Turn directed flow maps into an ``EdgePressure``. @@ -178,11 +341,28 @@ def identify_mesh_casts( An isl.Map of the form { [dst] -> [data] } corresponding to the data requested at the element at space `dst`. dist_fn: - A distance function { [src -> dst] -> [hops] } that accepts two points in - space, corresponding to the `src` and `dst`, and returns the distance + A distance function { [dst -> src] -> [hops] } that accepts two points in + space, corresponding to the `dst` and `src`, and returns the distance between the two points in terms of `hops`, a quantized atomic distance of data transmission cost. + # Design (D3): this Parameters entry previously read `{ [src -> dst] -> + # [hops] }`, the opposite of what the code below actually does -- + # `fills_to_matches.apply_range(dist_fn)` composes a `{ ... -> [dst -> + # src] }` map with `dist_fn`, which only type-checks (and only produces + # the intended "distance from this dst to this candidate src" value) if + # `dist_fn`'s domain is `[dst -> src]`. Every concrete `dist_fn` in the + # test suite and every caller in `distributed_buffers.py` already builds + # it this way; this is a documentation-only fix, no behavior change. + + Caller contract: the tuple names of `dst_fill`'s and `src_occupancy`'s + domains (the spacetime/node tuple, e.g. `noc[x, y]`) must match the + corresponding tuple names in `dist_fn`'s domain -- ISL will raise on a + name mismatch when `dist_fn` is applied. `dist_fn`'s range tuple must be + named `hops` (consumers such as `FullyConnectedMulticastModel` and + `StarMulticastModel` filter on `{ hops[h] : h >= 1 }` to distinguish + self-deliveries from fabric-crossing ones). + Returns ------- { [data] -> [dst -> src] } where { [dst] -> [data] } and { [src] -> [data] } are in @@ -338,19 +518,30 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ------- Fills that were fulfilled, Fills that were unfilled, and parent reads per position in spacetime. Then, gets hops per timestep. + + `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether + `identify_mesh_casts` found a matched source for that (dst, data) pair + (D2) -- a fill with no source holding its datum is `unfulfilled_fill`, + not silently treated as fulfilled. `edge_pressure` is left at its + default (`None`): the hypercube model costs a convex bounding box, which + has no notion of individual links to report pressure on (D4). """ mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_mesh_cast_hypercube(mcs) + # { dst -> data } fills actually covered by a matched source (D2). + covered: isl.Map = _covered_fills(mcs) # TODO: Read once from all buffers, assert that # card(mcs) == tensor_size * duplication factor n_meshcasts: int = mcs.card() return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), hops=result, link_transfer=True, + # No per-link decomposition defined for the hypercube abstraction. + edge_pressure=None, ) def _cost_mesh_cast_hypercube(self, mcns: isl.Map) -> int: @@ -459,16 +650,26 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- A TransferInfo whose `hops` is the number of fabric-crossing deliveries. + `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether + `identify_mesh_casts` matched a source (D2). `edge_pressure` is left at + its default (`None`): this model treats the fabric as a + contention-free full mesh (one dedicated link per pair), so there is no + per-link pressure to report -- see `StarMulticastModel` for the spokes + realization of the same fabric that does expose it (D4). """ mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_fully_connected(mcs) + # { dst -> data } fills actually covered by a matched source (D2). + covered: isl.Map = _covered_fills(mcs) return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), hops=result, link_transfer=True, + # No per-link decomposition defined for the full-mesh abstraction. + edge_pressure=None, ) def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: @@ -529,8 +730,15 @@ class XYRoutingMulticastModel(TransferModel): Preconditions ------------- - The NoC is two-dimensional, ``noc[x, y]`` (no temporal dimensions in the - spacetime), with ``x`` routed before ``y``. N-dimensional dimension-order + The NoC is two-dimensional (no temporal dimensions in the spacetime), with + the first coordinate routed before the second (X then Y). The node tuple's + *name* is generic -- read off ``fills``/``occs`` at ``apply`` time rather + than hardcoded (D1), so any name works (e.g. ``noc[x, y]`` or + ``pe[x, y]``) as long as ``fills``, ``occs``, and ``self.dist_fn`` all agree + on it (see the caller contract on ``identify_mesh_casts``'s ``dist_fn`` + parameter). The tuple must be exactly 2-D; ``apply`` raises ``ValueError`` + otherwise (a non-2-D tuple used to reach a raw, opaque ISL assertion deep + inside ``_directed_mesh_links`` instead). N-dimensional dimension-order routing is a future extension. The returned cost is a parameter-free constant (the validated regime); parametric spacetimes are not yet supported. @@ -559,7 +767,7 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ---------- buff: The buffer whose spatial analysis is being considered. Unused; the - topology is captured by ``dist_fn`` and the ``noc[x, y]`` coordinates. + topology is captured by ``dist_fn`` and the node coordinates. fills: The fill of `buffer` across time from parents. occs: @@ -567,45 +775,46 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- - A TransferInfo whose `hops` is the total XY-routing link count. + A TransferInfo whose `hops` is the total XY-routing link count and whose + `edge_pressure` is the per-directed-mesh-link decomposition backing it + (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d`` -- see + `_directed_mesh_links`). `hops` is derived from the same + `EdgePressure` (`pressure.total()`, D5) rather than recomputed + independently, so the two can never disagree. + `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether + `identify_mesh_casts` matched a source (D2). + + Raises + ------ + ValueError + If the node tuple embedded in `fills`/`occs` is not exactly 2-D + (XY routing is only defined for a 2-D mesh; see `_directed_mesh_links`). """ + # Design (D4): `identify_mesh_casts` is called exactly once per `apply`; + # both `hops` and `edge_pressure` are derived from this single `mcs` + # (previously the now-removed `edge_pressure` method recomputed it a + # second time from scratch, wasting the ISL/barvinok work and risking + # the two falling out of sync if `dist_fn`/inputs were mutated between + # calls). mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - result: isl.PwQPolynomial = self._cost_xy(mcs) + links: list[isl.Map] = self._directed_mesh_links(mcs) + pressure: EdgePressure = _edge_pressure_from_links(links) + # (D5) `hops` is the aggregation `EdgePressure.total()` already performs + # (sum of per-edge loads == total link count), wrapped as the constant + # `TransferInfo.hops` expects -- not a second, independent aggregation. + hops: isl.PwQPolynomial = _const_pwq(pressure.total()) + # { dst -> data } fills actually covered by a matched source (D2). + covered: isl.Map = _covered_fills(mcs) return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), - hops=result, + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), + hops=hops, link_transfer=True, + edge_pressure=pressure, ) - def edge_pressure(self, fills: Fill, occs: Occupancy) -> EdgePressure: - """ - Per-edge memory pressure (link load) of the XY-routing transfer. - - After ``identify_mesh_casts`` fixes the ``[src] -> [dst]`` pairs, this - decomposes each multicast tree onto the directed mesh links it traverses - and counts, per link, how many trees cross it. The result is the load a - per-link bandwidth limit acts on; ``EdgePressure.bottleneck`` is the - busiest link. - - Parameters - ---------- - fills: - The fill of `buffer` across time from parents. - occs: - The occupancy of `buffer` across time. - - Returns - ------- - An ``EdgePressure`` over the four directed mesh-edge types - (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). Its ``total`` equals - this model's ``hops`` -- the cross-check that the decomposition is exact. - """ - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - return _edge_pressure_from_links(self._directed_mesh_links(mcs)) - def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: """ Decompose every multicast tree in `mcns` onto the directed mesh links it @@ -629,9 +838,32 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: is the rightward link in row ``ys`` between columns ``t`` and ``t + 1``; ``yedge_u[x, t]`` is the upward link in column ``x`` between rows ``t`` and ``t + 1`` (and ``_l`` / ``_d`` the opposite directions). + + Raises + ------ + ValueError + If the node tuple embedded in ``mcns`` is not exactly 2-D. (D1) Every + map string below is built assuming a 2-D ``name[x, y]`` node tuple; + a 3-D-or-more tuple (e.g. ``noc[x, y, t]``) used to fail deep inside + an ``isl.Map.read_from_str``/``apply_range`` call with a raw, + uninformative ISL assertion instead of a clear error here. """ ctx = isl.DEFAULT_CONTEXT - # { [data -> src] -> dst noc[x, y] } and a handle on the source per tree. + # Design (D1): read the node tuple's name (and validate its + # dimensionality) off `mcns` instead of assuming the literal `'noc'`, + # following the precedent in `spatial.py`'s + # `SimpleLinkTransferModel.apply` (`occs.map_.get_tuple_name(...)`). + # Every map string below interpolates `name` so this method works for + # any 2-D node tuple, not just one literally called `noc`. + name, dims = _mesh_node_tuple(mcns) + if dims != 2: + raise ValueError( + "XYRoutingMulticastModel requires a 2-D node tuple " + f"(X then Y); got tuple '{name}' with {dims} dimensions. " + "N-dimensional dimension-order routing is not yet supported." + ) + + # { [data -> src] -> dst name[x, y] } and a handle on the source per tree. per_src: isl.Map = mcns.range_reverse().uncurry() keymap: isl.Map = per_src.domain().unwrap().range_map() # [data->src] -> src @@ -639,13 +871,14 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: # Key each destination's y by its column: { [data->src->col] -> yv[y] }. dst_y: isl.Map = per_src.apply_range( isl.Map.read_from_str( - ctx, "{ noc[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }" + ctx, + "{ %s[x, y] -> [col[x'] -> yv[y']] : x' = x and y' = y }" % name, ) ).uncurry() # Inject the source row ys into every destination column so each Y segment # starts from the source. src_y: isl.Map = keymap.apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> yv[ys] }") + isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> yv[ys] }" % name) ) src_row: isl.Map = dst_y.domain().unwrap().range_product(src_y).uncurry() col_ys: isl.Map = dst_y.union(src_row) @@ -663,7 +896,7 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: # { [data->src] -> [[col[x] -> p[t]] -> ysv[ys]] }. y_with_ys: isl.Map = ylinks.curry().range_product( keymap.apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> ysv[ys] }") + isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> ysv[ys] }" % name) ) ) yedge_u: isl.Map = y_with_ys.apply_range( @@ -682,10 +915,10 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: # explicitly, not from calculate_extents_per_dim, which keeps only the # extent length and discards which links are used.) col_x: isl.Map = per_src.apply_range( - isl.Map.read_from_str(ctx, "{ noc[x, y] -> cx[x] }") + isl.Map.read_from_str(ctx, "{ %s[x, y] -> cx[x] }" % name) ).union( keymap.apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> cx[xs] }") + isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> cx[xs] }" % name) ) ) xlinks: isl.Map = col_x.lexmin().apply_range( @@ -699,7 +932,7 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: # part of the edge identity: { [data->src] -> [ex[t] -> xy[xs, ys]] }. x_with_src: isl.Map = xlinks.range_product( keymap.apply_range( - isl.Map.read_from_str(ctx, "{ noc[xs, ys] -> xy[xs, ys] }") + isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> xy[xs, ys] }" % name) ) ) xedge_r: isl.Map = x_with_src.apply_range( @@ -715,39 +948,6 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: return [xedge_r, xedge_l, yedge_u, yedge_d] - def _cost_xy(self, mcns: isl.Map) -> isl.PwQPolynomial: - """ - Total XY-routing link count for the multicast networks `mcns`. - - Equals the sum of the per-edge loads from ``_directed_mesh_links`` (every - directed link of every tree counted once), returned as a parameter-free - constant. This is the X-phase links (per source row) plus the Y-phase - links (per destination column). - - Parameters - ---------- - mcns: - Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`, - grouped per destination by nearest source. - - Returns - ------- - The total link count as a constant piecewise quasi-polynomial. - """ - ctx = isl.DEFAULT_CONTEXT - total: isl.Val = isl.Val.zero(ctx) - for edge_map in self._directed_mesh_links(mcns): - total = total.add(self._eval_const(edge_map.wrap().card())) - zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) - return isl.PwQPolynomial.from_qpolynomial( - isl.QPolynomial.val_on_domain(zero_dim, total) - ) - - @staticmethod - def _eval_const(pwq: isl.PwQPolynomial) -> isl.Val: - """Evaluate a parameter-free piecewise quasi-polynomial to its value.""" - return pwq.eval(isl.Point.zero(pwq.domain().get_space())) - class StarMulticastModel(TransferModel): """ @@ -806,48 +1006,39 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- A TransferInfo whose `hops` is the total spoke traversals (injections plus - deliveries). Per-spoke load is available via ``edge_pressure``. + deliveries) and whose `edge_pressure` is the per-spoke decomposition + backing it (``spoke_in[n]``/``spoke_out[n]`` -- see `_spoke_loads`). + `hops` is derived from the same `EdgePressure` (`pressure.total()`, D5) + rather than recomputed independently, so the two can never disagree. + `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether + `identify_mesh_casts` matched a source (D2). """ + # Design (D4): `identify_mesh_casts` is called exactly once per `apply`; + # both `hops` and `edge_pressure` are derived from this single `mcs` + # (previously the now-removed `edge_pressure` method recomputed it a + # second time from scratch). mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - result: isl.PwQPolynomial = self._cost_star_multicast(mcs) + ingress, egress = self._spoke_loads(mcs) + acc: isl.UnionPwQPolynomial = isl.UnionPwQPolynomial.from_pw_qpolynomial( + ingress + ).add(isl.UnionPwQPolynomial.from_pw_qpolynomial(egress)) + pressure: EdgePressure = EdgePressure(acc) + # (D5) `hops` is the aggregation `EdgePressure.total()` already performs + # (sum(ingress) + sum(egress)), wrapped as the constant `TransferInfo.hops` + # expects -- not a second, independent aggregation. + hops: isl.PwQPolynomial = _const_pwq(pressure.total()) + # { dst -> data } fills actually covered by a matched source (D2). + covered: isl.Map = _covered_fills(mcs) return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_), + fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(fills.map_)), - hops=result, + unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), + hops=hops, link_transfer=True, + edge_pressure=pressure, ) - def edge_pressure(self, fills: Fill, occs: Occupancy) -> EdgePressure: - """ - Per-edge memory pressure (spoke load) of the star transfer. - - After ``identify_mesh_casts`` fixes the ``[src] -> [dst]`` pairs, this - reports, per node spoke, how much data crosses it -- egress (data the node - sources into the switch) and ingress (data it receives) as two directed - edges. ``EdgePressure.bottleneck`` is the busiest spoke, the link a - per-spoke bandwidth limit binds on first (e.g. for an N-way all-to-all the - ingress spokes are hottest at ``N - 1``). - - Parameters - ---------- - fills: - The fill of `buffer` across time from parents. - occs: - The occupancy of `buffer` across time. - - Returns - ------- - An ``EdgePressure`` over ``spoke_in[n]`` (ingress) and ``spoke_out[n]`` - (egress) edges. - """ - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - ingress, egress = self._spoke_loads(mcs) - acc = isl.UnionPwQPolynomial.from_pw_qpolynomial(ingress) - acc = acc.add(isl.UnionPwQPolynomial.from_pw_qpolynomial(egress)) - return EdgePressure(acc) - def _spoke_loads( self, mcns: isl.Map ) -> tuple[isl.PwQPolynomial, isl.PwQPolynomial]: @@ -865,6 +1056,14 @@ def _spoke_loads( node receives }`` and ``egress`` is ``{ spoke_out[n] -> #data the node sources }``, counting only fabric-crossing (>= 1 hop) deliveries. """ + # Design (D1): read the node tuple's name off `mcns` instead of assuming + # the literal `'noc'`, following the same precedent as + # `XYRoutingMulticastModel._directed_mesh_links`. Unlike XY, the star + # model has no dimensionality requirement -- `dims` (used below to build + # the `spoke_in`/`spoke_out` relabeling) is already read generically off + # `ingress_nodes`, so only the hardcoded name needed fixing here. + name, _dims = _mesh_node_tuple(mcns) + # Keep only deliveries that actually cross the fabric (>= 1 hop); a node # already holding its datum loads no spoke. crossing_hops: isl.Set = isl.Set.read_from_str( @@ -877,8 +1076,8 @@ def _spoke_loads( cur: isl.Map = crossing.reverse().curry() # Distinct (node, data) pairs per direction. A source injects each datum # once (multicast fans out at the switch); a destination receives each once. - ingress_nodes: isl.Map = cur.range_factor_range() # { noc[dst] -> data } - egress_nodes: isl.Map = cur.range().unwrap() # { noc[src] -> data } + ingress_nodes: isl.Map = cur.range_factor_range() # { name[dst] -> data } + egress_nodes: isl.Map = cur.range().unwrap() # { name[src] -> data } # Relabel the node tuple to the directed spoke edge, then count data per # spoke. { spoke_in[n] -> #data } and { spoke_out[n] -> #data }. @@ -886,38 +1085,12 @@ def _spoke_loads( idx: str = ", ".join(f"i{k}" for k in range(dims)) ingress: isl.PwQPolynomial = ingress_nodes.apply_domain( isl.Map.read_from_str( - isl.DEFAULT_CONTEXT, "{ noc[%s] -> spoke_in[%s] }" % (idx, idx) + isl.DEFAULT_CONTEXT, "{ %s[%s] -> spoke_in[%s] }" % (name, idx, idx) ) ).card() egress: isl.PwQPolynomial = egress_nodes.apply_domain( isl.Map.read_from_str( - isl.DEFAULT_CONTEXT, "{ noc[%s] -> spoke_out[%s] }" % (idx, idx) + isl.DEFAULT_CONTEXT, "{ %s[%s] -> spoke_out[%s] }" % (name, idx, idx) ) ).card() return ingress, egress - - def _cost_star_multicast(self, mcns: isl.Map) -> isl.PwQPolynomial: - """ - Total spoke traversals for the multicast networks `mcns`. - - Each datum costs one egress-spoke hop per injecting source plus one - ingress-spoke hop per receiving destination, so the total is - ``sum(egress) + sum(ingress)``. Returned as a parameter-free constant. - - Parameters - ---------- - mcns: - Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. - """ - ctx = isl.DEFAULT_CONTEXT - ingress, egress = self._spoke_loads(mcns) - total: isl.Val = isl.Val.zero(ctx) - for load in (ingress, egress): - summed: isl.PwQPolynomial = load.sum() - total = total.add( - summed.eval(isl.Point.zero(summed.domain().get_space())) - ) - zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) - return isl.PwQPolynomial.from_qpolynomial( - isl.QPolynomial.val_on_domain(zero_dim, total) - ) diff --git a/accelforge/model/_looptree/reuse/isl/spatial.py b/accelforge/model/_looptree/reuse/isl/spatial.py index 47d6fa42..856197be 100644 --- a/accelforge/model/_looptree/reuse/isl/spatial.py +++ b/accelforge/model/_looptree/reuse/isl/spatial.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Optional +from typing import TYPE_CHECKING, Optional import islpy as isl @@ -22,6 +22,18 @@ TaggedMap, ) +if TYPE_CHECKING: + # Design: `EdgePressure` lives in `distributed/distributed_buffers.py`, which + # itself imports `TransferInfo` from this module (see `spatial.py`'s role as + # the base contract every distributed model builds on). Importing it at + # runtime here would form an import cycle + # (spatial -> distributed_buffers -> spatial); a `TYPE_CHECKING`-guarded + # import plus a forward-reference string annotation on the `edge_pressure` + # field below gets static type-checking for free without paying that cost. + from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + EdgePressure, + ) + class Transfers(TaggedMap): """Transfers between regions in spacetime.""" @@ -37,9 +49,21 @@ class TransferInfo: # Crucial information to transfer info. fulfilled_fill: Transfers - """Fills done by peer-to-peer transfers.""" + """Fills done by peer-to-peer transfers. + + Restricted to the fills actually *covered* by a matched multicast source + (see `identify_mesh_casts`) -- a destination requesting a datum no source + holds is never "fulfilled" even though it is dropped from the multicast + network, hence the separate `unfulfilled_fill` below. + """ unfulfilled_fill: Fill - """Fills not performed.""" + """Fills not performed. + + Fills whose datum has no matching source in `identify_mesh_casts`'s + result -- i.e. `fills - covered`, the complement of `fulfilled_fill` + within the original fill set (`fulfilled_fill` and `unfulfilled_fill` + partition `fills` exactly; see each model's `apply` for the derivation). + """ parent_reads: Reads """Fills done by parent-to-child transfers.""" hops: isl.PwQPolynomial @@ -48,6 +72,23 @@ class TransferInfo: # Metadata on what is occurring. link_transfer: bool + # Optional, model-specific per-link decomposition -- last field / defaulted + # so this addition does not disturb any existing positional or keyword + # construction site (`grep`-verified: every `TransferInfo(...)` call in the + # tree already uses keyword arguments). + edge_pressure: Optional["EdgePressure"] = None + """Per-directed-edge load backing `hops`, for models that define one. + + Populated by models with an explicit per-link topology (mesh links for + `XYRoutingMulticastModel`, spokes for `StarMulticastModel`) from the same + `identify_mesh_casts` result used to compute `hops`, so the two are always + consistent. `None` for models with no per-link decomposition defined -- + `HypercubeMulticastModel` costs a convex bounding box (no notion of + individual links) and `FullyConnectedMulticastModel` treats the fabric as + a contention-free full mesh (one dedicated link per pair, so there is no + shared-link pressure to report); both leave this field at its default. + """ + class TransferModel(ABC): """ diff --git a/tests/not_working/distribuffers/helpers.py b/tests/not_working/distribuffers/helpers.py new file mode 100644 index 00000000..a190c9dd --- /dev/null +++ b/tests/not_working/distribuffers/helpers.py @@ -0,0 +1,158 @@ +""" +Shared test helpers for the ISL distributed-buffer multicast model test suite. + +Factors out two pieces of logic that used to be byte-for-byte duplicated across +``test_multicast.py``, ``test_fully_connected.py``, and ``test_xy_routing.py``: + +1. ``construct_spacetime`` -- turning a yaml ``dims`` list into ``Tag`` objects. +2. ``run_hops_gamut`` -- the "load a yaml of test cases, build a ``Fill`` + /``Occupancy``/``dist_fn`` triple, run one ``TransferModel``, and compare + ``.hops`` against an expected key" loop, which is identical across the + three hop-oracle test files and differs only in which model class, which + yaml directory, and which ``expected`` key (``hypercube_hops`` / + ``fully_connected_hops`` / ``xy_routing_hops``) is used. + +With this module in place, ``test_multicast.py``, ``test_fully_connected.py``, +and ``test_xy_routing.py`` reduce to thin parametrizations: a ``TestCase`` whose +single ``test_gamut`` method calls ``run_hops_gamut`` with its model class, yaml +path, and expected-key string. + +Import note +----------- +This module (and every test module in this package) imports the canonical +``load_solutions`` helper via the *absolute* path ``tests.isl.util`` rather than +a local copy. That import only resolves if the repository root is on +``sys.path``, which is the case when the suite is invoked from the repo root as + + PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest tests/not_working/distribuffers/ -q + +(pytest inserts the current working directory / rootdir onto ``sys.path``). +Running this module or its dependents from a different working directory, or +via a bare module path without the repo root on ``sys.path``, will raise +``ModuleNotFoundError: No module named 'tests'``. +""" + +from pathlib import Path + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + # Design (D6): the "evaluate a parameter-free PwQPolynomial at the zero + # point" idiom lives in exactly one place now -- see `_eval_const`'s + # docstring for the history of it drifting across four call sites with + # inconsistent return types. This module used to inline its own copy + # (`info.hops.eval(isl.Point.zero(...))`, which returns an `isl.Val` + # despite looking `int`-shaped); importing the canonical helper instead + # keeps this the single non-test call site left outside + # `distributed_buffers.py` and guarantees a real Python `int`. + _eval_const, +) +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( + # Data movement descriptors. + Fill, + Occupancy, + # Tags + Tag, + SpatialTag, + TemporalTag, +) +from accelforge.model._looptree.reuse.isl.spatial import TransferInfo, TransferModel + +# Design: `tests/not_working/distribuffers/util.py` used to be a byte-identical +# copy of `tests/isl/util.py`. Rather than keep two copies of `load_solutions` +# in sync by hand, this package now imports the canonical one directly -- see +# the "Import note" above for the run-from-repo-root requirement this implies. +from tests.isl.util import load_solutions + + +def construct_spacetime(dims: list[dict]) -> list[Tag]: + """ + Convert a yaml ``dims`` list (each entry a dict with a ``type`` key, plus + ``spatial_dim``/``target`` for spatial entries) into the corresponding + ``Tag`` objects, in order. + + Parameters + ---------- + dims: + The list of dim-tag dicts as loaded from a test-case yaml, e.g. + ``[{"type": "Spatial", "spatial_dim": 0, "target": 0}, ...]``. + + Returns + ------- + ``list[Tag]`` where ``list[i]`` is the tag corresponding to ``dims[i]``. + """ + spacetime: list[Tag] = [] + for dim in dims: + if dim["type"] == "Temporal": + spacetime.append(TemporalTag()) + elif dim["type"] == "Spatial": + spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) + + return spacetime + + +def run_hops_gamut(model_cls: type[TransferModel], yaml_path: Path, expected_key: str) -> None: + """ + Run every test case in a yaml gamut file through ``model_cls`` and assert + the resulting ``TransferInfo.hops`` matches the case's expected value. + + # Design: `test_multicast.py`, `test_fully_connected.py`, and + # `test_xy_routing.py` each had their own copy of this loop; the only + # differences were the model class being exercised, the yaml directory the + # cases were loaded from, and which key of `test["expected"]` held the + # oracle value (`hypercube_hops` / `fully_connected_hops` / + # `xy_routing_hops`). Parametrizing on those three makes the loop itself + # single-sourced. + + Parameters + ---------- + model_cls: + A ``TransferModel`` subclass constructible as ``model_cls(dist_fn)`` + (every model in ``distributed_buffers.py`` follows this signature). + yaml_path: + Path to a yaml file of test cases, each a dict with ``dims``, ``fill``, + ``occ``, ``dist_fn``, and ``expected`` keys (see any + ``*/test_cases.yaml`` under this package for the schema). + expected_key: + The key within each case's ``expected`` dict holding the oracle hop + count for this model (e.g. ``"xy_routing_hops"``). A ``None`` value at + this key marks a case as still in progress: rather than failing, the + case's inputs/output are printed for manual inspection (preserving the + original tests' "unimplemented case" debugging affordance). + + Raises + ------ + AssertionError + If a case's expected value is not ``None`` and the model's computed + ``hops`` does not match it. + """ + testcases: dict = load_solutions(yaml_path) + for test in testcases: + # Reads test case parameters and constructs the necessary objects. + dim_tags: list[Tag] = construct_spacetime(test["dims"]) + fill: Fill = Fill(dim_tags, test["fill"]) + occ: Occupancy = Occupancy(dim_tags, test["occ"]) + dist_fn: isl.Map = test["dist_fn"] + model: TransferModel = model_cls(dist_fn) + + # Applies the model. + info: TransferInfo = model.apply(0, fill, occ) + # Checks the results. `hops` is parameter-free (the validated regime + # every distributed model targets, see `_eval_const`'s precondition), + # so this is a scalar extraction, not a real evaluation-at-a-point. + sum_extract: int = _eval_const(info.hops) + + expected = test["expected"][expected_key] + # The block is used for debugging test cases not yet implemented. + if expected is None: + print("~~~Test case in progress:~~~") + print(f"Fill: {fill}") + print(f"Occ: {occ}") + print(f"Dist Fn: {dist_fn}") + print(f"Returned: {sum_extract}") + else: + assert sum_extract == expected, ( + f"{model_cls.__name__} hops mismatch: got {sum_extract}, " + f"expected {expected} ({expected_key})\n" + f"Fill: {fill}\nOcc: {occ}\nDist Fn: {dist_fn}" + ) diff --git a/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml b/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml index 5e1c21c4..36dcf1b8 100755 --- a/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml +++ b/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml @@ -1,97 +1,91 @@ - binding: + version: 0.4 nodes: - logical: name: ProcessingElement - l_dims: [i] + dims: [i] physical: name: ProcessingElement - p_dims: [x, y] + dims: [x, y] relations: tensorA: i = x + y * 2 # This is a dimension-major compression into the logical. It is bijective. tensorB: i = x + y * 2 # This is a dimension-major compression into the logical. It is bijective. - logical: name: Scratchpad - l_dims: [x, y] + dims: [x, y] physical: name: GLB - p_dims: [a, b] + dims: [a, b] relations: tensorA: x = a and y = b tensorB: x = b and y = a solution: + version: 0.4 nodes: - tensorA: | { - [ - tensorA_ranks[c, h, w, p, q, r, s] -> - l_ProcessingElement_dims[i] - ] -> - p_ProcessingElement_dims[x, y] : + l_ProcessingElement_dims[i] -> p_ProcessingElement_dims[x, y] : i = x + 2y } tensorB: | { - [ - tensorB_ranks[c, h, w, p, q, r, s] -> - l_ProcessingElement_dims[i] - ] -> - p_ProcessingElement_dims[x, y] : + l_ProcessingElement_dims[i] -> p_ProcessingElement_dims[x, y] : i = x + (y * 2) } - tensorA: | { - [ - tensorA_ranks[c, h, w, p, q, r, s] -> - l_Scratchpad_dims[x, y] - ] -> - p_GLB_dims[ a, b ] : + l_Scratchpad_dims[x, y] -> p_GLB_dims[ a, b ] : x = a and y = b } tensorB: | { - [ - tensorB_ranks[c, h, w, p, q, r, s] -> - l_Scratchpad_dims[x, y] - ] -> - p_GLB_dims[ a, b ] : + l_Scratchpad_dims[x, y] -> p_GLB_dims[ a, b ] : x = b and y = a } -- binding: - nodes: - - logical: - name: DRAM - l_dims: [i] - physical: - name: DRAM - p_dims: [i] - relation: # Compression relation where less DRAM chips than planned. - weights: i = i // 2 - inputs: i = i // 2 - outputs: i = i // 2 - - logical: - name: Scratchpad - l_dims: [i] - physical: - name: GLB - p_dims: [x, y, z] - relation: # weight stationary relation - weights: c = x and h = y and w = z - inputs: i=x and i=y and i=z - outputs: i=x and i=y and i=z - - logical: - name: PE_Buffer - l_dims: [i] - physical: - name: GLB - l_dims: [x, y, z] - relation: # Some weird bypass shenanigans. - weights: - inputs: - outputs: - - - -- +# NOTE: the block below predates the `Domain.dims` / `Binding.version` / +# `BindingNode.relations` schema in `accelforge/frontend/_binding.py` (it used +# `l_dims`/`p_dims`/`relation`, and node 3 even names its physical field +# `l_dims` instead of `p_dims`) and has no `solution` counterpart to check +# against, so `test_valid_bindings` never exercised it as a second spec entry. +# Commented out rather than "fixed" per the same reasoning already applied in +# `tests/isl/distributed/spec/binding/valid_bindings.yaml` (the canonical, +# passing twin of this fixture): deriving the right ISL solution strings for +# the floor-division compression relations (`i = i // 2`) and the bypass node +# (empty relations) is a separate piece of work from the D1-D6 network-model +# fixes this file accompanies, not a minimal, mechanical rename. +# - binding: +# version: 0.4 +# nodes: +# - logical: +# name: DRAM +# l_dims: [i] +# physical: +# name: DRAM +# p_dims: [i] +# relations: # Compression relation where less DRAM chips than planned. +# weights: i = i // 2 +# inputs: i = i // 2 +# outputs: i = i // 2 +# - logical: +# name: Scratchpad +# l_dims: [i] +# physical: +# name: GLB +# p_dims: [x, y, z] +# relations: # weight stationary relation +# weights: c = x and h = y and w = z +# inputs: i=x and i=y and i=z +# outputs: i=x and i=y and i=z +# - logical: +# name: PE_Buffer +# l_dims: [i] +# physical: +# name: GLB +# l_dims: [x, y, z] +# relations: # Some weird bypass shenanigans. +# weights: +# inputs: +# outputs: diff --git a/tests/not_working/distribuffers/test_contract.py b/tests/not_working/distribuffers/test_contract.py new file mode 100644 index 00000000..1bc30faa --- /dev/null +++ b/tests/not_working/distribuffers/test_contract.py @@ -0,0 +1,367 @@ +""" +Regression tests for the D1/D2/D3 contract fixes to +``distributed_buffers.py``'s multicast models (see the module's design notes +and the module-level ``_mesh_node_tuple``/``_covered_fills``/ +``identify_mesh_casts`` docstrings for the full rationale). Each class below +pins down one previously-broken-or-undocumented contract so a future change +cannot silently regress it: + +- ``TestTupleNameGenericity`` (D1): a spacetime/node tuple named anything + other than the literal ``'noc'`` (e.g. ``pe[x, y]``) used to crash deep + inside an opaque ISL assertion in ``XYRoutingMulticastModel``'s and + ``StarMulticastModel``'s helpers, which hardcoded ``'noc'`` into every map + string they built. Both now read the tuple name off the caller's own maps. +- ``TestXYDimensionalityGuard`` (D1): a non-2-D node tuple through + ``XYRoutingMulticastModel`` now raises a clear ``ValueError`` instead of the + same kind of opaque ISL assertion. +- ``TestUnfulfilledPartition`` (D2): ``fulfilled_fill``/``unfulfilled_fill`` + are a true partition of ``fills`` by whether ``identify_mesh_casts`` found a + matched source, for all four models -- previously ``unfulfilled_fill`` was + unconditionally empty regardless of whether a fill actually had a source. +- ``TestAsymmetricDistFnOrientation`` (D3): ``identify_mesh_casts``'s + ``dist_fn`` is documented (after the D3 fix) to be applied in the + ``{ [dst -> src] -> [hops] }`` orientation; this pins down that an + asymmetric ``dist_fn`` (direction-dependent cost) actually selects the + source that is *cheapest to reach from the destination*, not some other + pairing that a swapped orientation would silently produce. + +See accelforge/model/_looptree/reuse/isl/distributed/README.md for the models' +background, and accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +for the D1/D2/D3 design comments this file exercises. +""" + +import unittest + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( + Fill, + Occupancy, + SpatialTag, +) +from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( + HypercubeMulticastModel, + FullyConnectedMulticastModel, + XYRoutingMulticastModel, + StarMulticastModel, + _eval_const, +) + +_CTX = isl.DEFAULT_CONTEXT + + +def _manhattan_2d(name: str) -> isl.Map: + """ + Build a 2-D Manhattan ``{ [dst -> src] -> [hops] }`` distance map over a + node tuple named ``name`` (e.g. ``"noc"`` or ``"pe"``), identical in shape + to the ``2d_manhattan`` fixture in ``xy_routing/test_cases.yaml`` but with + the tuple name parametrized so it can be reused for the tuple-name + genericity regression tests (D1) below. + """ + return isl.Map.read_from_str( + _CTX, + ( + "{{ [{n}[xd, yd] -> {n}[xs, ys]] -> hops[(xd - xs) + (yd - ys)] :" + " xd >= xs and yd >= ys;" + " [{n}[xd, yd] -> {n}[xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] :" + " xd < xs and yd < ys;" + " [{n}[xd, yd] -> {n}[xs, ys]] -> hops[-(xd - xs) + (yd - ys)] :" + " xd < xs and yd >= ys;" + " [{n}[xd, yd] -> {n}[xs, ys]] -> hops[(xd - xs) + -(yd - ys)] :" + " xd >= xs and yd < ys }}" + ).format(n=name), + ) + + +def _manhattan_1d(name: str) -> isl.Map: + """ + Build a 1-D Manhattan ``{ [dst -> src] -> [hops] }`` distance map over a + node tuple named ``name``, for the line-of-nodes fixtures below. + """ + return isl.Map.read_from_str( + _CTX, + ( + "{{ [{n}[nd] -> {n}[ns]] -> hops[nd - ns] : nd >= ns;" + " [{n}[nd] -> {n}[ns]] -> hops[ns - nd] : nd < ns }}" + ).format(n=name), + ) + + +def _all_to_all(name: str, n: int) -> tuple[Fill, Occupancy, isl.Map]: + """ + Build ``(fill, occ, dist_fn)`` for an ``n``-node all-to-all on a 1-D line + of nodes tagged under the tuple name ``name``. + + Node ``g`` starts holding only ``data[g]`` and requests every other node's + datum, exactly the ``_all_to_all`` fixture in ``test_edge_pressure.py`` + but with the tuple name parametrized for the D1 genericity checks. + """ + tags = [SpatialTag(0, 0)] + occ = Occupancy( + tags, + isl.Map.read_from_str( + _CTX, f"{{ {name}[g] -> data[d] : 0 <= g < {n} and d = g }}" + ), + ) + fill = Fill( + tags, + isl.Map.read_from_str( + _CTX, + f"{{ {name}[g] -> data[d] : 0 <= g < {n} and 0 <= d < {n} and d != g }}", + ), + ) + return fill, occ, _manhattan_1d(name) + + +class TestTupleNameGenericity(unittest.TestCase): + """ + D1: a node tuple named anything other than ``'noc'`` must produce the same + numbers as the ``'noc'``-named equivalent, for every model whose helpers + used to hardcode the literal ``'noc'`` into ISL map strings. + """ + + def test_xy_non_noc_tuple_name_matches_noc(self): + """ + XY routing case B (source (1,0) -> (0,2),(2,2), 6 hops -- see + ``xy_routing/test_cases.yaml``) run under both ``noc[x, y]`` and + ``pe[x, y]`` must produce the identical hop count. Before D1, `'pe'` + crashed inside ``_directed_mesh_links``'s hardcoded ``'noc[...]'`` map + strings with a raw ISL tuple-name-mismatch assertion. + """ + tags = [SpatialTag(0, 0), SpatialTag(1, 0)] + for name in ("noc", "pe"): + with self.subTest(name=name): + occ = Occupancy( + tags, + isl.Map.read_from_str( + _CTX, f"{{ {name}[x, y] -> data[c] : c = 0 and x = 1 and y = 0 }}" + ), + ) + fill = Fill( + tags, + isl.Map.read_from_str( + _CTX, + f"{{ {name}[x, y] -> data[c] : c = 0 and" + f" ((x = 0 and y = 2) or (x = 2 and y = 2)) }}", + ), + ) + model = XYRoutingMulticastModel(_manhattan_2d(name)) + hops = _eval_const(model.apply(0, fill, occ).hops) + self.assertEqual(hops, 6) + + def test_star_and_fully_connected_non_noc_tuple_name_match_noc(self): + """ + The 8-node all-to-all oracle (FullyConnected 56, Star 64 -- see + ``TestStarSpokePressure`` in ``test_edge_pressure.py``) run under both + ``noc[g]`` and ``pe[g]`` must produce identical numbers. Before D1, + ``StarMulticastModel._spoke_loads`` hardcoded ``'noc'`` the same way + ``_directed_mesh_links`` did (``FullyConnectedMulticastModel`` never + hardcoded a tuple name, so it is included here only as a same-input + cross-check, not because it was previously broken). + """ + for name in ("noc", "pe"): + with self.subTest(name=name): + fill, occ, dist_fn = _all_to_all(name, 8) + fc_hops = _eval_const( + FullyConnectedMulticastModel(dist_fn).apply(0, fill, occ).hops + ) + star_hops = _eval_const( + StarMulticastModel(dist_fn).apply(0, fill, occ).hops + ) + self.assertEqual(fc_hops, 56) + self.assertEqual(star_hops, 64) + + +class TestXYDimensionalityGuard(unittest.TestCase): + """ + D1: ``XYRoutingMulticastModel`` only supports exactly 2-D node tuples, and + must say so with a clear ``ValueError`` rather than an opaque ISL abort. + """ + + def test_three_dimensional_node_tuple_raises_value_error(self): + """ + A 3-D node tuple (``noc[x, y, z]``) must raise ``ValueError`` from + ``apply`` (via ``_directed_mesh_links``), naming the offending tuple + and its dimensionality, instead of failing deep inside an + ``isl.Map.read_from_str``/``apply_range`` call the way it did before + D1's ``_mesh_node_tuple`` dimensionality check was added. The + `dist_fn` here is a trivial constant (correctness of routing over a + 3-D mesh is out of scope -- this test only exercises the guard). + """ + tags = [SpatialTag(0, 0), SpatialTag(1, 0), SpatialTag(2, 0)] + occ = Occupancy( + tags, + isl.Map.read_from_str( + _CTX, "{ noc[x, y, z] -> data[c] : c = 0 and x = 0 and y = 0 and z = 0 }" + ), + ) + fill = Fill( + tags, + isl.Map.read_from_str( + _CTX, "{ noc[x, y, z] -> data[c] : c = 0 and x = 1 and y = 0 and z = 0 }" + ), + ) + dist_fn = isl.Map.read_from_str( + _CTX, "{ [noc[xd, yd, zd] -> noc[xs, ys, zs]] -> hops[0] }" + ) + model = XYRoutingMulticastModel(dist_fn) + with self.assertRaisesRegex( + ValueError, r"2-D node tuple.*'noc'.*3 dimensions" + ): + model.apply(0, fill, occ) + + +class TestUnfulfilledPartition(unittest.TestCase): + """ + D2: ``fulfilled_fill``/``unfulfilled_fill`` must be a true partition of + ``fills`` by whether ``identify_mesh_casts`` matched a source, for every + model. Shared geometry: node 0 holds only ``data[0]``; node 1 requests + both ``data[0]`` (has a source -- fulfilled) and ``data[1]`` (no source + anywhere -- unfulfilled). Before D2, every model reported + ``unfulfilled_fill`` as unconditionally empty (``fills - fills``) and + ``fulfilled_fill`` as the entire (uncovered) fill map. + """ + + @staticmethod + def _geometry(name: str, two_d: bool): + """ + Build ``(tags, occ, fill, dist_fn, expected_fulfilled, + expected_unfulfilled)`` for the shared node0/node1 geometry, in either + a 1-D node tuple (for models with no dimensionality requirement) or a + 2-D one (required by ``XYRoutingMulticastModel``). + """ + if two_d: + tags = [SpatialTag(0, 0), SpatialTag(1, 0)] + occ = Occupancy( + tags, + isl.Map.read_from_str( + _CTX, f"{{ {name}[x, y] -> data[c] : x = 0 and y = 0 and c = 0 }}" + ), + ) + fill = Fill( + tags, + isl.Map.read_from_str( + _CTX, + f"{{ {name}[x, y] -> data[c] : x = 1 and y = 0 and" + f" (c = 0 or c = 1) }}", + ), + ) + dist_fn = _manhattan_2d(name) + expected_fulfilled = isl.Map.read_from_str( + _CTX, f"{{ {name}[x, y] -> data[c] : x = 1 and y = 0 and c = 0 }}" + ) + expected_unfulfilled = isl.Map.read_from_str( + _CTX, f"{{ {name}[x, y] -> data[c] : x = 1 and y = 0 and c = 1 }}" + ) + else: + tags = [SpatialTag(0, 0)] + occ = Occupancy( + tags, isl.Map.read_from_str(_CTX, f"{{ {name}[n] -> data[c] : n = 0 and c = 0 }}") + ) + fill = Fill( + tags, + isl.Map.read_from_str( + _CTX, f"{{ {name}[n] -> data[c] : n = 1 and (c = 0 or c = 1) }}" + ), + ) + dist_fn = _manhattan_1d(name) + expected_fulfilled = isl.Map.read_from_str( + _CTX, f"{{ {name}[n] -> data[c] : n = 1 and c = 0 }}" + ) + expected_unfulfilled = isl.Map.read_from_str( + _CTX, f"{{ {name}[n] -> data[c] : n = 1 and c = 1 }}" + ) + return tags, occ, fill, dist_fn, expected_fulfilled, expected_unfulfilled + + def test_partition_across_models(self): + """ + For each model, ``unfulfilled_fill.map_`` must be exactly + ``{ dst -> data[1] }`` (node 1's unmatched request), and + ``fulfilled_fill.map_`` must be exactly ``{ dst -> data[0] }`` (the + covered rest). ``hops`` is checked too, as a sanity cross-check that + only the matched (data[0]) delivery is costed -- 1 for the models that + count a single crossing, 2 for ``StarMulticastModel`` (its `hops` is + injection + delivery, i.e. egress + ingress, for the one crossing + delivery). + """ + # (model class, needs-2-D node tuple, expected `hops` for the one + # matched delivery). + cases = [ + (HypercubeMulticastModel, False, 1), + (FullyConnectedMulticastModel, False, 1), + (StarMulticastModel, False, 2), + (XYRoutingMulticastModel, True, 1), + ] + for model_cls, two_d, expected_hops in cases: + with self.subTest(model=model_cls.__name__): + _, occ, fill, dist_fn, expected_fulfilled, expected_unfulfilled = ( + self._geometry("noc", two_d) + ) + info = model_cls(dist_fn).apply(0, fill, occ) + self.assertTrue( + info.unfulfilled_fill.map_.is_equal(expected_unfulfilled), + f"unfulfilled_fill was {info.unfulfilled_fill.map_}, " + f"expected {expected_unfulfilled}", + ) + self.assertTrue( + info.fulfilled_fill.map_.is_equal(expected_fulfilled), + f"fulfilled_fill was {info.fulfilled_fill.map_}, " + f"expected {expected_fulfilled}", + ) + self.assertEqual(_eval_const(info.hops), expected_hops) + + +class TestAsymmetricDistFnOrientation(unittest.TestCase): + """ + D3: ``identify_mesh_casts`` applies ``dist_fn`` in the + ``{ [dst -> src] -> [hops] }`` orientation (the docstring previously said + the opposite, though the code itself was never wrong -- D3 was a + documentation-only fix). This pins down the now-correctly-documented + behavior with a ``dist_fn`` that is *directionally* asymmetric (cost + depends on which side is source vs. destination, not just on distance), + so a caller who mis-orients their own ``dist_fn`` -- or a future change + that swaps `identify_mesh_casts`'s composition order -- would select the + wrong source and fail this test. + """ + + def test_nearest_source_selected_by_directional_cost_not_distance(self): + """ + Three nodes on a line: 0, 1, 2. Nodes 0 and 2 both hold ``data[0]``; + node 1 requests it. Both candidate sources are Euclidean-distance 1 + away, so a distance-based tiebreak could not distinguish them -- the + ``dist_fn`` instead makes the *direction* the deciding factor: a + source below the destination (``src < dst``) costs 1, a source above + (``src > dst``) costs 5. The nearest (min-cost) source is therefore + node 0, not node 2. `identify_mesh_casts`'s `parent_reads` records + exactly which source served the delivery, so this is checked directly + against the expected ``{ data[0] -> [noc[1] -> noc[0]] }`` mapping, + not just inferred from a hop count. + """ + tags = [SpatialTag(0, 0)] + occ = Occupancy( + tags, isl.Map.read_from_str(_CTX, "{ noc[n] -> data[c] : c = 0 and (n = 0 or n = 2) }") + ) + fill = Fill(tags, isl.Map.read_from_str(_CTX, "{ noc[n] -> data[c] : c = 0 and n = 1 }")) + # `{ [dst -> src] -> hops }`: cheap (1) if src is below dst, expensive + # (5) if src is above -- direction, not magnitude, decides. + dist_fn = isl.Map.read_from_str( + _CTX, + "{ [noc[nd] -> noc[ns]] -> hops[0] : nd = ns;" + " [noc[nd] -> noc[ns]] -> hops[1] : ns < nd;" + " [noc[nd] -> noc[ns]] -> hops[5] : ns > nd }", + ) + model = FullyConnectedMulticastModel(dist_fn) + info = model.apply(0, fill, occ) + + expected_match = isl.Map.read_from_str(_CTX, "{ data[0] -> [noc[1] -> noc[0]] }") + self.assertTrue( + info.parent_reads.map_.is_equal(expected_match), + f"parent_reads was {info.parent_reads.map_}, expected {expected_match} " + "(source node 0, the directionally-cheap side)", + ) + # A single crossing delivery, regardless of the asymmetric magnitude. + self.assertEqual(_eval_const(info.hops), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/not_working/distribuffers/test_edge_pressure.py b/tests/not_working/distribuffers/test_edge_pressure.py index 0f272bc9..6df5e1ad 100644 --- a/tests/not_working/distribuffers/test_edge_pressure.py +++ b/tests/not_working/distribuffers/test_edge_pressure.py @@ -5,17 +5,19 @@ per *physical* directed edge: how many multicast trees cross each link. Two models are covered: -- ``XYRoutingMulticastModel.edge_pressure`` -- directed mesh links - (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). The decisive, oracle-free - check is the invariant ``sum over edges of load == total XY hops``: the per-edge - loads must sum back to the already-validated A-F hop totals (4/6/6/3/4/448), so - a wrong decomposition fails here. Bottlenecks and a couple of individual edge - loads (hand-derived, geometry documented inline) pin the shape. - -- ``StarMulticastModel.edge_pressure`` -- the spokes realization of a - fully-connected fabric (``spoke_in[n]`` ingress, ``spoke_out[n]`` egress). For - an N-way all-to-all each node receives N-1 and sources 1, so the ingress spokes - are hottest at N-1. Tied to ``FullyConnectedMulticastModel`` by +- ``XYRoutingMulticastModel``'s ``edge_pressure`` (now reached via + ``model.apply(0, fill, occ).edge_pressure``, not a standalone method -- see + the API-change note on ``test_load_sums_to_hops`` below) -- directed mesh + links (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). ``test_load_sums_to_hops`` + checks the per-edge loads sum back to the already-validated A-F hop totals + (4/6/6/3/4/448). Bottlenecks and a couple of individual edge loads + (hand-derived, geometry documented inline) pin the shape. + +- ``StarMulticastModel``'s ``edge_pressure`` (same access-pattern change) -- + the spokes realization of a fully-connected fabric (``spoke_in[n]`` ingress, + ``spoke_out[n]`` egress). For an N-way all-to-all each node receives N-1 and + sources 1, so the ingress spokes are hottest at N-1. Tied to + ``FullyConnectedMulticastModel`` by ``sum over nodes of ingress == FullyConnected crossing count``. See accelforge/model/_looptree/reuse/isl/distributed/README.md. @@ -29,27 +31,24 @@ from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( Fill, Occupancy, - Tag, SpatialTag, - TemporalTag, ) from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( XYRoutingMulticastModel, StarMulticastModel, FullyConnectedMulticastModel, + # Design: this used to be a local copy (identical signature/behavior) kept + # in sync by hand; now that the source module exposes one module-level, + # `int`-returning `_eval_const` (D6), the test suite imports it directly + # instead of re-deriving the "evaluate a parameter-free PwQPolynomial at + # the space's zero point" idiom a second time. + _eval_const, ) -from .util import load_solutions - -def construct_spacetime(dims: list) -> list[Tag]: - """Convert a list of dim-tag dicts (from yaml) into ``Tag`` objects.""" - spacetime: list[Tag] = [] - for dim in dims: - if dim["type"] == "Temporal": - spacetime.append(TemporalTag()) - elif dim["type"] == "Spatial": - spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) - return spacetime +# `construct_spacetime` and `load_solutions` are hoisted into `helpers.py` +# (single-sourced there, see that module's docstring for the run-from-repo-root +# import requirement) rather than redefined/re-imported locally here. +from .helpers import construct_spacetime, load_solutions # 1-D Manhattan distance over a line of GPUs, for the star/all-to-all cases. @@ -59,11 +58,6 @@ def construct_spacetime(dims: list) -> list[Tag]: ) -def _eval_const(pwq: isl.PwQPolynomial) -> int: - """Evaluate a parameter-free piecewise quasi-polynomial to an int.""" - return int(str(pwq.eval(isl.Point.zero(pwq.domain().get_space())))) - - class TestXYRoutingEdgePressure(unittest.TestCase): """Per-mesh-edge pressure for XY routing, reusing the XY hop test geometries.""" @@ -72,8 +66,25 @@ class TestXYRoutingEdgePressure(unittest.TestCase): def test_load_sums_to_hops(self): """ - Invariant: summed per-edge load == total XY hops (the cross-check that the - edge decomposition is exact). Validated against the trusted A-F totals. + Summed per-edge load matches the trusted A-F hop totals. + + # NOTE (post-D5 API/semantics change): before D5, `XYRoutingMulticastModel` + # computed `hops` via a standalone `_cost_xy` and `edge_pressure` via a + # *second*, independent traversal of `mcns`; this test's "sum(load) == + # hops" check was therefore a genuine cross-implementation invariant -- + # a bug in either path's edge decomposition would surface here even if + # each path's own totals looked locally correct. D5 replaced that with + # one aggregation path: `apply()` now computes `hops = _const_pwq( + # pressure.total())` *from* the same `EdgePressure`, so `pressure.total() + # == info.hops` is true by construction, not by cross-checked + # computation -- the invariant, as originally framed, is now + # tautological. What is *not* tautological is comparing `pressure.total()` + # against the yaml's hand-derived `xy_routing_hops` oracle (an + # independent, human-computed number, unrelated to how the code is + # wired) -- that is what this test still checks, so it keeps its + # regression value as a per-case oracle check even though it can no + # longer catch a hops/edge_pressure disagreement (there structurally + # isn't one to catch anymore). """ for test in self.testcases: expected = test["expected"]["xy_routing_hops"] @@ -83,7 +94,7 @@ def test_load_sums_to_hops(self): fill = Fill(dim_tags, test["fill"]) occ = Occupancy(dim_tags, test["occ"]) model = XYRoutingMulticastModel(test["dist_fn"]) - pressure = model.edge_pressure(fill, occ) + pressure = model.apply(0, fill, occ).edge_pressure assert pressure.total() == expected, ( f"Σ edge load {pressure.total()} != hops {expected}" ) @@ -99,8 +110,9 @@ def test_case_F_bottleneck_and_edges(self): f = next(t for t in self.testcases if t["expected"]["xy_routing_hops"] == 448) dim_tags = construct_spacetime(f["dims"]) model = XYRoutingMulticastModel(f["dist_fn"]) - pressure = model.edge_pressure(Fill(dim_tags, f["fill"]), - Occupancy(dim_tags, f["occ"])) + pressure = model.apply( + 0, Fill(dim_tags, f["fill"]), Occupancy(dim_tags, f["occ"]) + ).edge_pressure assert pressure.bottleneck() == 7 assert pressure.eval_edge("yedge_u", [0, 6]) == 7 # top link, all 7 below assert pressure.eval_edge("yedge_d", [0, 1]) == 6 # links below row 1 @@ -125,7 +137,7 @@ def test_monotone_overlap_bottleneck(self): " or (c=2 and x=0 and y=0) }"))) fill = Fill(tags, isl.Map.read_from_str( isl.DEFAULT_CONTEXT, "{ noc[x,y]->data[c] : x=0 and y=3 and 0<=c<3 }")) - pressure = XYRoutingMulticastModel(manhattan).edge_pressure(fill, occ) + pressure = XYRoutingMulticastModel(manhattan).apply(0, fill, occ).edge_pressure assert pressure.eval_edge("yedge_u", [0, 2]) == 3 # top link, all three assert pressure.bottleneck() == 3 # NOT 1 or 2 @@ -140,8 +152,9 @@ def test_single_tree_cases_have_unit_bottleneck(self): continue dim_tags = construct_spacetime(test["dims"]) model = XYRoutingMulticastModel(test["dist_fn"]) - pressure = model.edge_pressure(Fill(dim_tags, test["fill"]), - Occupancy(dim_tags, test["occ"])) + pressure = model.apply( + 0, Fill(dim_tags, test["fill"]), Occupancy(dim_tags, test["occ"]) + ).edge_pressure assert pressure.bottleneck() == 1 @@ -168,7 +181,7 @@ def test_all_to_all_spoke_loads(self): """ for n in (4, 8): fill, occ, dist_fn = self._all_to_all(n) - pressure = StarMulticastModel(dist_fn).edge_pressure(fill, occ) + pressure = StarMulticastModel(dist_fn).apply(0, fill, occ).edge_pressure for node in range(n): assert pressure.eval_edge("spoke_in", [node]) == n - 1 assert pressure.eval_edge("spoke_out", [node]) == 1 @@ -181,7 +194,7 @@ def test_ingress_sum_equals_fully_connected_count(self): """ for n in (4, 8): fill, occ, dist_fn = self._all_to_all(n) - pressure = StarMulticastModel(dist_fn).edge_pressure(fill, occ) + pressure = StarMulticastModel(dist_fn).apply(0, fill, occ).edge_pressure total_ingress = sum(pressure.eval_edge("spoke_in", [g]) for g in range(n)) fc_hops = _eval_const( FullyConnectedMulticastModel(dist_fn).apply(0, fill, occ).hops diff --git a/tests/not_working/distribuffers/test_fully_connected.py b/tests/not_working/distribuffers/test_fully_connected.py index e2a895db..3cc0f9af 100644 --- a/tests/not_working/distribuffers/test_fully_connected.py +++ b/tests/not_working/distribuffers/test_fully_connected.py @@ -10,46 +10,10 @@ import unittest from pathlib import Path -import islpy as isl - -from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( - # Data movement descriptors. - Fill, - Occupancy, - # Tags - Tag, - SpatialTag, - TemporalTag, -) from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( FullyConnectedMulticastModel, ) -from accelforge.model._looptree.reuse.isl.spatial import TransferInfo -from .util import load_solutions - - -def construct_spacetime(dims: list) -> list[Tag]: - """ - Given a list of dimension tags as strings, convert them into the proper `Tag` - objects. - - Parameters - ---------- - dims: - The list of dim tags as strings. - - Returns - ------- - list[Tag] where list[i] is the tag corresponding to dims[i]. - """ - spacetime: list[Tag] = [] - for dim in dims: - if dim["type"] == "Temporal": - spacetime.append(TemporalTag()) - elif dim["type"] == "Spatial": - spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) - - return spacetime +from .helpers import run_hops_gamut class TestFullyConnectedMulticastModel(unittest.TestCase): @@ -57,34 +21,14 @@ class TestFullyConnectedMulticastModel(unittest.TestCase): Tests the FullyConnectedMulticastModel with a series of premade test cases. """ - TEST_CASES_FILE: str = Path(__file__).parent / "fully_connected" / "test_cases.yaml" - testcases: dict = load_solutions(TEST_CASES_FILE) + TEST_CASES_FILE: Path = ( + Path(__file__).parent / "fully_connected" / "test_cases.yaml" + ) def test_gamut(self): """ Tests the entire gamut of test cases we have specified in the yaml. """ - for test in self.testcases: - # Reads test case parameters and constructs the necessary objects. - dim_tags: list[Tag] = construct_spacetime(test["dims"]) - fill: Fill = Fill(dim_tags, test["fill"]) - occ: Occupancy = Occupancy(dim_tags, test["occ"]) - dist_fn: isl.Map = test["dist_fn"] - model: FullyConnectedMulticastModel = FullyConnectedMulticastModel(dist_fn) - - # Applies the model. - info: TransferInfo = model.apply(0, fill, occ) - # Checks the results. - sum_extract: int = info.hops.eval( - isl.Point.zero(info.hops.domain().get_space()) - ) - - # The block is used for debugging test cases not yet implemented. - if test["expected"]["fully_connected_hops"] is None: - print("~~~Test case in progress:~~~") - print(f"Fill: {fill}") - print(f"Occ: {occ}") - print(f"Dist Fn: {dist_fn}") - print(f"Returned: {sum_extract}") - else: - assert sum_extract == test["expected"]["fully_connected_hops"] + run_hops_gamut( + FullyConnectedMulticastModel, self.TEST_CASES_FILE, "fully_connected_hops" + ) diff --git a/tests/not_working/distribuffers/test_multicast.py b/tests/not_working/distribuffers/test_multicast.py index 6916de5b..61a41ced 100644 --- a/tests/not_working/distribuffers/test_multicast.py +++ b/tests/not_working/distribuffers/test_multicast.py @@ -6,46 +6,10 @@ import unittest from pathlib import Path -import islpy as isl - -from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( - # Data movement descriptors. - Fill, - Occupancy, - # Tags - Tag, - SpatialTag, - TemporalTag, -) from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( HypercubeMulticastModel, ) -from accelforge.model._looptree.reuse.isl.spatial import TransferInfo -from .util import load_solutions - - -def construct_spacetime(dims: list) -> list[Tag]: - """ - Given a list of dimension tags as strings, convert them into the proper `Tag` - objects. - - Parameters - ---------- - dims: - The list of dim tags as strings. - - Returns - ------- - list[Tag] where list[i] is the tag corresponding to dims[i]. - """ - spacetime: list[Tag] = [] - for dim in dims: - if dim["type"] == "Temporal": - spacetime.append(TemporalTag()) - elif dim["type"] == "Spatial": - spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) - - return spacetime +from .helpers import run_hops_gamut class TestHypercubeMulticastModel(unittest.TestCase): @@ -53,34 +17,10 @@ class TestHypercubeMulticastModel(unittest.TestCase): Tests the HypercubeMulticastModel with a series of premade test cases. """ - TEST_CASES_FILE: str = Path(__file__).parent / "multicast" / "test_cases.yaml" - testcases: dict = load_solutions(TEST_CASES_FILE) + TEST_CASES_FILE: Path = Path(__file__).parent / "multicast" / "test_cases.yaml" def test_gamut(self): """ Tests the entire gamut of test cases we have specified in the yaml. """ - for test in self.testcases: - # Reads test case parameters and constructs the necessary objects. - dim_tags: list[Tag] = construct_spacetime(test["dims"]) - fill: Fill = Fill(dim_tags, test["fill"]) - occ: Occupancy = Occupancy(dim_tags, test["occ"]) - dist_fn: isl.Map = test["dist_fn"] - multicast_model: HypercubeMulticastModel = HypercubeMulticastModel(dist_fn) - - # Applies the model. - info: TransferInfo = multicast_model.apply(0, fill, occ) - # Checks the results. - sum_extract: int = info.hops.eval( - isl.Point.zero(info.hops.domain().get_space()) - ) - - # The block is used for debugging test cases not yet implemented. - if test["expected"]["hypercube_hops"] is None: - print("~~~Test case in progress:~~~") - print(f"Fill: {fill}") - print(f"Occ: {occ}") - print(f"Dist Fn: {dist_fn}") - print(f"Returned: {sum_extract}") - else: - assert sum_extract == test["expected"]["hypercube_hops"] + run_hops_gamut(HypercubeMulticastModel, self.TEST_CASES_FILE, "hypercube_hops") diff --git a/tests/not_working/distribuffers/test_xy_routing.py b/tests/not_working/distribuffers/test_xy_routing.py index 7e3662f3..6fcfe5f9 100644 --- a/tests/not_working/distribuffers/test_xy_routing.py +++ b/tests/not_working/distribuffers/test_xy_routing.py @@ -13,46 +13,10 @@ import unittest from pathlib import Path -import islpy as isl - -from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( - # Data movement descriptors. - Fill, - Occupancy, - # Tags - Tag, - SpatialTag, - TemporalTag, -) from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( XYRoutingMulticastModel, ) -from accelforge.model._looptree.reuse.isl.spatial import TransferInfo -from .util import load_solutions - - -def construct_spacetime(dims: list) -> list[Tag]: - """ - Given a list of dimension tags as strings, convert them into the proper `Tag` - objects. - - Parameters - ---------- - dims: - The list of dim tags as strings. - - Returns - ------- - list[Tag] where list[i] is the tag corresponding to dims[i]. - """ - spacetime: list[Tag] = [] - for dim in dims: - if dim["type"] == "Temporal": - spacetime.append(TemporalTag()) - elif dim["type"] == "Spatial": - spacetime.append(SpatialTag(dim["spatial_dim"], dim["target"])) - - return spacetime +from .helpers import run_hops_gamut class TestXYRoutingMulticastModel(unittest.TestCase): @@ -60,34 +24,10 @@ class TestXYRoutingMulticastModel(unittest.TestCase): Tests the XYRoutingMulticastModel with a series of premade test cases. """ - TEST_CASES_FILE: str = Path(__file__).parent / "xy_routing" / "test_cases.yaml" - testcases: dict = load_solutions(TEST_CASES_FILE) + TEST_CASES_FILE: Path = Path(__file__).parent / "xy_routing" / "test_cases.yaml" def test_gamut(self): """ Tests the entire gamut of test cases we have specified in the yaml. """ - for test in self.testcases: - # Reads test case parameters and constructs the necessary objects. - dim_tags: list[Tag] = construct_spacetime(test["dims"]) - fill: Fill = Fill(dim_tags, test["fill"]) - occ: Occupancy = Occupancy(dim_tags, test["occ"]) - dist_fn: isl.Map = test["dist_fn"] - model: XYRoutingMulticastModel = XYRoutingMulticastModel(dist_fn) - - # Applies the model. - info: TransferInfo = model.apply(0, fill, occ) - # Checks the results. - sum_extract: int = info.hops.eval( - isl.Point.zero(info.hops.domain().get_space()) - ) - - # The block is used for debugging test cases not yet implemented. - if test["expected"]["xy_routing_hops"] is None: - print("~~~Test case in progress:~~~") - print(f"Fill: {fill}") - print(f"Occ: {occ}") - print(f"Dist Fn: {dist_fn}") - print(f"Returned: {sum_extract}") - else: - assert sum_extract == test["expected"]["xy_routing_hops"] + run_hops_gamut(XYRoutingMulticastModel, self.TEST_CASES_FILE, "xy_routing_hops") diff --git a/tests/not_working/distribuffers/util.py b/tests/not_working/distribuffers/util.py deleted file mode 100644 index bcb86ffa..00000000 --- a/tests/not_working/distribuffers/util.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Utility functions common to testing the isl mapper functions. -""" - -from pathlib import Path -import islpy as isl - -from ruamel.yaml import YAML - - -def to_isl_maps(obj: str | list | dict) -> dict: - """ - Given an object, attempt to reduce all strings in tree with isl.Map - - Parameters - ---------- - obj: - A DAG which can be explored and contains isl.Map strings within it. - - Returns - ------- - `obj` but all strings are converted to isl.Map. - """ - - def _to_isl_maps(obj: str | dict | list) -> isl.Map | dict | list: - """Recursively convert string ISL maps to isl.Map; leave others alone.""" - if isinstance(obj, str): - return isl.Map.read_from_str(isl.DEFAULT_CONTEXT, obj) - if isinstance(obj, dict): - return {k: (_to_isl_maps(v) if k != "type" else v) for k, v in obj.items()} - if isinstance(obj, list): - return [_to_isl_maps(v) for v in obj] - return obj - - return _to_isl_maps(obj) # type: ignore - - -def load_solutions(path: Path) -> dict: - """ - Loads in a dictionary with the isl solutions to a workload problem. - - Parameters - ---------- - path: - The path to the solutions. - - Returns - ------- - A dictionary relating Python-based keys generated by the mapper (e.g., - `BufferTensorEinsum` to their corresponding isl.Map.) - """ - # Load expected solutions (YAML file with string ISL maps) - yaml: YAML = YAML(typ="safe") - - with open(path, "r", encoding="utf-8") as f: - return to_isl_maps(yaml.load(f)) From 1537a06bdd86340421de3ab0bb4baf207ec6d5fc Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Fri, 3 Jul 2026 10:21:30 -0400 Subject: [PATCH 14/19] pre-plane items --- .../_looptree/reuse/isl/distributed/README.md | 12 +- .../isl/distributed/distributed_buffers.py | 434 ++++++------------ .../model/_looptree/reuse/isl/spatial.py | 33 +- 3 files changed, 169 insertions(+), 310 deletions(-) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index cad8a6a5..2f084d50 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -179,7 +179,7 @@ It folds the per-dimension extents into a single cost, then `.sum()`s over all n ### Return value ```python -# { dst -> data } fills actually covered by a matched source (D2). +# { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) return TransferInfo( @@ -199,7 +199,7 @@ Note the partition idiom: `_covered_fills(mcs)` reshapes the multicast networks dst and data"), which lines up with `fills.map_` so `intersect`/`subtract` split the fills into the covered and uncovered halves. Earlier revisions returned the *entire* fill map as `fulfilled_fill` and an always-empty `unfulfilled_fill` (`fills.map_.subtract(fills.map_)`); that silently -mis-reported fills whose datum no source holds, and was fixed in D2 — do not copy that idiom. +mis-reported fills whose datum no source holds — do not copy that idiom. --- @@ -286,7 +286,7 @@ class MyTransferModel(TransferModel): # 2. TODO: compute your cost as an isl.PwQPolynomial. hops: isl.PwQPolynomial = self._cost(mcs) - # 3. Partition the fills by whether a source was matched (D2). + # 3. Partition the fills by whether a source was matched. covered: isl.Map = _covered_fills(mcs) # { dst -> data } # 4. Assemble the result. @@ -347,7 +347,7 @@ class FullyConnectedMulticastModel(TransferModel): def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_fully_connected(mcs) - # { dst -> data } fills actually covered by a matched source (D2). + # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) return TransferInfo( @@ -361,7 +361,7 @@ class FullyConnectedMulticastModel(TransferModel): ) def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: - """Count the deliveries in ``mcns`` that traverse the fabric (dist >= 1).""" + """Count the deliveries in `mcns` that traverse the fabric (dist >= 1).""" # [dst -> src] pairs that actually traverse the fabric (>= 1 hop). crossing_hops: isl.Set = isl.Set.read_from_str( isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" @@ -449,7 +449,7 @@ view can never disagree: ```python def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: # `identify_mesh_casts` is called exactly once; `hops`, `edge_pressure`, - # and the fill partition all derive from this single `mcs` (D4/D5). + # and the fill partition all derive from this single `mcs`. mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) links: list[isl.Map] = self._directed_mesh_links(mcs) # see §8 pressure: EdgePressure = _edge_pressure_from_links(links) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index ce2b6447..06faa62f 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -28,40 +28,33 @@ class EdgePressure: """ Per-edge memory pressure (link load) for a spatial transfer. - Where a ``TransferModel``'s ``hops`` collapses a whole routing to a single - scalar, ``EdgePressure`` keeps the load broken out *per physical edge*: how - many multicast trees cross each individual link. This is what a per-link - bandwidth limit acts on -- the busiest edge (``bottleneck``) is what saturates - first, exactly the quantity the symbolic network model calls - ``max_traffic`` and divides by an edge's throughput to get a bandwidth-bound - latency. - - "Edge" is a *directed* physical link. Its identity is encoded in the tuple - name and coordinates of ``load``'s domain, e.g. ``yedge_u[x, t]`` (the upward - vertical link in column ``x`` between rows ``t`` and ``t + 1``) for the XY - mesh model, or ``spoke_in[n]`` / ``spoke_out[n]`` (a node's ingress / egress - link to the central switch) for the star/fully-connected spokes model. - - The load is keyed on the multicast *tree* ``(data, src)``, not on individual - destinations: within one tree a link is traversed once regardless of how many - leaves hang off it, so ``load`` measures pressure (distinct flows over a link) - rather than summed hops. Consequently the per-direction sums satisfy useful - invariants -- e.g. for XY routing ``sum over edges of load == total hops``. - - Attributes - ---------- - load: - An ``isl.UnionPwQPolynomial`` ``{ edge -> number-of-trees }`` spanning - every directed edge type the model emits. + Where a `TransferModel`'s `hops` collapses a whole routing to a single + scalar, `EdgePressure` keeps the load broken out per physical edge: how many + multicast trees cross each individual link -- the quantity a per-link + bandwidth limit acts on, and what the symbolic network model calls + `max_traffic`. + + "Edge" is a directed physical link, identified by the tuple name and + coordinates of `load`'s domain, e.g. `yedge_u[x, t]` (the upward vertical + link in column `x` between rows `t` and `t + 1`, XY mesh) or `spoke_in[n]` / + `spoke_out[n]` (a node's ingress / egress link to the switch, star model). + + The load is keyed on the multicast tree `(data, src)`, not on individual + destinations: within one tree a link is traversed once regardless of how + many leaves hang off it, so `load` measures pressure (distinct flows over a + link) rather than summed hops -- e.g. for XY routing, sum over edges of load + == total hops. - Notes - ----- - ``bottleneck`` and ``eval_edge`` assume the load is piecewise constant (the + Preconditions + ------------- + `bottleneck` and `eval_edge` assume the load is piecewise constant (the parameter-free regime the distributed models are validated in); there is no clean ISL "max of a quasi-polynomial over its domain" primitive, so the bottleneck is obtained by enumerating pieces. """ + # { edge -> number-of-trees } spanning every directed edge type the model + # emits. load: isl.UnionPwQPolynomial def total(self) -> int: @@ -69,9 +62,9 @@ def total(self) -> int: Sum the load over every edge. For a single-direction (or single edge-type) pressure this is the total - traffic; for the full mesh pressure it equals the model's total ``hops`` - (the ``sum over edges of load == total hops`` invariant), which is the - primary cross-check that the per-edge decomposition is correct. + traffic; for the full mesh pressure it equals the model's total `hops` + (sum over edges of load == total hops), which is the primary cross-check + that the per-edge decomposition is correct. """ pieces: list[isl.PwQPolynomial] = [] self.load.foreach_pw_qpolynomial(pieces.append) @@ -91,8 +84,8 @@ def bottleneck(self) -> int: bandwidth, the most-congested edge saturates first, so its load sets the transfer's bandwidth-bound latency. - The per-edge load is generally *not* constant across an edge type (e.g. a - flooded column's upward link ``yedge_u[x, t]`` carries ``t + 1`` trees), so + The per-edge load is generally not constant across an edge type (e.g. a + flooded column's upward link `yedge_u[x, t]` carries `t + 1` trees), so the maximum is found by enumerating the (finite, parameter-free) edge domain and evaluating the load at each edge -- sampling one point per piece would under-report a monotone load. @@ -109,15 +102,15 @@ def bottleneck(self) -> int: def eval_edge(self, name: str, coords: list[int]) -> int: """ - Look up the load on one named edge, e.g. ``eval_edge("yedge_u", [0, 6])``. + Look up the load on one named edge, e.g. `eval_edge("yedge_u", [0, 6])`. Returns 0 if no flow crosses that edge (it is outside the load's support). Parameters ---------- name: - The edge tuple name (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d`` - for the mesh model, ``spoke_in``/``spoke_out`` for the spokes model). + The edge tuple name (`xedge_r`/`xedge_l`/`yedge_u`/`yedge_d` + for the mesh model, `spoke_in`/`spoke_out` for the spokes model). coords: The integer edge coordinates within that tuple. """ @@ -139,56 +132,29 @@ def _eval_const(pwq: isl.PwQPolynomial) -> int: Evaluate a parameter-free, already-reduced piecewise quasi-polynomial to its scalar value. - # Design (D6): this idiom -- evaluate at the space's zero point to pull a - # single int out of an ``isl.PwQPolynomial`` -- previously existed four times - # with drifting return types: ``EdgePressure.total`` did it inline and - # returned ``int``, ``XYRoutingMulticastModel._eval_const`` returned the raw - # ``isl.Val`` (forcing every caller to convert), ``_cost_star_multicast`` - # inlined a variant that accumulated ``isl.Val``s, and the test suite had its - # own copy. Consolidating to one module-level, ``int``-returning helper means - # every call site agrees on both the operation and its return type, and the - # test suite can import this directly instead of re-deriving it. - - Parameters - ---------- - pwq: - A piecewise quasi-polynomial with **no free set dimensions** (its domain - is a point, up to parameters) and **no parameters** -- e.g. the output of - ``.card()`` on a parameter-free map/set, ``.sum()`` on a - parameter-free polynomial (which sums away every set dimension), or a - constant built by ``_const_pwq``. This is the "parameter-free regime" - every distributed model in this module is validated in; a piecewise - quasi-polynomial that still varies over real domain points or - parameters is evaluated only at the space's zero point, which is - **not** meaningful for such inputs -- callers are responsible for - reducing to a true constant first (via ``.sum()``/``.card()``). + Preconditions + ------------- + `pwq` has no free set dimensions and no parameters -- e.g. the output of + `.card()` on a parameter-free map/set, `.sum()` on a parameter-free + polynomial, or a constant built by `_const_pwq`. Evaluated only at the + space's zero point, so a `pwq` that still varies over domain points or + parameters gives a meaningless result; callers must reduce to a true + constant first (via `.sum()`/`.card()`). Returns ------- - The polynomial's constant value as a Python ``int``. - - Notes - ----- - ``isl.Val`` has no direct ``int()`` conversion in this islpy build (it - raises ``TypeError``); round-tripping through ``str()`` is the working - idiom already used throughout this module and ``EdgePressure``. + The polynomial's constant value as a Python `int`. """ + # Note: isl.Val has no direct int() conversion in this islpy build (it + # raises TypeError); round-tripping through str() is the working idiom + # used throughout this module. return int(str(pwq.eval(isl.Point.zero(pwq.domain().get_space())))) def _const_pwq(value: int) -> isl.PwQPolynomial: """ - Build a 0-dimensional, parameter-free constant ``isl.PwQPolynomial`` equal to - ``value``. - - # Design (D5): ``_cost_xy`` and ``_cost_star_multicast`` each independently - # built this "wrap an int as a constant PwQPolynomial" idiom (zero-dim space - # + ``QPolynomial.val_on_domain`` + ``PwQPolynomial.from_qpolynomial``) right - # before returning. Both models now compute their cost by aggregating an - # ``EdgePressure`` (``pressure.total()``) and need to hand that scalar back - # as a ``TransferInfo.hops`` polynomial, so this is factored out once rather - # than duplicated in ``XYRoutingMulticastModel.apply`` and - # ``StarMulticastModel.apply``. + Build a 0-dimensional, parameter-free constant `isl.PwQPolynomial` equal to + `value`. Parameters ---------- @@ -198,9 +164,9 @@ def _const_pwq(value: int) -> isl.PwQPolynomial: Returns ------- - An ``isl.PwQPolynomial`` over the empty (0-dim, 0-param) space, suitable - wherever a parameter-free ``hops`` cost is expected. Round-trips through - ``_eval_const`` back to ``value``. + An `isl.PwQPolynomial` over the empty (0-dim, 0-param) space, suitable + wherever a parameter-free `hops` cost is expected. Round-trips through + `_eval_const` back to `value`. """ ctx = isl.DEFAULT_CONTEXT zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) @@ -211,47 +177,28 @@ def _const_pwq(value: int) -> isl.PwQPolynomial: def _covered_fills(mcs: isl.Map) -> isl.Map: """ - Recover the ``{ dst -> data }`` fills actually covered by a matched - multicast source, from ``identify_mesh_casts``'s result. - - # Design (D2): all four models used to return - # ``fulfilled_fill = fills`` (the *entire* fill map, unconditionally) and - # ``unfulfilled_fill = fills.map_.subtract(fills.map_)`` (always empty -- - # subtracting a map from itself). That silently dropped the case - # ``identify_mesh_casts`` is explicitly built to detect: a destination - # requesting a datum that *no* source holds simply never appears in ``mcs``, - # so it should show up as unfulfilled, not as (falsely) fulfilled. - # - # ``mcs`` is ``{ [data] -> [dst -> src] }`` (one entry per *matched* - # dst/datum pair -- see ``identify_mesh_casts``). To turn that into the - # ``{ dst -> data }`` shape that lines up with ``fills.map_`` (also - # ``{ dst -> data }``, see ``Fill``/``Occupancy`` in - # ``mapping_to_isl/types.py``) for a direct ``intersect``/``subtract``: - # 1. ``range_reverse()`` { data -> [dst -> src] } -> { data -> [src -> dst] } - # 2. ``uncurry()`` { data -> [src -> dst] } -> { [data -> src] -> dst } - # 3. ``domain_factor_domain()`` { [data -> src] -> dst } -> { data -> dst } - # (keeps the *domain*'s domain factor -- i.e. drops ``src`` -- while - # preserving the ``-> dst`` range; this is exactly "forget which - # source served it, keep dst and data".) - # 4. ``reverse()`` { data -> dst } -> { dst -> data } - # Empirically verified against a hand-built example (see the scratchpad - # snippet referenced in the PR/commit that introduced this function) before - # being wired in here. + Recover the { dst -> data } fills actually covered by a matched multicast + source, from `identify_mesh_casts`'s result. Parameters ---------- mcs: - ``{ [data] -> [dst -> src] }``, the multicast-network map returned by - ``identify_mesh_casts``. + { [data] -> [dst -> src] }, the multicast-network map returned by + `identify_mesh_casts`. Returns ------- - ``{ dst -> data }`` -- every (destination, datum) pair that has a matched - source in ``mcs``. A model's fill partition is then - ``fulfilled = fills.map_.intersect(covered)`` and - ``unfulfilled = fills.map_.subtract(covered)``. + { dst -> data } -- every (destination, datum) pair that has a matched + source in `mcs`. A model's fill partition is then + `fulfilled = fills.map_.intersect(covered)` and + `unfulfilled = fills.map_.subtract(covered)`. """ - return mcs.range_reverse().uncurry().domain_factor_domain().reverse() + return ( + mcs.range_reverse() # { data -> [src -> dst] } + .uncurry() # { [data -> src] -> dst } + .domain_factor_domain() # Drops src, keeps -> dst. { data -> dst } + .reverse() # { dst -> data } + ) def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: @@ -259,38 +206,22 @@ def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: Read the spacetime/node tuple's name and dimensionality off a multicast network map, instead of assuming a hardcoded name. - # Design (D1): ``XYRoutingMulticastModel._directed_mesh_links`` and - # ``StarMulticastModel._spoke_loads`` used to hardcode the ISL tuple name - # ``'noc'`` directly into every map string they built - # (e.g. ``"{ noc[x, y] -> ... }"``). Any caller using a different spacetime - # tuple name (real pipelines derive names from occupancy maps -- see - # ``SimpleLinkTransferModel.apply`` in ``spatial.py``, which reads - # ``occs.map_.get_tuple_name(isl.dim_type.in_)`` rather than assuming a - # literal) would hit a raw ISL assertion deep inside these helpers instead - # of a clear error, because the name in the caller's maps would silently - # fail to match the literal ``'noc'`` baked into the helper's map strings. - # Reading the name back off ``mcns`` (already threaded through - # ``identify_mesh_casts`` from the caller's ``occs``/``fills``) and - # interpolating it into every map string generalizes both helpers to any - # tuple name, following the same precedent. - Parameters ---------- mcns: - ``{ [data] -> [dst -> src] }``, the multicast-network map returned by - ``identify_mesh_casts``. Both ``dst`` and ``src`` share the same node - tuple (``identify_mesh_casts``'s caller contract requires - ``src_occupancy``'s and ``dst_fill``'s domains to name the same - spacetime tuple), so it suffices to read the name/dims off one side - (``dst``, via ``.range().unwrap()``'s domain). + { [data] -> [dst -> src] }, the multicast-network map returned by + `identify_mesh_casts`. `dst` and `src` share the same node tuple, so + it suffices to read the name/dims off one side (`dst`, via + `.range().unwrap()`'s domain). Returns ------- - ``(name, dims)``: the node tuple's ISL tuple name (e.g. ``"noc"`` or - ``"pe"``) and its dimensionality. Correct even when ``mcns`` is empty -- - ISL preserves space/tuple-name information on empty relations (verified - empirically), so this does not require any data to be present. + `(name, dims)`: the node tuple's ISL tuple name (e.g. "noc" or "pe") and + its dimensionality. """ + # Note: correct even when mcns is empty -- ISL preserves space/tuple-name + # information on empty relations, so this does not require data to be + # present. node_space: isl.Map = mcns.range().unwrap() name: str = node_space.get_tuple_name(isl.dim_type.in_) dims: int = node_space.dim(isl.dim_type.in_) @@ -299,20 +230,20 @@ def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: def _edge_pressure_from_links(edge_maps: list[isl.Map]) -> EdgePressure: """ - Turn directed flow maps into an ``EdgePressure``. + Turn directed flow maps into an `EdgePressure`. Parameters ---------- edge_maps: - A list of ``{ [data -> src] -> edge }`` maps, one per directed edge type, + A list of { [data -> src] -> edge } maps, one per directed edge type, each associating a multicast tree with every edge its route traverses. Returns ------- - An ``EdgePressure`` whose ``load`` is ``{ edge -> number-of-trees }``: for - each map we reverse it and take the cardinality (``reverse().card()`` counts, - per edge, how many distinct ``(data, src)`` trees cross it), then union the - per-type results into one ``UnionPwQPolynomial``. + An `EdgePressure` whose `load` is { edge -> number-of-trees }: for each map + we reverse it and take the cardinality (`reverse().card()` counts, per + edge, how many distinct `(data, src)` trees cross it), then union the + per-type results into one `UnionPwQPolynomial`. """ acc: Optional[isl.UnionPwQPolynomial] = None for edge_map in edge_maps: @@ -346,15 +277,6 @@ def identify_mesh_casts( between the two points in terms of `hops`, a quantized atomic distance of data transmission cost. - # Design (D3): this Parameters entry previously read `{ [src -> dst] -> - # [hops] }`, the opposite of what the code below actually does -- - # `fills_to_matches.apply_range(dist_fn)` composes a `{ ... -> [dst -> - # src] }` map with `dist_fn`, which only type-checks (and only produces - # the intended "distance from this dst to this candidate src" value) if - # `dist_fn`'s domain is `[dst -> src]`. Every concrete `dist_fn` in the - # test suite and every caller in `distributed_buffers.py` already builds - # it this way; this is a documentation-only fix, no behavior change. - Caller contract: the tuple names of `dst_fill`'s and `src_occupancy`'s domains (the spacetime/node tuple, e.g. `noc[x, y]`) must match the corresponding tuple names in `dist_fn`'s domain -- ISL will raise on a @@ -502,13 +424,13 @@ def __init__(self, dist_fn: isl.Map): def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: """ Given a buffer, its fills across time, and its occupancies across time, - calculate the spatial transfers." + calculate the spatial transfers. Parameters ---------- buff: The buffer whose spatial analysis is being considered. Currently, - we rely on dist_fn to deal with this rather than buffer. + we rely on `dist_fn` to deal with this rather than `buff`. fills: The fill of `buffer` across time from parents. occs: @@ -516,19 +438,17 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- - Fills that were fulfilled, Fills that were unfilled, and parent reads per - position in spacetime. Then, gets hops per timestep. - - `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` found a matched source for that (dst, data) pair - (D2) -- a fill with no source holding its datum is `unfulfilled_fill`, - not silently treated as fulfilled. `edge_pressure` is left at its - default (`None`): the hypercube model costs a convex bounding box, which - has no notion of individual links to report pressure on (D4). + A TransferInfo whose `fulfilled_fill`/`unfulfilled_fill` partition + `fills` by whether `identify_mesh_casts` found a matched source for that + (dst, data) pair -- a fill with no source holding its datum is + `unfulfilled_fill`, not silently treated as fulfilled. `edge_pressure` + is left at its default (`None`): the hypercube model costs a convex + bounding box, which has no notion of individual links to report + pressure on. """ mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_mesh_cast_hypercube(mcs) - # { dst -> data } fills actually covered by a matched source (D2). + # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) # TODO: Read once from all buffers, assert that @@ -600,23 +520,13 @@ class FullyConnectedMulticastModel(TransferModel): Multicast cost model for a fully-connected fabric (e.g. an NVSwitch-style all-to-all interconnect). - On a fully-connected fabric every cross-node delivery costs exactly one hop - regardless of distance, and a self-delivery (source node == destination node) - costs zero. The cost of a mapping is therefore the number of deliveries that - actually traverse the fabric: + Every cross-node delivery costs exactly one hop regardless of distance, and + a self-delivery (source node == destination node) costs zero: cost = | { (data, dst, src) in mcs : dist_fn(dst, src) >= 1 } | - This is distance-independent in magnitude (one hop per crossing); ``dist_fn`` + This is distance-independent in magnitude (one hop per crossing); `dist_fn` is used only to tell self-deliveries (0 hops) apart from fabric-crossing ones. - - See Also - -------- - HypercubeMulticastModel : - Worst-case convex-hypercube cost. On a one-hot fully-connected encoding it - overestimates all-to-all traffic ~3x relative to this model (e.g. 168 vs - 56 hops for an 8-GPU all-to-all), because each unicast is costed as a - (1 + 1)(1 + 1) - 1 = 3 hypercube instead of a single crossing. """ def __init__(self, dist_fn: isl.Map): @@ -641,7 +551,7 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ---------- buff: The buffer whose spatial analysis is being considered. Unused; the - topology is captured entirely by ``dist_fn``. + topology is captured entirely by `dist_fn`. fills: The fill of `buffer` across time from parents. occs: @@ -651,15 +561,14 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ------- A TransferInfo whose `hops` is the number of fabric-crossing deliveries. `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source (D2). `edge_pressure` is left at - its default (`None`): this model treats the fabric as a - contention-free full mesh (one dedicated link per pair), so there is no - per-link pressure to report -- see `StarMulticastModel` for the spokes - realization of the same fabric that does expose it (D4). + `identify_mesh_casts` matched a source. `edge_pressure` is left at its + default (`None`): this model treats the fabric as a contention-free + full mesh (one dedicated link per pair), so there is no per-link + pressure to report. """ mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) result: isl.PwQPolynomial = self._cost_fully_connected(mcs) - # { dst -> data } fills actually covered by a matched source (D2). + # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) return TransferInfo( @@ -708,8 +617,8 @@ class XYRoutingMulticastModel(TransferModel): 2. an independent Y segment down each of those columns, starting from the source's row. - The hop cost of one such tree (source ``s = (xs, ys)`` with destination set - ``D``) is therefore:: + The hop cost of one such tree (source `s = (xs, ys)` with destination set + `D`) is therefore:: x_extent({xs} u {xd : (xd, yd) in D}) + sum over destination columns xd of @@ -721,10 +630,10 @@ class XYRoutingMulticastModel(TransferModel): extent_DOR (floor) <= XY routing <= hypercube - For example, source ``(1, 0)`` casting to ``(0, 2)`` and ``(2, 2)`` costs 4 + For example, source `(1, 0)` casting to `(0, 2)` and `(2, 2)` costs 4 (floor), 6 (XY), and 8 (hypercube) respectively. - Source selection is per destination: ``identify_mesh_casts`` pairs each + Source selection is per destination: `identify_mesh_casts` pairs each destination with its nearest source (devolving ties), and destinations that share a source form one tree; the cost sums over all such trees and all data. @@ -732,20 +641,14 @@ class XYRoutingMulticastModel(TransferModel): ------------- The NoC is two-dimensional (no temporal dimensions in the spacetime), with the first coordinate routed before the second (X then Y). The node tuple's - *name* is generic -- read off ``fills``/``occs`` at ``apply`` time rather - than hardcoded (D1), so any name works (e.g. ``noc[x, y]`` or - ``pe[x, y]``) as long as ``fills``, ``occs``, and ``self.dist_fn`` all agree - on it (see the caller contract on ``identify_mesh_casts``'s ``dist_fn`` - parameter). The tuple must be exactly 2-D; ``apply`` raises ``ValueError`` - otherwise (a non-2-D tuple used to reach a raw, opaque ISL assertion deep - inside ``_directed_mesh_links`` instead). N-dimensional dimension-order - routing is a future extension. The returned cost is a parameter-free constant - (the validated regime); parametric spacetimes are not yet supported. - - See Also - -------- - HypercubeMulticastModel : - Reaches every node in the bounding box -- an upper bound on XY routing. + name is generic -- read off `fills`/`occs` at `apply` time, so any name + works (e.g. `noc[x, y]` or `pe[x, y]`) as long as `fills`, `occs`, and + `self.dist_fn` all agree on it (see the caller contract on + `identify_mesh_casts`'s `dist_fn` parameter). The tuple must be exactly + 2-D; `apply` raises `ValueError` otherwise. N-dimensional dimension-order + routing is a future extension. The returned cost is a parameter-free + constant (the validated regime); parametric spacetimes are not yet + supported. """ def __init__(self, dist_fn: isl.Map): @@ -767,7 +670,7 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ---------- buff: The buffer whose spatial analysis is being considered. Unused; the - topology is captured by ``dist_fn`` and the node coordinates. + topology is captured by `dist_fn` and the node coordinates. fills: The fill of `buffer` across time from parents. occs: @@ -777,33 +680,27 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ------- A TransferInfo whose `hops` is the total XY-routing link count and whose `edge_pressure` is the per-directed-mesh-link decomposition backing it - (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d`` -- see - `_directed_mesh_links`). `hops` is derived from the same - `EdgePressure` (`pressure.total()`, D5) rather than recomputed - independently, so the two can never disagree. + (`xedge_r`/`xedge_l`/`yedge_u`/`yedge_d` -- see `_directed_mesh_links`). + `hops` is `pressure.total()`, so the two can never disagree. `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source (D2). + `identify_mesh_casts` matched a source. - Raises - ------ - ValueError - If the node tuple embedded in `fills`/`occs` is not exactly 2-D - (XY routing is only defined for a 2-D mesh; see `_directed_mesh_links`). + Preconditions + ------------- + The node tuple embedded in `fills`/`occs` must be exactly 2-D (XY + routing is only defined for a 2-D mesh; see `_directed_mesh_links`), + else raises `ValueError`. """ - # Design (D4): `identify_mesh_casts` is called exactly once per `apply`; - # both `hops` and `edge_pressure` are derived from this single `mcs` - # (previously the now-removed `edge_pressure` method recomputed it a - # second time from scratch, wasting the ISL/barvinok work and risking - # the two falling out of sync if `dist_fn`/inputs were mutated between - # calls). + # Note: `identify_mesh_casts` is called exactly once; both `hops` and + # `edge_pressure` are derived from this single `mcs` so they cannot + # fall out of sync. mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) links: list[isl.Map] = self._directed_mesh_links(mcs) pressure: EdgePressure = _edge_pressure_from_links(links) - # (D5) `hops` is the aggregation `EdgePressure.total()` already performs - # (sum of per-edge loads == total link count), wrapped as the constant - # `TransferInfo.hops` expects -- not a second, independent aggregation. + # `hops` is `EdgePressure.total()` (sum of per-edge loads == total link + # count), wrapped as the constant `TransferInfo.hops` expects. hops: isl.PwQPolynomial = _const_pwq(pressure.total()) - # { dst -> data } fills actually covered by a matched source (D2). + # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) return TransferInfo( @@ -820,12 +717,12 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: Decompose every multicast tree in `mcns` onto the directed mesh links it traverses under XY routing. - Each returned map is ``{ [data -> src] -> edge }`` for one directed edge + Each returned map is { [data -> src] -> edge } for one directed edge type, associating a tree with every link of that type on its route. The X phase runs along the source row out to every destination column; the Y phase runs down each destination column from the source row. Directions - split at the source: rightward/leftward in X (at the source column ``xs``) - and upward/downward in Y (at the source row ``ys``). + split at the source: rightward/leftward in X (at the source column `xs`) + and upward/downward in Y (at the source row `ys`). Parameters ---------- @@ -834,27 +731,21 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: Returns ------- - ``[xedge_r, xedge_l, yedge_u, yedge_d]`` maps. An edge ``xedge_r[t, ys]`` - is the rightward link in row ``ys`` between columns ``t`` and ``t + 1``; - ``yedge_u[x, t]`` is the upward link in column ``x`` between rows ``t`` and - ``t + 1`` (and ``_l`` / ``_d`` the opposite directions). - - Raises - ------ - ValueError - If the node tuple embedded in ``mcns`` is not exactly 2-D. (D1) Every - map string below is built assuming a 2-D ``name[x, y]`` node tuple; - a 3-D-or-more tuple (e.g. ``noc[x, y, t]``) used to fail deep inside - an ``isl.Map.read_from_str``/``apply_range`` call with a raw, - uninformative ISL assertion instead of a clear error here. + `[xedge_r, xedge_l, yedge_u, yedge_d]` maps. An edge `xedge_r[t, ys]` + is the rightward link in row `ys` between columns `t` and `t + 1`; + `yedge_u[x, t]` is the upward link in column `x` between rows `t` and + `t + 1` (and `_l` / `_d` the opposite directions). + + Preconditions + ------------- + The node tuple embedded in `mcns` must be exactly 2-D; every map string + below assumes a 2-D `name[x, y]` node tuple. Raises `ValueError` + otherwise. """ ctx = isl.DEFAULT_CONTEXT - # Design (D1): read the node tuple's name (and validate its - # dimensionality) off `mcns` instead of assuming the literal `'noc'`, - # following the precedent in `spatial.py`'s - # `SimpleLinkTransferModel.apply` (`occs.map_.get_tuple_name(...)`). - # Every map string below interpolates `name` so this method works for - # any 2-D node tuple, not just one literally called `noc`. + # Reads the node tuple's name (and validates its dimensionality) off + # `mcns` instead of assuming the literal 'noc', so this method works + # for any 2-D node tuple. name, dims = _mesh_node_tuple(mcns) if dims != 2: raise ValueError( @@ -956,23 +847,11 @@ class StarMulticastModel(TransferModel): GPU connects to a shared switch rather than to a full mesh of peers). Every node has exactly one spoke (its bidirectional link to the switch). A - delivery routes ``src -> switch -> dst``: the source injects each datum once + delivery routes `src -> switch -> dst`: the source injects each datum once up its egress spoke (multicast fan-out happens at the switch, so one copy per datum regardless of how many destinations want it), and every destination receives its datum down its ingress spoke. Self-deliveries (a node already holding the datum) never cross the fabric and so load no spoke. - - Where ``FullyConnectedMulticastModel`` treats the fabric as a contention-free - full mesh (one dedicated link per pair, no hotspot) and only counts crossings, - this model exposes *where the contention is*: the per-spoke load. The two are - tied by the invariant ``sum over nodes of ingress == FullyConnected crossing - count`` (every crossing delivery is exactly one node's ingress). - - See Also - -------- - FullyConnectedMulticastModel : - The same all-to-all traffic costed as crossings on a full mesh; its hop - count equals this model's total ingress. """ def __init__(self, dist_fn: isl.Map): @@ -997,7 +876,7 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ---------- buff: The buffer whose spatial analysis is being considered. Unused; the - topology is captured entirely by ``dist_fn``. + topology is captured entirely by `dist_fn`. fills: The fill of `buffer` across time from parents. occs: @@ -1007,27 +886,23 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo ------- A TransferInfo whose `hops` is the total spoke traversals (injections plus deliveries) and whose `edge_pressure` is the per-spoke decomposition - backing it (``spoke_in[n]``/``spoke_out[n]`` -- see `_spoke_loads`). - `hops` is derived from the same `EdgePressure` (`pressure.total()`, D5) - rather than recomputed independently, so the two can never disagree. + backing it (`spoke_in[n]`/`spoke_out[n]` -- see `_spoke_loads`). `hops` + is `pressure.total()`, so the two can never disagree. `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source (D2). + `identify_mesh_casts` matched a source. """ - # Design (D4): `identify_mesh_casts` is called exactly once per `apply`; - # both `hops` and `edge_pressure` are derived from this single `mcs` - # (previously the now-removed `edge_pressure` method recomputed it a - # second time from scratch). + # Note: `identify_mesh_casts` is called exactly once; both `hops` and + # `edge_pressure` are derived from this single `mcs`. mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) ingress, egress = self._spoke_loads(mcs) acc: isl.UnionPwQPolynomial = isl.UnionPwQPolynomial.from_pw_qpolynomial( ingress ).add(isl.UnionPwQPolynomial.from_pw_qpolynomial(egress)) pressure: EdgePressure = EdgePressure(acc) - # (D5) `hops` is the aggregation `EdgePressure.total()` already performs - # (sum(ingress) + sum(egress)), wrapped as the constant `TransferInfo.hops` - # expects -- not a second, independent aggregation. + # `hops` is `EdgePressure.total()` (sum(ingress) + sum(egress)), wrapped + # as the constant `TransferInfo.hops` expects. hops: isl.PwQPolynomial = _const_pwq(pressure.total()) - # { dst -> data } fills actually covered by a matched source (D2). + # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) return TransferInfo( @@ -1052,16 +927,13 @@ def _spoke_loads( Returns ------- - ``(ingress, egress)`` where ``ingress`` is ``{ spoke_in[n] -> #data the - node receives }`` and ``egress`` is ``{ spoke_out[n] -> #data the node - sources }``, counting only fabric-crossing (>= 1 hop) deliveries. + `(ingress, egress)` where `ingress` is { spoke_in[n] -> #data the node + receives } and `egress` is { spoke_out[n] -> #data the node sources }, + counting only fabric-crossing (>= 1 hop) deliveries. """ - # Design (D1): read the node tuple's name off `mcns` instead of assuming - # the literal `'noc'`, following the same precedent as - # `XYRoutingMulticastModel._directed_mesh_links`. Unlike XY, the star - # model has no dimensionality requirement -- `dims` (used below to build - # the `spoke_in`/`spoke_out` relabeling) is already read generically off - # `ingress_nodes`, so only the hardcoded name needed fixing here. + # Reads the node tuple's name off `mcns` instead of assuming the + # literal 'noc'; unlike XY, the star model has no dimensionality + # requirement. name, _dims = _mesh_node_tuple(mcns) # Keep only deliveries that actually cross the fabric (>= 1 hop); a node diff --git a/accelforge/model/_looptree/reuse/isl/spatial.py b/accelforge/model/_looptree/reuse/isl/spatial.py index 856197be..a278c377 100644 --- a/accelforge/model/_looptree/reuse/isl/spatial.py +++ b/accelforge/model/_looptree/reuse/isl/spatial.py @@ -23,13 +23,9 @@ ) if TYPE_CHECKING: - # Design: `EdgePressure` lives in `distributed/distributed_buffers.py`, which - # itself imports `TransferInfo` from this module (see `spatial.py`'s role as - # the base contract every distributed model builds on). Importing it at - # runtime here would form an import cycle - # (spatial -> distributed_buffers -> spatial); a `TYPE_CHECKING`-guarded - # import plus a forward-reference string annotation on the `edge_pressure` - # field below gets static type-checking for free without paying that cost. + # Note: guarded to avoid a runtime import cycle -- distributed_buffers.py + # imports TransferInfo from this module, so importing EdgePressure back + # here at runtime would form a loop. from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( EdgePressure, ) @@ -51,18 +47,14 @@ class TransferInfo: fulfilled_fill: Transfers """Fills done by peer-to-peer transfers. - Restricted to the fills actually *covered* by a matched multicast source - (see `identify_mesh_casts`) -- a destination requesting a datum no source - holds is never "fulfilled" even though it is dropped from the multicast - network, hence the separate `unfulfilled_fill` below. + Restricted to fills covered by a matched multicast source (see + `identify_mesh_casts`); see `unfulfilled_fill`. """ unfulfilled_fill: Fill """Fills not performed. - Fills whose datum has no matching source in `identify_mesh_casts`'s - result -- i.e. `fills - covered`, the complement of `fulfilled_fill` - within the original fill set (`fulfilled_fill` and `unfulfilled_fill` - partition `fills` exactly; see each model's `apply` for the derivation). + `fills - covered`, the complement of `fulfilled_fill` within the original + fill set (the two partition `fills` exactly). """ parent_reads: Reads """Fills done by parent-to-child transfers.""" @@ -79,14 +71,9 @@ class TransferInfo: edge_pressure: Optional["EdgePressure"] = None """Per-directed-edge load backing `hops`, for models that define one. - Populated by models with an explicit per-link topology (mesh links for - `XYRoutingMulticastModel`, spokes for `StarMulticastModel`) from the same - `identify_mesh_casts` result used to compute `hops`, so the two are always - consistent. `None` for models with no per-link decomposition defined -- - `HypercubeMulticastModel` costs a convex bounding box (no notion of - individual links) and `FullyConnectedMulticastModel` treats the fabric as - a contention-free full mesh (one dedicated link per pair, so there is no - shared-link pressure to report); both leave this field at its default. + Populated by `XYRoutingMulticastModel` (mesh links) and + `StarMulticastModel` (spokes); `None` where a model has no per-link + decomposition (`HypercubeMulticastModel`, `FullyConnectedMulticastModel`). """ From ea739244347f1885560056fd76785ede698c38ad Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Mon, 6 Jul 2026 16:59:31 -0400 Subject: [PATCH 15/19] better refactor --- .../_looptree/reuse/isl/distributed/README.md | 196 ++-- .../isl/distributed/distributed_buffers.py | 911 ++++++----------- .../reuse/isl/distributed/edge_pressure.py | 251 +++++ .../reuse/isl/distributed/mesh_casts.py | 268 +++++ .../model/_looptree/reuse/isl/spatial.py | 15 +- .../distributed_models_demo.ipynb | 920 ++++++++++++++++++ 6 files changed, 1786 insertions(+), 775 deletions(-) create mode 100644 accelforge/model/_looptree/reuse/isl/distributed/edge_pressure.py create mode 100644 accelforge/model/_looptree/reuse/isl/distributed/mesh_casts.py create mode 100644 notebooks/astrasim2_correlation/distributed_models_demo.ipynb diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index 2f084d50..a3936169 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -28,6 +28,15 @@ implementations: > test suite — [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py). > The `not_working/` location signals these models are work-in-progress. +> **Module layout.** This directory splits into three files: the four model classes above and +> the shared `MulticastModel` base they all inherit (one `apply()`, one abstract `_transfer_cost` +> hook per model) live in [`distributed_buffers.py`](distributed_buffers.py); the `EdgePressure` +> per-link-load abstraction and its scalar-extraction helpers live in +> [`edge_pressure.py`](edge_pressure.py); multicast-network construction (`identify_mesh_casts` +> and the helpers built on its result) lives in [`mesh_casts.py`](mesh_casts.py). +> `distributed_buffers.py` re-exports the moved names, so existing imports of the four models and +> `_eval_const` from `distributed_buffers.py` keep working (see §9 for the full map). + --- ## 2. The `TransferModel` contract @@ -133,7 +142,7 @@ occs.map_ { [spacetime] -> [data] } fills.map_ { [spacetime] -> [data] } ``` **Step A — `identify_mesh_casts(src_occupancy, dst_fill, dist_fn)`** -([`distributed_buffers.py`](distributed_buffers.py)). +([`mesh_casts.py`](mesh_casts.py)). For every datum, it pairs the destinations that request it with the *nearest* source that holds it. Conceptually: @@ -150,7 +159,7 @@ The result is the set of **multicast networks** (`mcns`): per datum, the destina their chosen source. **Step B — extents per dimension: `calculate_extents_per_dim(mcns)`** -([`distributed_buffers.py`](distributed_buffers.py)). +([`mesh_casts.py`](mesh_casts.py)). For each multicast network it unions the sources with the destinations, then for each NoC dimension projects away the others and takes `dim_max − dim_min`. That difference is the **extent** (the side @@ -227,23 +236,24 @@ So these two models bracket the design space: **stateful + distance-aware** (hyp 1. **Pick a home.** Add your class under `reuse/isl/`. Put distributed / NoC / mesh models in this `distributed/` directory next to `distributed_buffers.py`. -2. **Subclass `TransferModel`** and decide your constructor state — a `dist_fn`, topology - parameters, bandwidth, or nothing (like `SimpleLinkTransferModel`). -3. **Implement `apply(self, buff, fills, occs) -> TransferInfo`.** If your model is mesh/multicast - shaped, reuse the existing kernels: - - `identify_mesh_casts(occs.map_, fills.map_, self.dist_fn)` to get `{ [data] -> [dst -> src] }`. - - `_covered_fills(mcs)` to partition the fills into fulfilled/unfulfilled. +2. **Subclass `MulticastModel`** if your model is distance-driven multicast (one `dist_fn`, + nearest-source matching) — the base owns the constructor, the single `identify_mesh_casts` + call, the fill partition, and the `TransferInfo` assembly. Only subclass `TransferModel` + directly (and write your own `apply`) for a different shape entirely, like the stateless + `SimpleLinkTransferModel`. +3. **Implement `_transfer_cost(self, mcs) -> isl.PwQPolynomial | EdgePressure`.** `mcs` is the + `{ [data] -> [dst -> src] }` multicast networks, computed once by the base `apply`. Reuse the + existing kernels: - `calculate_extents_per_dim(mcns)` if you want per-dimension bounding-box extents. - - `_edge_pressure_from_links(...)` / `_const_pwq(...)` / `_eval_const(...)` if you report a - per-link `EdgePressure` (see §8) and derive `hops` from it. - Otherwise write your own cost kernel over the ISL maps. Call `identify_mesh_casts` **once** per - `apply` and derive everything (`hops`, `edge_pressure`, the fill partition) from that single - result, so the outputs can never disagree. -4. **Honor the invariants.** Assert `fills.tags == occs.tags` if your model needs aligned tags. - Build `hops` as an `isl.PwQPolynomial` over the correct domain, and partition the fills with - `_covered_fills` (`fulfilled = fills ∩ covered`, `unfulfilled = fills − covered`) rather than - declaring everything fulfilled. Decide deliberately whether your topology defines an - `edge_pressure` (per-link decomposition) or leaves it `None`. + - `_fabric_crossing(mcns, self.dist_fn)` to keep only deliveries that cross the fabric. + - `_edge_pressure_from_links(...)` if you report a per-link `EdgePressure` (see §8) — return + it directly and the base derives `hops` from its total; return a bare `isl.PwQPolynomial` + and the base leaves `edge_pressure` at `None`. +4. **Honor the invariants.** The base guarantees the big ones structurally: the fill partition + (`fulfilled = fills ∩ covered`, `unfulfilled = fills − covered`), one `identify_mesh_casts` + call per `apply`, and `hops == EdgePressure.total()` whenever you return a pressure. What is + left to you: build the cost over the correct domain, and decide deliberately whether your + topology defines a per-link decomposition or not. 5. **Add a test** mirroring [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py): a YAML-driven gamut of `(dims, fill, occ, dist_fn, expected_hops)` cases. @@ -253,70 +263,36 @@ So these two models bracket the design space: **stateful + distance-aware** (hyp ```python import islpy as isl -from accelforge.frontend.mapping import MappingNode -from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import Fill, Occupancy -from accelforge.model._looptree.reuse.isl.spatial import ( - Reads, - Transfers, - TransferInfo, - TransferModel, -) -# Reuse these if your model is mesh/multicast-shaped: from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( - identify_mesh_casts, - calculate_extents_per_dim, - _covered_fills, + MulticastModel, ) -class MyTransferModel(TransferModel): +class MyMulticastModel(MulticastModel): """One-line description of the topology/assumptions this model encodes.""" - def __init__(self, dist_fn: isl.Map): - # TODO: store whatever state your cost kernel needs (or drop the arg entirely). - self.dist_fn = dist_fn - - def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - # TODO (optional): assert fills.tags == occs.tags - - # 1. Group destinations with their nearest source per datum. Call this - # ONCE and derive every output from the same `mcs`. - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - - # 2. TODO: compute your cost as an isl.PwQPolynomial. - hops: isl.PwQPolynomial = self._cost(mcs) - - # 3. Partition the fills by whether a source was matched. - covered: isl.Map = _covered_fills(mcs) # { dst -> data } - - # 4. Assemble the result. - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=hops, - link_transfer=True, - # TODO: an EdgePressure if your topology defines per-link loads - # (see §8); None if it has no per-link decomposition. - edge_pressure=None, - ) - - def _cost(self, mcns: isl.Map) -> isl.PwQPolynomial: - # TODO: your cost kernel. + def _transfer_cost(self, mcs: isl.Map) -> isl.PwQPolynomial: + # `mcs` is { [data] -> [dst -> src] } from `identify_mesh_casts`, + # computed once by `MulticastModel.apply`; the fill partition and the + # `TransferInfo` assembly are already handled there. + # TODO: your cost kernel. Return an `EdgePressure` instead (see §8) if + # your topology defines per-link loads -- `apply` then derives `hops` + # from its total automatically. raise NotImplementedError ``` +(A non-multicast model — different matching, no `dist_fn` — instead subclasses `TransferModel` +directly and implements the full `apply`; use `SimpleLinkTransferModel` in +[`../spatial.py`](../spatial.py) as the reference for that shape.) + **Checklist** -- [ ] Subclasses `TransferModel`, implements `apply`. -- [ ] Constructor state matches what the cost kernel needs. -- [ ] `hops` is an `isl.PwQPolynomial` over the right domain. -- [ ] `fulfilled_fill` + `unfulfilled_fill` partition the fills via `_covered_fills(mcs)` - (`fills ∩ covered` / `fills − covered`), not "everything fulfilled / empty unfulfilled". -- [ ] `identify_mesh_casts` is called once per `apply`; `hops`, `edge_pressure`, and the fill - partition all derive from the same result. -- [ ] `edge_pressure` is a deliberate choice: an `EdgePressure` if the topology has per-link loads, - `None` (with a comment saying why) if not. +- [ ] Subclasses `MulticastModel` and implements `_transfer_cost` (only non-multicast shapes + subclass `TransferModel` and hand-roll `apply`). +- [ ] Constructor state matches what the cost kernel needs (the base stores `dist_fn`; override + `__init__` only to add state). +- [ ] `_transfer_cost` returns an `isl.PwQPolynomial` over the right domain — or an + `EdgePressure` for per-link topologies, making `hops == Σ_edges load` structural. - [ ] A YAML-driven gamut test exists. --- @@ -338,44 +314,25 @@ cost = | { (data, dst, src) ∈ mcs : dist_fn(dst, src) ≥ 1 } | *magnitude* never enters the cost. This sidesteps the hypercube extent overestimate (see §3) entirely. ```python -class FullyConnectedMulticastModel(TransferModel): +class FullyConnectedMulticastModel(MulticastModel): """Multicast cost on a fully-connected fabric: 1 hop per fabric crossing.""" - def __init__(self, dist_fn: isl.Map): - self.dist_fn = dist_fn - - def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - result: isl.PwQPolynomial = self._cost_fully_connected(mcs) - # { dst -> data } fills actually covered by a matched source. - covered: isl.Map = _covered_fills(mcs) - - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=result, - link_transfer=True, - # No per-link decomposition defined for the full-mesh abstraction. - edge_pressure=None, - ) + def _transfer_cost(self, mcs: isl.Map) -> isl.PwQPolynomial: + return self._cost_fully_connected(mcs) def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: """Count the deliveries in `mcns` that traverse the fabric (dist >= 1).""" # [dst -> src] pairs that actually traverse the fabric (>= 1 hop). - crossing_hops: isl.Set = isl.Set.read_from_str( - isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" - ) - crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() - crossing: isl.Map = mcns.intersect_range(crossing_pairs) + crossing: isl.Map = _fabric_crossing(mcns, self.dist_fn) return crossing.wrap().card() ``` -How the kernel works: `dist_fn.intersect_range({ hops[h] : h ≥ 1 }).domain()` is the set of -`[dst -> src]` pairs that cross the fabric; intersecting `mcns`' range with it keeps only crossing -deliveries; `.wrap().card()` counts the `(data, dst, src)` points as an `isl.PwQPolynomial` -(constant when there are no parameters). `intersect_range`, `domain`, `wrap`, and `card` are all -standard `islpy`/barvinok operations already used elsewhere in this subsystem. +How the kernel works: `_fabric_crossing` ([`mesh_casts.py`](mesh_casts.py)) computes +`dist_fn.intersect_range({ hops[h] : h ≥ 1 }).domain()` — the set of `[dst -> src]` pairs that +cross the fabric — and intersects `mcns`' range with it, keeping only crossing deliveries; +`.wrap().card()` then counts the `(data, dst, src)` points as an `isl.PwQPolynomial` (constant +when there are no parameters). Everything else — the single `identify_mesh_casts` call, the fill +partition, `edge_pressure=None` — is `MulticastModel.apply`'s job, not this class's. **Verified numbers** (8-GPU one-hot encoding, from the test below): @@ -441,33 +398,18 @@ model): source `(1,0)` casting to `(0,2)` and `(2,2)`: ### Cost kernel -There is no standalone cost method: the cost *is* the per-edge decomposition, aggregated. `apply()` -decomposes every tree onto directed mesh links, wraps the per-link loads as an `EdgePressure` -(see §8), and derives `hops` from its total — one aggregation path, so the scalar and the per-edge -view can never disagree: +There is no standalone cost method: the cost *is* the per-edge decomposition, aggregated. The +whole model is one hook — ```python -def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - # `identify_mesh_casts` is called exactly once; `hops`, `edge_pressure`, - # and the fill partition all derive from this single `mcs`. - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - links: list[isl.Map] = self._directed_mesh_links(mcs) # see §8 - pressure: EdgePressure = _edge_pressure_from_links(links) - # `hops` == the sum of per-edge loads (Σ_edges load == total link count), - # wrapped as the constant PwQPolynomial `TransferInfo.hops` expects. - hops: isl.PwQPolynomial = _const_pwq(pressure.total()) - covered: isl.Map = _covered_fills(mcs) # { dst -> data } - - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=hops, - link_transfer=True, - edge_pressure=pressure, - ) +def _transfer_cost(self, mcs: isl.Map) -> EdgePressure: + return _edge_pressure_from_links(self._directed_mesh_links(mcs)) ``` +— because it returns an `EdgePressure` (see §8), `MulticastModel.apply` sets `edge_pressure` to +it and derives `hops = _const_pwq(pressure.total())` in one place: one aggregation path, so the +scalar and the per-edge view can never disagree (Σ_edges load == total link count, structurally). + `_directed_mesh_links` builds the X segments explicitly along each source row (out to every destination column, split rightward/leftward at the source column `xs`) and the Y segments down each destination column from the source row (split upward/downward at `ys`), keeping every link's @@ -578,8 +520,12 @@ PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ ## 9. References - Interface + `SimpleLinkTransferModel` + `TransferInfo`: [`../spatial.py`](../spatial.py) -- `HypercubeMulticastModel`, `identify_mesh_casts`, `calculate_extents_per_dim`: - [`distributed_buffers.py`](distributed_buffers.py) +- `MulticastModel` base + the four model classes, `HypercubeMulticastModel`'s + `_cost_mesh_cast_hypercube`: [`distributed_buffers.py`](distributed_buffers.py) +- `identify_mesh_casts`, `calculate_extents_per_dim`, `_covered_fills`, `_mesh_node_tuple`: + [`mesh_casts.py`](mesh_casts.py) +- `EdgePressure`, `_eval_const`, `_const_pwq`, `_edge_pressure_from_links`: + [`edge_pressure.py`](edge_pressure.py) - Tagged-map / tag types: [`../mapping_to_isl/types.py`](../mapping_to_isl/types.py) - Example test harness: [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index 06faa62f..601f4d5a 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -1,17 +1,24 @@ """ Models for handling calculating the cost of a Workload on distributed buffer architectures. + +The shared shape (one `dist_fn`, one `identify_mesh_casts` call per `apply`, +one `TransferInfo` assembly) lives on the `MulticastModel` base class; each +concrete model below contributes only its cost kernel via `_transfer_cost`. +The underlying primitives -- `EdgePressure` and its scalar-extraction helpers, +and the multicast-network construction (`identify_mesh_casts` and friends) -- +live in `edge_pressure.py` and `mesh_casts.py` respectively, and are +re-exported below for backward compatibility (see the Note at the import +site). """ -import logging +from abc import abstractmethod -from dataclasses import dataclass +from typing import Optional import islpy as isl from accelforge.frontend.mapping import MappingNode -from accelforge.model._looptree.reuse.isl.isl_functions import dim_projector_mask -from accelforge.model._looptree.reuse.isl.mapping_to_isl import DUMP_ISL_IR from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import Fill, Occupancy from accelforge.model._looptree.reuse.isl.spatial import ( Reads, @@ -20,404 +27,70 @@ TransferModel, ) -from typing import Optional +# Note: these re-exports preserve this module's historical public surface. +# `EdgePressure`/`_eval_const`/`_const_pwq`/`_edge_pressure_from_links` and +# `identify_mesh_casts`/`calculate_extents_per_dim`/`_covered_fills`/ +# `_mesh_node_tuple` used to be defined directly in this file; the test suite +# and `correlation.ipynb` import them from this module path, so the split +# into `edge_pressure.py` / `mesh_casts.py` must not break that import. +# `_union_pwqs`, `_per_src`, and `_fabric_crossing` are pulled in for this +# module's own internal use (Star/XY/FullyConnected kernels below), not part +# of the historical surface. +from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import ( + EdgePressure, + _eval_const, + _const_pwq, + _edge_pressure_from_links, + _union_pwqs, +) +from accelforge.model._looptree.reuse.isl.distributed.mesh_casts import ( + identify_mesh_casts, + calculate_extents_per_dim, + _covered_fills, + _mesh_node_tuple, + _per_src, + _fabric_crossing, +) -@dataclass(frozen=True) -class EdgePressure: +class MulticastModel(TransferModel): """ - Per-edge memory pressure (link load) for a spatial transfer. - - Where a `TransferModel`'s `hops` collapses a whole routing to a single - scalar, `EdgePressure` keeps the load broken out per physical edge: how many - multicast trees cross each individual link -- the quantity a per-link - bandwidth limit acts on, and what the symbolic network model calls - `max_traffic`. - - "Edge" is a directed physical link, identified by the tuple name and - coordinates of `load`'s domain, e.g. `yedge_u[x, t]` (the upward vertical - link in column `x` between rows `t` and `t + 1`, XY mesh) or `spoke_in[n]` / - `spoke_out[n]` (a node's ingress / egress link to the switch, star model). - - The load is keyed on the multicast tree `(data, src)`, not on individual - destinations: within one tree a link is traversed once regardless of how - many leaves hang off it, so `load` measures pressure (distinct flows over a - link) rather than summed hops -- e.g. for XY routing, sum over edges of load - == total hops. - - Preconditions - ------------- - `bottleneck` and `eval_edge` assume the load is piecewise constant (the - parameter-free regime the distributed models are validated in); there is no - clean ISL "max of a quasi-polynomial over its domain" primitive, so the - bottleneck is obtained by enumerating pieces. + Common shape shared by every distance-driven multicast transfer model. + + Every concrete model below (`HypercubeMulticastModel`, + `FullyConnectedMulticastModel`, `XYRoutingMulticastModel`, + `StarMulticastModel`) is built from one `dist_fn` and computes `apply` + identically up to its cost kernel: call `identify_mesh_casts` exactly + once, derive the cost from that single result, partition the fills, and + assemble a `TransferInfo`. This base class owns that shared shape; a + subclass need only implement `_transfer_cost`. + + Because `hops` and `edge_pressure` are both derived here from the same + `_transfer_cost` return value, `Σ_edges load == hops` (for models that + report an `EdgePressure`) is structural rather than an invariant each + subclass has to maintain by hand: whenever `_transfer_cost` returns an + `EdgePressure` `p`, `hops` is *always* `_const_pwq(p.total())`. """ - # { edge -> number-of-trees } spanning every directed edge type the model - # emits. - load: isl.UnionPwQPolynomial - - def total(self) -> int: - """ - Sum the load over every edge. - - For a single-direction (or single edge-type) pressure this is the total - traffic; for the full mesh pressure it equals the model's total `hops` - (sum over edges of load == total hops), which is the primary cross-check - that the per-edge decomposition is correct. - """ - pieces: list[isl.PwQPolynomial] = [] - self.load.foreach_pw_qpolynomial(pieces.append) - total = 0 - for pwq in pieces: - # `.sum()` collapses the edge-indexed domain away (summing over every - # edge of this piece), leaving a 0-set-dim, parameter-free - # polynomial -- exactly `_eval_const`'s precondition. - total += _eval_const(pwq.sum()) - return total - - def bottleneck(self) -> int: - """ - Return the load on the single most-pressured edge. - - This is the bandwidth-binding quantity: with a uniform per-link - bandwidth, the most-congested edge saturates first, so its load sets the - transfer's bandwidth-bound latency. - - The per-edge load is generally not constant across an edge type (e.g. a - flooded column's upward link `yedge_u[x, t]` carries `t + 1` trees), so - the maximum is found by enumerating the (finite, parameter-free) edge - domain and evaluating the load at each edge -- sampling one point per - piece would under-report a monotone load. - """ - best = 0 - pieces: list[isl.PwQPolynomial] = [] - self.load.foreach_pw_qpolynomial(pieces.append) - for pwq in pieces: - edges: list[isl.Point] = [] - pwq.domain().foreach_point(edges.append) - for edge in edges: - best = max(best, int(str(pwq.eval(edge)))) - return best - - def eval_edge(self, name: str, coords: list[int]) -> int: + def __init__(self, dist_fn: isl.Map): """ - Look up the load on one named edge, e.g. `eval_edge("yedge_u", [0, 6])`. - - Returns 0 if no flow crosses that edge (it is outside the load's support). - Parameters ---------- - name: - The edge tuple name (`xedge_r`/`xedge_l`/`yedge_u`/`yedge_d` - for the mesh model, `spoke_in`/`spoke_out` for the spokes model). - coords: - The integer edge coordinates within that tuple. - """ - pieces: list[isl.PwQPolynomial] = [] - self.load.foreach_pw_qpolynomial(pieces.append) - for pwq in pieces: - if pwq.domain().get_space().get_tuple_name(isl.dim_type.set) != name: - continue - point: isl.Point = isl.Set.read_from_str( - isl.DEFAULT_CONTEXT, - "{ %s[%s] }" % (name, ", ".join(str(c) for c in coords)), - ).sample_point() - return int(str(pwq.eval(point))) - return 0 - - -def _eval_const(pwq: isl.PwQPolynomial) -> int: - """ - Evaluate a parameter-free, already-reduced piecewise quasi-polynomial to its - scalar value. - - Preconditions - ------------- - `pwq` has no free set dimensions and no parameters -- e.g. the output of - `.card()` on a parameter-free map/set, `.sum()` on a parameter-free - polynomial, or a constant built by `_const_pwq`. Evaluated only at the - space's zero point, so a `pwq` that still varies over domain points or - parameters gives a meaningless result; callers must reduce to a true - constant first (via `.sum()`/`.card()`). - - Returns - ------- - The polynomial's constant value as a Python `int`. - """ - # Note: isl.Val has no direct int() conversion in this islpy build (it - # raises TypeError); round-tripping through str() is the working idiom - # used throughout this module. - return int(str(pwq.eval(isl.Point.zero(pwq.domain().get_space())))) - - -def _const_pwq(value: int) -> isl.PwQPolynomial: - """ - Build a 0-dimensional, parameter-free constant `isl.PwQPolynomial` equal to - `value`. - - Parameters - ---------- - value: - The integer the returned polynomial evaluates to everywhere (it has no - domain dimensions or parameters to vary over). - - Returns - ------- - An `isl.PwQPolynomial` over the empty (0-dim, 0-param) space, suitable - wherever a parameter-free `hops` cost is expected. Round-trips through - `_eval_const` back to `value`. - """ - ctx = isl.DEFAULT_CONTEXT - zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) - return isl.PwQPolynomial.from_qpolynomial( - isl.QPolynomial.val_on_domain(zero_dim, isl.Val(value, ctx)) - ) - - -def _covered_fills(mcs: isl.Map) -> isl.Map: - """ - Recover the { dst -> data } fills actually covered by a matched multicast - source, from `identify_mesh_casts`'s result. - - Parameters - ---------- - mcs: - { [data] -> [dst -> src] }, the multicast-network map returned by - `identify_mesh_casts`. - - Returns - ------- - { dst -> data } -- every (destination, datum) pair that has a matched - source in `mcs`. A model's fill partition is then - `fulfilled = fills.map_.intersect(covered)` and - `unfulfilled = fills.map_.subtract(covered)`. - """ - return ( - mcs.range_reverse() # { data -> [src -> dst] } - .uncurry() # { [data -> src] -> dst } - .domain_factor_domain() # Drops src, keeps -> dst. { data -> dst } - .reverse() # { dst -> data } - ) - - -def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: - """ - Read the spacetime/node tuple's name and dimensionality off a multicast - network map, instead of assuming a hardcoded name. - - Parameters - ---------- - mcns: - { [data] -> [dst -> src] }, the multicast-network map returned by - `identify_mesh_casts`. `dst` and `src` share the same node tuple, so - it suffices to read the name/dims off one side (`dst`, via - `.range().unwrap()`'s domain). - - Returns - ------- - `(name, dims)`: the node tuple's ISL tuple name (e.g. "noc" or "pe") and - its dimensionality. - """ - # Note: correct even when mcns is empty -- ISL preserves space/tuple-name - # information on empty relations, so this does not require data to be - # present. - node_space: isl.Map = mcns.range().unwrap() - name: str = node_space.get_tuple_name(isl.dim_type.in_) - dims: int = node_space.dim(isl.dim_type.in_) - return name, dims - - -def _edge_pressure_from_links(edge_maps: list[isl.Map]) -> EdgePressure: - """ - Turn directed flow maps into an `EdgePressure`. - - Parameters - ---------- - edge_maps: - A list of { [data -> src] -> edge } maps, one per directed edge type, - each associating a multicast tree with every edge its route traverses. - - Returns - ------- - An `EdgePressure` whose `load` is { edge -> number-of-trees }: for each map - we reverse it and take the cardinality (`reverse().card()` counts, per - edge, how many distinct `(data, src)` trees cross it), then union the - per-type results into one `UnionPwQPolynomial`. - """ - acc: Optional[isl.UnionPwQPolynomial] = None - for edge_map in edge_maps: - # { edge -> #trees crossing it } - per_edge: isl.PwQPolynomial = edge_map.reverse().card() - contribution = isl.UnionPwQPolynomial.from_pw_qpolynomial(per_edge) - acc = contribution if acc is None else acc.add(contribution) - if acc is None: - acc = isl.UnionPwQPolynomial.read_from_str(isl.DEFAULT_CONTEXT, "{ }") - return EdgePressure(acc) - - -def identify_mesh_casts( - src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map -) -> isl.Map: - """ - Given srcs with data, fills to destinations, and a distance function, identify per data - the srcs delivering that data to dsts. - - Parameters - ---------- - src_occupancy: - An isl.Map of the form { [src] -> [data] } corresponding to the data held - at the buffer at space `src`. - dst_fill: - An isl.Map of the form { [dst] -> [data] } corresponding to the data requested - at the element at space `dst`. - dist_fn: - A distance function { [dst -> src] -> [hops] } that accepts two points in - space, corresponding to the `dst` and `src`, and returns the distance - between the two points in terms of `hops`, a quantized atomic distance of - data transmission cost. - - Caller contract: the tuple names of `dst_fill`'s and `src_occupancy`'s - domains (the spacetime/node tuple, e.g. `noc[x, y]`) must match the - corresponding tuple names in `dist_fn`'s domain -- ISL will raise on a - name mismatch when `dist_fn` is applied. `dist_fn`'s range tuple must be - named `hops` (consumers such as `FullyConnectedMulticastModel` and - `StarMulticastModel` filter on `{ hops[h] : h >= 1 }` to distinguish - self-deliveries from fabric-crossing ones). - - Returns - ------- - { [data] -> [dst -> src] } where { [dst] -> [data] } and { [src] -> [data] } are in - `src_occupancy` and `dst_fill` respectively, and where `[dst -> src]` is the infimum of - `dst_fn(src, dst), ∀ src, dst s.t. { [src] -> [data] } ∈ `src_occupancy` and - `{ [dst] -> [data] }` ∈ `dst_fill`. - """ - # Makes { [dst -> data] -> [dst -> data] } - fill_to_fill: isl.Map = dst_fill.wrap().identity() - if DUMP_ISL_IR: - logging.info(f"fill_to_fill: {fill_to_fill}") - - # Inverts src_occupancy s.t. data -> src. - # i.e. { [xs, ys] -> [d0, d1] } to { [d0, d1] -> [xs, ys] } - data_presence: isl.Map = src_occupancy.reverse() - - # { [dst -> data] -> [dst -> src] } where src contains data. - fills_to_matches: isl.Map = ( - fill_to_fill.uncurry() # { [[dst -> data] -> dst] -> data } - .apply_range(data_presence) # { [[dst -> data] -> dst] -> src } - .curry() - ) # { [[dst -> data] -> [dst -> src] } - if DUMP_ISL_IR: - logging.info(f"fills_to_matches: {fills_to_matches}") - - # Calculates the distance of a fill to the nearest src satisfying the fill. - # { [dst -> data] -> [dist] } - fill_min_dist: isl.Map = fills_to_matches.apply_range(dist_fn).lexmin() - # Isolates the relevant minimal pairs. - # { [dst -> data] -> [dst -> src] :.dst -> src is minimized distance } - minimal_pairs: isl.Map = ( - fill_min_dist.apply_range( - # Note: Need to match fill -> min_dist with min_dist -> [fill -> match] as lexmin over - # fill and match will minimize distance over the tuple (src, dst, data), but that - # overconstrains the optimization as we want to minimize over distance (dst, data) - # only for all src. - fills_to_matches.range_map() - .apply_range(dist_fn) - .reverse() - ) - .range() - .unwrap() - ) - if DUMP_ISL_IR: - logging.info(f"minimal_pairs: {minimal_pairs}") - - # Isolates the multicast networks. - # { [data] -> [dst -> src] : dst -> src is minimized distance } - multicast_networks: isl.Map = minimal_pairs.curry().range().unwrap() - # Devolves to a single source if multiple sources per domain point. - multicast_networks = multicast_networks.uncurry().lexmin().curry() - - return multicast_networks - - -def calculate_extents_per_dim(mcns: isl.Map) -> list[isl.PwAff]: - """ - Parameters - ---------- - mcns: - Mesh cast-networks, or networks in which all dsts per data are grouped with - the closest src containing the data. - - Returns - ------- - A list of `isl.PwAff` that gives the max extent (length) along dim_i per mcn, - where i is the i-th `isl.PwAff`. - - Preconditions - ------------- - `mcns` were generated with a Manhattan distance `dst_fn` by `identify_mesh_casts` - s.t. all dimensions are orthogonal to each other in a metric space, where each - unit movement in a dimension counts as 1 hop. - - We also assume `dst_fn` is translationally invariant (i.e., ∀src, dst, - src', dst' ∈ space, if |src - dst| = |src' - dst'|, - dst_fn(src, dst) = dst_fn(src', dst'). - """ - # Makes mcns from { [data] -> [dst -> src] } to { [data -> src] -> [dst] } - potential_srcs: isl.Map = mcns.range_reverse().uncurry() - # Sources are part of the extents, so we union it with the destinations. - # { [data -> src] -> [src] } - srcs: isl.Map = potential_srcs.domain().unwrap().range_map() - # { [data -> src] -> [spacetime] } - casting_extents: isl.Map = srcs.union(potential_srcs) - - # Projects away all dimensions but one to find their extent for hypercube. - dims: int = potential_srcs.range_tuple_dim() - # Creates a mask of what to project out. - project_out_mask: list[bool] = [True] * dims - dim_extents: list[isl.PwAff] = [None] * dims - - # Gets the extents of all dimensions - for noc_dim in range(dims): - # Project out all the dimensions of the output besides noc_dim. - project_out_mask[noc_dim] = False - # { [spacetime] -> [dimension] } - extent_mapper: isl.Map = dim_projector_mask( - casting_extents.range().get_space(), project_out_mask - ).reverse() - dim_extent_space: isl.Map = casting_extents.apply_range(extent_mapper) - project_out_mask[noc_dim] = True - - # Finds max(noc_dim) - min(noc_dim) for each [data -> src] - max_extent: isl.PwAff = dim_extent_space.dim_max(0) - min_extent: isl.PwAff = dim_extent_space.dim_min(0) - - # Subtracts the max from the min to get the extent per [data -> src] - dim_extents[noc_dim] = max_extent.sub(min_extent).coalesce() - - return dim_extents - - -class HypercubeMulticastModel(TransferModel): - """ - Does distributed multicasting a mesh using worst-case multicasting - behavior by assuming all multicasts are broadcasting to the convex - hypercube that encapsulates all their destinations and sources. - """ - - def __init__(self, dist_fn: isl.Map): - """ - Initializes the HypercubeMulticastModel with the distance function - over the metric space. - - Because we are using calculate_extents_per_dim(mcns), we inherit the - following requirements: - `dst_fn` holds all dimensions are orthogonal to each other in a metric space, - where each unit movement in a dimension counts as 1 hop. - - We also assume `dst_fn` is translationally invariant (i.e., ∀src, dst, - src', dst' ∈ space, if |src - dst| = |src' - dst'|, - dst_fn(src, dst) = dst_fn(src', dst'). + dist_fn: + A distance function { [dst -> src] -> [hops] } that accepts two + points in space, corresponding to `dst` and `src`, and returns + the distance between them in `hops`, a quantized atomic distance + of data transmission cost. Stored and passed straight through to + `identify_mesh_casts` on every `apply` call. + + Caller contract (inherited from `identify_mesh_casts`): the + tuple names of `fills`/`occs`'s domains (the spacetime/node + tuple, e.g. `noc[x, y]`) must match the corresponding tuple + names in `dist_fn`'s domain -- ISL raises on a name mismatch + when `dist_fn` is applied. `dist_fn`'s range tuple must be named + `hops` (`_fabric_crossing`, used by `FullyConnectedMulticastModel` + and `StarMulticastModel`, filters on `{ hops[h] : h >= 1 }` to + distinguish self-deliveries from fabric-crossing ones). """ self.dist_fn = dist_fn @@ -429,8 +102,9 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Parameters ---------- buff: - The buffer whose spatial analysis is being considered. Currently, - we rely on `dist_fn` to deal with this rather than `buff`. + The buffer whose spatial analysis is being considered. Not used + by any current subclass (they all rely on `dist_fn` instead); + kept for interface symmetry with `TransferModel.apply`. fills: The fill of `buffer` across time from parents. occs: @@ -438,33 +112,88 @@ def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo Returns ------- - A TransferInfo whose `fulfilled_fill`/`unfulfilled_fill` partition - `fills` by whether `identify_mesh_casts` found a matched source for that - (dst, data) pair -- a fill with no source holding its datum is - `unfulfilled_fill`, not silently treated as fulfilled. `edge_pressure` - is left at its default (`None`): the hypercube model costs a convex - bounding box, which has no notion of individual links to report - pressure on. + A `TransferInfo` whose `fulfilled_fill`/`unfulfilled_fill` partition + `fills` by whether `identify_mesh_casts` found a matched source for + that (dst, data) pair -- a fill with no source holding its datum is + `unfulfilled_fill`, never silently treated as fulfilled. `hops` and + `edge_pressure` both come from `_transfer_cost(mcs)`: if it returns + an `EdgePressure` `p`, `edge_pressure=p` and + `hops=_const_pwq(p.total())`; otherwise `edge_pressure=None` and + `hops` is the returned polynomial directly (the topology has no + per-link decomposition to report). """ + # `identify_mesh_casts` is called exactly once; every output below is + # derived from this single `mcs`, so they cannot fall out of sync. mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - result: isl.PwQPolynomial = self._cost_mesh_cast_hypercube(mcs) + cost: isl.PwQPolynomial | EdgePressure = self._transfer_cost(mcs) + edge_pressure: Optional[EdgePressure] + hops: isl.PwQPolynomial + if isinstance(cost, EdgePressure): + edge_pressure = cost + hops = _const_pwq(cost.total()) + else: + edge_pressure = None + hops = cost # { dst -> data } fills actually covered by a matched source. covered: isl.Map = _covered_fills(mcs) - # TODO: Read once from all buffers, assert that - # card(mcs) == tensor_size * duplication factor - n_meshcasts: int = mcs.card() return TransferInfo( fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), parent_reads=Reads(occs.tags, mcs), unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=result, + hops=hops, link_transfer=True, - # No per-link decomposition defined for the hypercube abstraction. - edge_pressure=None, + edge_pressure=edge_pressure, ) - def _cost_mesh_cast_hypercube(self, mcns: isl.Map) -> int: + @abstractmethod + def _transfer_cost(self, mcs: isl.Map) -> isl.PwQPolynomial | EdgePressure: + """ + Compute this topology's transfer cost from a fixed multicast-network map. + + Parameters + ---------- + mcs: + Multicast networks { [data] -> [dst -> src] } from + `identify_mesh_casts`, computed once by `apply` and shared with + every other output `apply` derives. + + Returns + ------- + Either a bare `isl.PwQPolynomial` (a topology with no per-link + decomposition -- `apply` sets `hops` to exactly this polynomial and + leaves `edge_pressure` at `None`), or an `EdgePressure` (a topology + with one -- `apply` sets `hops` to `_const_pwq(edge_pressure.total())`, + structurally keeping the scalar and the per-edge view consistent). + """ + raise NotImplementedError + + +class HypercubeMulticastModel(MulticastModel): + """ + Does distributed multicasting a mesh using worst-case multicasting + behavior by assuming all multicasts are broadcasting to the convex + hypercube that encapsulates all their destinations and sources. + + `edge_pressure` is always `None` on the resulting `TransferInfo`: the + hypercube model costs a convex bounding box, which has no notion of + individual links to report pressure on. + + Preconditions + ------------- + Because the cost kernel uses `calculate_extents_per_dim(mcns)`, `dist_fn` + must hold all dimensions orthogonal to each other in a metric space, + where each unit movement in a dimension counts as 1 hop, and must be + translationally invariant (i.e., ∀src, dst, src', dst' ∈ space, if + |src - dst| = |src' - dst'|, dist_fn(src, dst) = dist_fn(src', dst')). + """ + + def _transfer_cost(self, mcs: isl.Map) -> isl.PwQPolynomial: + # TODO: Read once from all buffers, assert that + # card(mcs) == tensor_size * duplication factor + return self._cost_mesh_cast_hypercube(mcs) + + def _cost_mesh_cast_hypercube(self, mcns: isl.Map) -> isl.PwQPolynomial: """ Given a multicast_network, calculate the hypercube. @@ -515,7 +244,7 @@ def _cost_mesh_cast_hypercube(self, mcns: isl.Map) -> int: return hypercube_costs.sum() -class FullyConnectedMulticastModel(TransferModel): +class FullyConnectedMulticastModel(MulticastModel): """ Multicast cost model for a fully-connected fabric (e.g. an NVSwitch-style all-to-all interconnect). @@ -527,59 +256,14 @@ class FullyConnectedMulticastModel(TransferModel): This is distance-independent in magnitude (one hop per crossing); `dist_fn` is used only to tell self-deliveries (0 hops) apart from fabric-crossing ones. - """ - def __init__(self, dist_fn: isl.Map): - """ - Initializes the model with the distance function over the metric space. - - Parameters - ---------- - dist_fn: - A distance function { [dst -> src] -> [hops] }. Only used to classify a - delivery as self (0 hops) vs. fabric-crossing (>= 1 hop); the hop - magnitude does not enter the cost. - """ - self.dist_fn = dist_fn - - def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - """ - Given a buffer, its fills across time, and its occupancies across time, - calculate the spatial transfers on a fully-connected fabric. - - Parameters - ---------- - buff: - The buffer whose spatial analysis is being considered. Unused; the - topology is captured entirely by `dist_fn`. - fills: - The fill of `buffer` across time from parents. - occs: - The occupancy of `buffer` across time. - - Returns - ------- - A TransferInfo whose `hops` is the number of fabric-crossing deliveries. - `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source. `edge_pressure` is left at its - default (`None`): this model treats the fabric as a contention-free - full mesh (one dedicated link per pair), so there is no per-link - pressure to report. - """ - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - result: isl.PwQPolynomial = self._cost_fully_connected(mcs) - # { dst -> data } fills actually covered by a matched source. - covered: isl.Map = _covered_fills(mcs) + `edge_pressure` is always `None` on the resulting `TransferInfo`: this + model treats the fabric as a contention-free full mesh (one dedicated + link per pair), so there is no per-link pressure to report. + """ - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=result, - link_transfer=True, - # No per-link decomposition defined for the full-mesh abstraction. - edge_pressure=None, - ) + def _transfer_cost(self, mcs: isl.Map) -> isl.PwQPolynomial: + return self._cost_fully_connected(mcs) def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: """ @@ -596,15 +280,83 @@ def _cost_fully_connected(self, mcns: isl.Map) -> isl.PwQPolynomial: piecewise quasi-polynomial (constant when there are no parameters). """ # [dst -> src] pairs that actually traverse the fabric (>= 1 hop). - crossing_hops: isl.Set = isl.Set.read_from_str( - isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" - ) - crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() - crossing: isl.Map = mcns.intersect_range(crossing_pairs) + crossing: isl.Map = _fabric_crossing(mcns, self.dist_fn) return crossing.wrap().card() -class XYRoutingMulticastModel(TransferModel): +# Directed-edge vocabulary + the four name-independent link-emitting maps for +# XY routing (parsed once at import). `_XY_EDGE_TYPES` is the single source +# of truth for the edge names: it feeds both the maps below and +# `XYRoutingMulticastModel.EDGE_TYPES`, so the class constant and the parsed +# maps can never name the edges differently. +_XY_EDGE_TYPES: tuple[str, str, str, str] = ( + "xedge_r", + "xedge_l", + "yedge_u", + "yedge_d", +) +_XEDGE_R, _XEDGE_L, _YEDGE_U, _YEDGE_D = _XY_EDGE_TYPES + +# { [[col[x] -> span[t]] -> ysv[ys]] -> yedge_u/d[x, t] : ... } -- re-key a +# column's spanned vertical links (see `_span_links`) onto the up/down edge +# tuples, splitting direction at the source row `ys`. +_YEDGE_U_MAP: isl.Map = isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + "{ [[col[x] -> span[t]] -> ysv[ys]] -> %s[x, t] : t >= ys }" % _YEDGE_U, +) +_YEDGE_D_MAP: isl.Map = isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + "{ [[col[x] -> span[t]] -> ysv[ys]] -> %s[x, t] : t < ys }" % _YEDGE_D, +) +# { [span[t] -> xy[xs, ys]] -> xedge_r/l[t, ys] : ... } -- re-key the source +# row's spanned horizontal links onto the right/left edge tuples, splitting +# direction at the source column `xs`. +_XEDGE_R_MAP: isl.Map = isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + "{ [span[t] -> xy[xs, ys]] -> %s[t, ys] : t >= xs }" % _XEDGE_R, +) +_XEDGE_L_MAP: isl.Map = isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, + "{ [span[t] -> xy[xs, ys]] -> %s[t, ys] : t < xs }" % _XEDGE_L, +) + + +def _span_links(coords: isl.Map) -> isl.Map: + """ + Build the set of link indices spanned by a per-key coordinate map. + + Parameters + ---------- + coords: + { key -> name[c] }, one point per (key, coordinate) pair to span -- + e.g. every destination-plus-source y within one tree's column, or + every destination-plus-source column along one tree's row. `name` is + read directly off `coords`'s range tuple, so this works for any + single-dimension range tuple. + + Returns + ------- + { key -> span[t] : min(key) <= t < max(key) } -- the contiguous run of + link indices between each key's minimum and maximum coordinate. Built as + an explicit lexmin/lexmax intersection (a `card()`-friendly link set) + rather than by summing a min/max polynomial, which trips a barvinok + `summate` assertion at scale (e.g. the 8x8 case). + """ + name: str = coords.get_tuple_name(isl.dim_type.out) + lo: isl.Map = coords.lexmin().apply_range( + isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, "{ %s[lo] -> span[t] : t >= lo }" % name + ) + ) + hi: isl.Map = coords.lexmax().apply_range( + isl.Map.read_from_str( + isl.DEFAULT_CONTEXT, "{ %s[hi] -> span[t] : t < hi }" % name + ) + ) + return lo.intersect(hi) + + +class XYRoutingMulticastModel(MulticastModel): """ Multicast cost model for XY (dimension-order) routing on a 2-D mesh. @@ -636,6 +388,12 @@ class XYRoutingMulticastModel(TransferModel): Source selection is per destination: `identify_mesh_casts` pairs each destination with its nearest source (devolving ties), and destinations that share a source form one tree; the cost sums over all such trees and all data. + `dist_fn` is expected to be Manhattan and translationally invariant, like the + hypercube model (see `HypercubeMulticastModel`'s Preconditions), since nearest- + source selection relies on the same distance shape. + + Its four directed link types are declared once as the class constant + `EDGE_TYPES = ("xedge_r", "xedge_l", "yedge_u", "yedge_d")`. Preconditions ------------- @@ -651,66 +409,10 @@ class XYRoutingMulticastModel(TransferModel): supported. """ - def __init__(self, dist_fn: isl.Map): - """ - Parameters - ---------- - dist_fn: - A distance function { [dst -> src] -> [hops] } used to pick each - destination's nearest source (Manhattan, like the hypercube model). - """ - self.dist_fn = dist_fn - - def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - """ - Given a buffer, its fills across time, and its occupancies across time, - calculate the XY-routing spatial transfers. + EDGE_TYPES = _XY_EDGE_TYPES - Parameters - ---------- - buff: - The buffer whose spatial analysis is being considered. Unused; the - topology is captured by `dist_fn` and the node coordinates. - fills: - The fill of `buffer` across time from parents. - occs: - The occupancy of `buffer` across time. - - Returns - ------- - A TransferInfo whose `hops` is the total XY-routing link count and whose - `edge_pressure` is the per-directed-mesh-link decomposition backing it - (`xedge_r`/`xedge_l`/`yedge_u`/`yedge_d` -- see `_directed_mesh_links`). - `hops` is `pressure.total()`, so the two can never disagree. - `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source. - - Preconditions - ------------- - The node tuple embedded in `fills`/`occs` must be exactly 2-D (XY - routing is only defined for a 2-D mesh; see `_directed_mesh_links`), - else raises `ValueError`. - """ - # Note: `identify_mesh_casts` is called exactly once; both `hops` and - # `edge_pressure` are derived from this single `mcs` so they cannot - # fall out of sync. - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - links: list[isl.Map] = self._directed_mesh_links(mcs) - pressure: EdgePressure = _edge_pressure_from_links(links) - # `hops` is `EdgePressure.total()` (sum of per-edge loads == total link - # count), wrapped as the constant `TransferInfo.hops` expects. - hops: isl.PwQPolynomial = _const_pwq(pressure.total()) - # { dst -> data } fills actually covered by a matched source. - covered: isl.Map = _covered_fills(mcs) - - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=hops, - link_transfer=True, - edge_pressure=pressure, - ) + def _transfer_cost(self, mcs: isl.Map) -> EdgePressure: + return _edge_pressure_from_links(self._directed_mesh_links(mcs)) def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: """ @@ -731,10 +433,11 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: Returns ------- - `[xedge_r, xedge_l, yedge_u, yedge_d]` maps. An edge `xedge_r[t, ys]` - is the rightward link in row `ys` between columns `t` and `t + 1`; - `yedge_u[x, t]` is the upward link in column `x` between rows `t` and - `t + 1` (and `_l` / `_d` the opposite directions). + `[xedge_r, xedge_l, yedge_u, yedge_d]` maps (see `EDGE_TYPES`). An + edge `xedge_r[t, ys]` is the rightward link in row `ys` between + columns `t` and `t + 1`; `yedge_u[x, t]` is the upward link in + column `x` between rows `t` and `t + 1` (and `_l` / `_d` the + opposite directions). Preconditions ------------- @@ -755,8 +458,7 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: ) # { [data -> src] -> dst name[x, y] } and a handle on the source per tree. - per_src: isl.Map = mcns.range_reverse().uncurry() - keymap: isl.Map = per_src.domain().unwrap().range_map() # [data->src] -> src + per_src, keymap = _per_src(mcns) # keymap: { [data->src] -> src } # --- Y phase: vertical links per (tree, destination column). --- # Key each destination's y by its column: { [data->src->col] -> yv[y] }. @@ -773,33 +475,18 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: ) src_row: isl.Map = dst_y.domain().unwrap().range_product(src_y).uncurry() col_ys: isl.Map = dst_y.union(src_row) - # Every link {ymin <= t < ymax} touched in a column, as a relation keyed by - # the column (card of an explicit link set, robust where a min/max sum is - # not). { [data->src->col] -> p[t] }. - ylinks: isl.Map = col_ys.lexmin().apply_range( - isl.Map.read_from_str(ctx, "{ yv[ymin] -> p[t] : t >= ymin }") - ).intersect( - col_ys.lexmax().apply_range( - isl.Map.read_from_str(ctx, "{ yv[ymax] -> p[t] : t < ymax }") - ) - ) + # Every link spanned in a column, keyed by the column (see + # `_span_links`): { [data->src->col] -> span[t] }. + ylinks: isl.Map = _span_links(col_ys) # Re-key links by the tree and carry ys so direction splits at the source: - # { [data->src] -> [[col[x] -> p[t]] -> ysv[ys]] }. + # { [data->src] -> [[col[x] -> span[t]] -> ysv[ys]] }. y_with_ys: isl.Map = ylinks.curry().range_product( keymap.apply_range( isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> ysv[ys] }" % name) ) ) - yedge_u: isl.Map = y_with_ys.apply_range( - isl.Map.read_from_str( - ctx, "{ [[col[x] -> p[t]] -> ysv[ys]] -> yedge_u[x, t] : t >= ys }" - ) - ) - yedge_d: isl.Map = y_with_ys.apply_range( - isl.Map.read_from_str( - ctx, "{ [[col[x] -> p[t]] -> ysv[ys]] -> yedge_d[x, t] : t < ys }" - ) - ) + yedge_u: isl.Map = y_with_ys.apply_range(_YEDGE_U_MAP) + yedge_d: isl.Map = y_with_ys.apply_range(_YEDGE_D_MAP) # --- X phase: horizontal links along the source row. --- # Columns spanned per tree = {src column} u {destination columns}. (Built @@ -812,35 +499,28 @@ def _directed_mesh_links(self, mcns: isl.Map) -> list[isl.Map]: isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> cx[xs] }" % name) ) ) - xlinks: isl.Map = col_x.lexmin().apply_range( - isl.Map.read_from_str(ctx, "{ cx[xmin] -> ex[t] : t >= xmin }") - ).intersect( - col_x.lexmax().apply_range( - isl.Map.read_from_str(ctx, "{ cx[xmax] -> ex[t] : t < xmax }") - ) - ) + # Every link spanned along the row, keyed by the tree (see + # `_span_links`): { [data->src] -> span[t] }. + xlinks: isl.Map = _span_links(col_x) # Carry the source (xs, ys) so direction splits at xs and the row ys is - # part of the edge identity: { [data->src] -> [ex[t] -> xy[xs, ys]] }. + # part of the edge identity: { [data->src] -> [span[t] -> xy[xs, ys]] }. x_with_src: isl.Map = xlinks.range_product( keymap.apply_range( isl.Map.read_from_str(ctx, "{ %s[xs, ys] -> xy[xs, ys] }" % name) ) ) - xedge_r: isl.Map = x_with_src.apply_range( - isl.Map.read_from_str( - ctx, "{ [ex[t] -> xy[xs, ys]] -> xedge_r[t, ys] : t >= xs }" - ) - ) - xedge_l: isl.Map = x_with_src.apply_range( - isl.Map.read_from_str( - ctx, "{ [ex[t] -> xy[xs, ys]] -> xedge_l[t, ys] : t < xs }" - ) - ) + xedge_r: isl.Map = x_with_src.apply_range(_XEDGE_R_MAP) + xedge_l: isl.Map = x_with_src.apply_range(_XEDGE_L_MAP) return [xedge_r, xedge_l, yedge_u, yedge_d] -class StarMulticastModel(TransferModel): +# Spoke vocabulary for the star model, declared once and shared with +# `StarMulticastModel.EDGE_TYPES`. +_STAR_EDGE_TYPES: tuple[str, str] = ("spoke_in", "spoke_out") + + +class StarMulticastModel(MulticastModel): """ Multicast cost model for a star / central-switch fabric -- the spokes realization of a fully-connected interconnect (e.g. an NVSwitch, where every @@ -851,68 +531,20 @@ class StarMulticastModel(TransferModel): up its egress spoke (multicast fan-out happens at the switch, so one copy per datum regardless of how many destinations want it), and every destination receives its datum down its ingress spoke. Self-deliveries (a node already - holding the datum) never cross the fabric and so load no spoke. + holding the datum) never cross the fabric and so load no spoke; `dist_fn`'s + hop magnitude otherwise never enters the spoke load -- on a star every + crossing is one switch hop each way, so `dist_fn` is used only to pick each + destination's nearest source and to classify self- vs. fabric-crossing + deliveries (`_fabric_crossing`). + + Its two spoke directions are declared once as the class constant + `EDGE_TYPES = ("spoke_in", "spoke_out")`. """ - def __init__(self, dist_fn: isl.Map): - """ - Parameters - ---------- - dist_fn: - A distance function { [dst -> src] -> [hops] }, used both to pick each - destination's nearest source and to tell self-deliveries (0 hops, no - spoke load) apart from fabric-crossing ones (>= 1 hop). The hop - magnitude does not enter the spoke load -- on a star every crossing is - one switch hop each way. - """ - self.dist_fn = dist_fn - - def apply(self, buff: MappingNode, fills: Fill, occs: Occupancy) -> TransferInfo: - """ - Given a buffer, its fills across time, and its occupancies across time, - calculate the spatial transfers on a star / central-switch fabric. - - Parameters - ---------- - buff: - The buffer whose spatial analysis is being considered. Unused; the - topology is captured entirely by `dist_fn`. - fills: - The fill of `buffer` across time from parents. - occs: - The occupancy of `buffer` across time. + EDGE_TYPES = _STAR_EDGE_TYPES - Returns - ------- - A TransferInfo whose `hops` is the total spoke traversals (injections plus - deliveries) and whose `edge_pressure` is the per-spoke decomposition - backing it (`spoke_in[n]`/`spoke_out[n]` -- see `_spoke_loads`). `hops` - is `pressure.total()`, so the two can never disagree. - `fulfilled_fill`/`unfulfilled_fill` partition `fills` by whether - `identify_mesh_casts` matched a source. - """ - # Note: `identify_mesh_casts` is called exactly once; both `hops` and - # `edge_pressure` are derived from this single `mcs`. - mcs: isl.Map = identify_mesh_casts(occs.map_, fills.map_, self.dist_fn) - ingress, egress = self._spoke_loads(mcs) - acc: isl.UnionPwQPolynomial = isl.UnionPwQPolynomial.from_pw_qpolynomial( - ingress - ).add(isl.UnionPwQPolynomial.from_pw_qpolynomial(egress)) - pressure: EdgePressure = EdgePressure(acc) - # `hops` is `EdgePressure.total()` (sum(ingress) + sum(egress)), wrapped - # as the constant `TransferInfo.hops` expects. - hops: isl.PwQPolynomial = _const_pwq(pressure.total()) - # { dst -> data } fills actually covered by a matched source. - covered: isl.Map = _covered_fills(mcs) - - return TransferInfo( - fulfilled_fill=Transfers(fills.tags, fills.map_.intersect(covered)), - parent_reads=Reads(occs.tags, mcs), - unfulfilled_fill=Fill(fills.tags, fills.map_.subtract(covered)), - hops=hops, - link_transfer=True, - edge_pressure=pressure, - ) + def _transfer_cost(self, mcs: isl.Map) -> EdgePressure: + return EdgePressure(_union_pwqs([*self._spoke_loads(mcs)])) def _spoke_loads( self, mcns: isl.Map @@ -938,11 +570,7 @@ def _spoke_loads( # Keep only deliveries that actually cross the fabric (>= 1 hop); a node # already holding its datum loads no spoke. - crossing_hops: isl.Set = isl.Set.read_from_str( - isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" - ) - crossing_pairs: isl.Set = self.dist_fn.intersect_range(crossing_hops).domain() - crossing: isl.Map = mcns.intersect_range(crossing_pairs) + crossing: isl.Map = _fabric_crossing(mcns, self.dist_fn) # { dst -> [src -> data] }: regroup so each delivery is keyed by destination. cur: isl.Map = crossing.reverse().curry() @@ -955,14 +583,17 @@ def _spoke_loads( # spoke. { spoke_in[n] -> #data } and { spoke_out[n] -> #data }. dims: int = ingress_nodes.dim(isl.dim_type.in_) idx: str = ", ".join(f"i{k}" for k in range(dims)) + spoke_in, spoke_out = self.EDGE_TYPES ingress: isl.PwQPolynomial = ingress_nodes.apply_domain( isl.Map.read_from_str( - isl.DEFAULT_CONTEXT, "{ %s[%s] -> spoke_in[%s] }" % (name, idx, idx) + isl.DEFAULT_CONTEXT, + "{ %s[%s] -> %s[%s] }" % (name, idx, spoke_in, idx), ) ).card() egress: isl.PwQPolynomial = egress_nodes.apply_domain( isl.Map.read_from_str( - isl.DEFAULT_CONTEXT, "{ %s[%s] -> spoke_out[%s] }" % (name, idx, idx) + isl.DEFAULT_CONTEXT, + "{ %s[%s] -> %s[%s] }" % (name, idx, spoke_out, idx), ) ).card() return ingress, egress diff --git a/accelforge/model/_looptree/reuse/isl/distributed/edge_pressure.py b/accelforge/model/_looptree/reuse/isl/distributed/edge_pressure.py new file mode 100644 index 00000000..629a7cfc --- /dev/null +++ b/accelforge/model/_looptree/reuse/isl/distributed/edge_pressure.py @@ -0,0 +1,251 @@ +""" +Per-edge memory pressure (link load) for distributed transfer models: +`EdgePressure` and the scalar-extraction / accumulation helpers that build and +consume it. + +Leaf module -- depends only on `islpy`, `functools`, `dataclasses`, and +`typing`, so every other module in this package (`mesh_casts.py`, +`distributed_buffers.py`, and `../spatial.py`) can import from here without +risk of a cycle. +""" + +import functools + +from dataclasses import dataclass + +from typing import Optional + +import islpy as isl + + +@dataclass(frozen=True) +class EdgePressure: + """ + Per-edge memory pressure (link load) for a spatial transfer. + + Where a `TransferModel`'s `hops` collapses a whole routing to a single + scalar, `EdgePressure` keeps the load broken out per physical edge: how many + multicast trees cross each individual link -- the quantity a per-link + bandwidth limit acts on, and what the symbolic network model calls + `max_traffic`. + + "Edge" is a directed physical link, identified by the tuple name and + coordinates of `load`'s domain, e.g. `yedge_u[x, t]` (the upward vertical + link in column `x` between rows `t` and `t + 1`, XY mesh) or `spoke_in[n]` / + `spoke_out[n]` (a node's ingress / egress link to the switch, star model). + + The load is keyed on the multicast tree `(data, src)`, not on individual + destinations: within one tree a link is traversed once regardless of how + many leaves hang off it, so `load` measures pressure (distinct flows over a + link) rather than summed hops -- e.g. for XY routing, sum over edges of load + == total hops. + + Preconditions + ------------- + `bottleneck` and `eval_edge` assume the load is piecewise constant (the + parameter-free regime the distributed models are validated in); there is no + clean ISL "max of a quasi-polynomial over its domain" primitive, so the + bottleneck is obtained by enumerating pieces. + """ + + # { edge -> number-of-trees } spanning every directed edge type the model + # emits. + load: isl.UnionPwQPolynomial + + @functools.cached_property + def _pieces(self) -> list[isl.PwQPolynomial]: + """ + Cache `load`'s decomposition into per-edge-type pieces. + + Returns + ------- + The list of `isl.PwQPolynomial` pieces `load` unions together, in + `foreach_pw_qpolynomial` enumeration order. + """ + # Note: `cached_property` works on this frozen dataclass only because + # it is not `slots=True` -- the cache is written straight into the + # instance `__dict__`, bypassing the frozen `__setattr__`. + pieces: list[isl.PwQPolynomial] = [] + self.load.foreach_pw_qpolynomial(pieces.append) + return pieces + + def total(self) -> int: + """ + Sum the load over every edge. + + For a single-direction (or single edge-type) pressure this is the total + traffic; for the full mesh pressure it equals the model's total `hops` + (sum over edges of load == total hops), which is the primary cross-check + that the per-edge decomposition is correct. + """ + total = 0 + for pwq in self._pieces: + # `.sum()` collapses the edge-indexed domain away (summing over every + # edge of this piece), leaving a 0-set-dim, parameter-free + # polynomial -- exactly `_eval_const`'s precondition. + total += _eval_const(pwq.sum()) + return total + + def bottleneck(self) -> int: + """ + Return the load on the single most-pressured edge. + + This is the bandwidth-binding quantity: with a uniform per-link + bandwidth, the most-congested edge saturates first, so its load sets the + transfer's bandwidth-bound latency. + + The per-edge load is generally not constant across an edge type (e.g. a + flooded column's upward link `yedge_u[x, t]` carries `t + 1` trees), so + the maximum is found by enumerating the (finite, parameter-free) edge + domain and evaluating the load at each edge -- sampling one point per + piece would under-report a monotone load. + """ + best = 0 + for pwq in self._pieces: + edges: list[isl.Point] = [] + pwq.domain().foreach_point(edges.append) + for edge in edges: + best = max(best, _eval_at(pwq, edge)) + return best + + def eval_edge(self, name: str, coords: list[int]) -> int: + """ + Look up the load on one named edge, e.g. `eval_edge("yedge_u", [0, 6])`. + + Returns 0 if no flow crosses that edge (it is outside the load's support). + + Parameters + ---------- + name: + The edge tuple name (`xedge_r`/`xedge_l`/`yedge_u`/`yedge_d` + for the mesh model, `spoke_in`/`spoke_out` for the spokes model). + coords: + The integer edge coordinates within that tuple. + """ + for pwq in self._pieces: + if pwq.domain().get_space().get_tuple_name(isl.dim_type.set) != name: + continue + point: isl.Point = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, + "{ %s[%s] }" % (name, ", ".join(str(c) for c in coords)), + ).sample_point() + return _eval_at(pwq, point) + return 0 + + +def _eval_at(pwq: isl.PwQPolynomial, point: isl.Point) -> int: + """ + Evaluate a piecewise quasi-polynomial at one point and return a Python `int`. + + Parameters + ---------- + pwq: + The polynomial to evaluate. + point: + The point to evaluate at. Must lie in a space compatible with `pwq`'s + domain (matching tuple name and dimensionality). + + Returns + ------- + `pwq`'s value at `point`, as a Python `int`. + + Note: isl.Val has no direct int() conversion in this islpy build (it + raises TypeError); round-tripping through str() is the working idiom + used throughout this module. + """ + return int(str(pwq.eval(point))) + + +def _eval_const(pwq: isl.PwQPolynomial) -> int: + """ + Evaluate a parameter-free, already-reduced piecewise quasi-polynomial to its + scalar value. + + Preconditions + ------------- + `pwq` has no free set dimensions and no parameters -- e.g. the output of + `.card()` on a parameter-free map/set, `.sum()` on a parameter-free + polynomial, or a constant built by `_const_pwq`. Evaluated only at the + space's zero point, so a `pwq` that still varies over domain points or + parameters gives a meaningless result; callers must reduce to a true + constant first (via `.sum()`/`.card()`). + + Returns + ------- + The polynomial's constant value as a Python `int`. + """ + return _eval_at(pwq, isl.Point.zero(pwq.domain().get_space())) + + +def _const_pwq(value: int) -> isl.PwQPolynomial: + """ + Build a 0-dimensional, parameter-free constant `isl.PwQPolynomial` equal to + `value`. + + Parameters + ---------- + value: + The integer the returned polynomial evaluates to everywhere (it has no + domain dimensions or parameters to vary over). + + Returns + ------- + An `isl.PwQPolynomial` over the empty (0-dim, 0-param) space, suitable + wherever a parameter-free `hops` cost is expected. Round-trips through + `_eval_const` back to `value`. + """ + ctx = isl.DEFAULT_CONTEXT + zero_dim: isl.Space = isl.Space.set_alloc(ctx, 0, 0) + return isl.PwQPolynomial.from_qpolynomial( + isl.QPolynomial.val_on_domain(zero_dim, isl.Val(value, ctx)) + ) + + +def _union_pwqs(pwqs: list[isl.PwQPolynomial]) -> isl.UnionPwQPolynomial: + """ + Accumulate a list of piecewise quasi-polynomials into one union. + + Parameters + ---------- + pwqs: + The polynomials to sum, e.g. one per directed edge type. May be + empty. + + Returns + ------- + The union-add of every `pwqs` entry (each promoted to an + `isl.UnionPwQPolynomial` first). An empty `pwqs` yields the empty union + `{ }`, so a caller with zero edge types (or a degenerate empty mesh) does + not need to special-case the accumulation. + """ + acc: Optional[isl.UnionPwQPolynomial] = None + for pwq in pwqs: + contribution = isl.UnionPwQPolynomial.from_pw_qpolynomial(pwq) + acc = contribution if acc is None else acc.add(contribution) + if acc is None: + acc = isl.UnionPwQPolynomial.read_from_str(isl.DEFAULT_CONTEXT, "{ }") + return acc + + +def _edge_pressure_from_links(edge_maps: list[isl.Map]) -> EdgePressure: + """ + Turn directed flow maps into an `EdgePressure`. + + Parameters + ---------- + edge_maps: + A list of { [data -> src] -> edge } maps, one per directed edge type, + each associating a multicast tree with every edge its route traverses. + + Returns + ------- + An `EdgePressure` whose `load` is { edge -> number-of-trees }: for each map + we reverse it and take the cardinality (`reverse().card()` counts, per + edge, how many distinct `(data, src)` trees cross it), then union the + per-type results into one `UnionPwQPolynomial` (via `_union_pwqs`). + """ + # { edge -> #trees crossing it }, one per edge type. + per_edge_loads: list[isl.PwQPolynomial] = [ + edge_map.reverse().card() for edge_map in edge_maps + ] + return EdgePressure(_union_pwqs(per_edge_loads)) diff --git a/accelforge/model/_looptree/reuse/isl/distributed/mesh_casts.py b/accelforge/model/_looptree/reuse/isl/distributed/mesh_casts.py new file mode 100644 index 00000000..c6318a7f --- /dev/null +++ b/accelforge/model/_looptree/reuse/isl/distributed/mesh_casts.py @@ -0,0 +1,268 @@ +""" +Multicast-network construction for distributed transfer models: matching each +requested datum to its nearest source (`identify_mesh_casts`) and the +shared-shape helpers built on top of its result (fill-partition recovery, +per-dimension extents, tree re-keying, and fabric-crossing filtering). + +Imports `islpy`, `isl_functions.dim_projector_mask`, and +`mapping_to_isl.DUMP_ISL_IR` only -- no dependency on `edge_pressure.py`, +`spatial.py`, or `distributed_buffers.py`, so this module cannot participate +in the import cycle those would otherwise risk. +""" + +import logging + +import islpy as isl + +from accelforge.model._looptree.reuse.isl.isl_functions import dim_projector_mask +from accelforge.model._looptree.reuse.isl.mapping_to_isl import DUMP_ISL_IR + +# [dst -> src] pairs whose distance is >= 1 hop, i.e. every fabric-crossing +# delivery (as opposed to a self-delivery, which never traverses a link). +# Parsed once at import rather than per `_fabric_crossing` call -- the string +# encodes `identify_mesh_casts`'s `dist_fn` contract (range tuple named +# `hops`), not anything instance- or call-specific. +_CROSSING_HOPS: isl.Set = isl.Set.read_from_str( + isl.DEFAULT_CONTEXT, "{ hops[h] : h >= 1 }" +) + + +def identify_mesh_casts( + src_occupancy: isl.Map, dst_fill: isl.Map, dist_fn: isl.Map +) -> isl.Map: + """ + Given srcs with data, fills to destinations, and a distance function, identify per data + the srcs delivering that data to dsts. + + Parameters + ---------- + src_occupancy: + An isl.Map of the form { [src] -> [data] } corresponding to the data held + at the buffer at space `src`. + dst_fill: + An isl.Map of the form { [dst] -> [data] } corresponding to the data requested + at the element at space `dst`. + dist_fn: + A distance function { [dst -> src] -> [hops] } that accepts two points in + space, corresponding to the `dst` and `src`, and returns the distance + between the two points in terms of `hops`, a quantized atomic distance of + data transmission cost. + + Caller contract: the tuple names of `dst_fill`'s and `src_occupancy`'s + domains (the spacetime/node tuple, e.g. `noc[x, y]`) must match the + corresponding tuple names in `dist_fn`'s domain -- ISL will raise on a + name mismatch when `dist_fn` is applied. `dist_fn`'s range tuple must be + named `hops` (consumers such as `FullyConnectedMulticastModel` and + `StarMulticastModel` filter on `{ hops[h] : h >= 1 }` to distinguish + self-deliveries from fabric-crossing ones). + + Returns + ------- + { [data] -> [dst -> src] } where { [dst] -> [data] } and { [src] -> [data] } are in + `src_occupancy` and `dst_fill` respectively, and where `[dst -> src]` is the infimum of + `dst_fn(src, dst), ∀ src, dst s.t. { [src] -> [data] } ∈ `src_occupancy` and + `{ [dst] -> [data] }` ∈ `dst_fill`. + """ + # Makes { [dst -> data] -> [dst -> data] } + fill_to_fill: isl.Map = dst_fill.wrap().identity() + if DUMP_ISL_IR: + logging.info(f"fill_to_fill: {fill_to_fill}") + + # Inverts src_occupancy s.t. data -> src. + # i.e. { [xs, ys] -> [d0, d1] } to { [d0, d1] -> [xs, ys] } + data_presence: isl.Map = src_occupancy.reverse() + + # { [dst -> data] -> [dst -> src] } where src contains data. + fills_to_matches: isl.Map = ( + fill_to_fill.uncurry() # { [[dst -> data] -> dst] -> data } + .apply_range(data_presence) # { [[dst -> data] -> dst] -> src } + .curry() + ) # { [[dst -> data] -> [dst -> src] } + if DUMP_ISL_IR: + logging.info(f"fills_to_matches: {fills_to_matches}") + + # Calculates the distance of a fill to the nearest src satisfying the fill. + # { [dst -> data] -> [dist] } + fill_min_dist: isl.Map = fills_to_matches.apply_range(dist_fn).lexmin() + # Isolates the relevant minimal pairs. + # { [dst -> data] -> [dst -> src] :.dst -> src is minimized distance } + minimal_pairs: isl.Map = ( + fill_min_dist.apply_range( + # Note: Need to match fill -> min_dist with min_dist -> [fill -> match] as lexmin over + # fill and match will minimize distance over the tuple (src, dst, data), but that + # overconstrains the optimization as we want to minimize over distance (dst, data) + # only for all src. + fills_to_matches.range_map() + .apply_range(dist_fn) + .reverse() + ) + .range() + .unwrap() + ) + if DUMP_ISL_IR: + logging.info(f"minimal_pairs: {minimal_pairs}") + + # Isolates the multicast networks. + # { [data] -> [dst -> src] : dst -> src is minimized distance } + multicast_networks: isl.Map = minimal_pairs.curry().range().unwrap() + # Devolves to a single source if multiple sources per domain point. + multicast_networks = multicast_networks.uncurry().lexmin().curry() + + return multicast_networks + + +def _covered_fills(mcs: isl.Map) -> isl.Map: + """ + Recover the { dst -> data } fills actually covered by a matched multicast + source, from `identify_mesh_casts`'s result. + + Parameters + ---------- + mcs: + { [data] -> [dst -> src] }, the multicast-network map returned by + `identify_mesh_casts`. + + Returns + ------- + { dst -> data } -- every (destination, datum) pair that has a matched + source in `mcs`. A model's fill partition is then + `fulfilled = fills.map_.intersect(covered)` and + `unfulfilled = fills.map_.subtract(covered)`. + """ + return ( + mcs.range_reverse() # { data -> [src -> dst] } + .uncurry() # { [data -> src] -> dst } + .domain_factor_domain() # Drops src, keeps -> dst. { data -> dst } + .reverse() # { dst -> data } + ) + + +def _mesh_node_tuple(mcns: isl.Map) -> tuple[str, int]: + """ + Read the spacetime/node tuple's name and dimensionality off a multicast + network map, instead of assuming a hardcoded name. + + Parameters + ---------- + mcns: + { [data] -> [dst -> src] }, the multicast-network map returned by + `identify_mesh_casts`. `dst` and `src` share the same node tuple, so + it suffices to read the name/dims off one side (`dst`, via + `.range().unwrap()`'s domain). + + Returns + ------- + `(name, dims)`: the node tuple's ISL tuple name (e.g. "noc" or "pe") and + its dimensionality. + """ + # Note: correct even when mcns is empty -- ISL preserves space/tuple-name + # information on empty relations, so this does not require data to be + # present. + node_space: isl.Map = mcns.range().unwrap() + name: str = node_space.get_tuple_name(isl.dim_type.in_) + dims: int = node_space.dim(isl.dim_type.in_) + return name, dims + + +def _per_src(mcns: isl.Map) -> tuple[isl.Map, isl.Map]: + """ + Re-key multicast networks by tree `(data, src)`, plus a source lookup. + + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. + + Returns + ------- + `(per_src, keymap)`: + - `per_src`: { [data -> src] -> dst } -- every destination grouped under + its tree's key. + - `keymap`: { [data -> src] -> src } -- the tree's own source point, + recovered from `per_src`'s (wrapped) domain. Used to inject/carry the + source's own coordinates (e.g. its row/column) into computations keyed + on the same tree. + """ + per_src: isl.Map = mcns.range_reverse().uncurry() # { [data -> src] -> dst } + keymap: isl.Map = per_src.domain().unwrap().range_map() # { [data -> src] -> src } + return per_src, keymap + + +def _fabric_crossing(mcns: isl.Map, dist_fn: isl.Map) -> isl.Map: + """ + Filter multicast networks down to fabric-crossing deliveries (>= 1 hop). + + Parameters + ---------- + mcns: + Multicast networks { [data] -> [dst -> src] } from `identify_mesh_casts`. + dist_fn: + The same distance function { [dst -> src] -> [hops] } passed to + `identify_mesh_casts` -- see its caller contract on the `hops` range + tuple name, which this filters on. + + Returns + ------- + { [data] -> [dst -> src] }, restricted to entries whose `[dst -> src]` + pair has `dist_fn(dst, src) >= 1`, i.e. every delivery that actually + crosses the fabric. Self-deliveries (0 hops) are dropped, since they load + no fabric link. + """ + crossing_pairs: isl.Set = dist_fn.intersect_range(_CROSSING_HOPS).domain() + return mcns.intersect_range(crossing_pairs) + + +def calculate_extents_per_dim(mcns: isl.Map) -> list[isl.PwAff]: + """ + Parameters + ---------- + mcns: + Mesh cast-networks, or networks in which all dsts per data are grouped with + the closest src containing the data. + + Returns + ------- + A list of `isl.PwAff` that gives the max extent (length) along dim_i per mcn, + where i is the i-th `isl.PwAff`. + + Preconditions + ------------- + `mcns` were generated with a Manhattan distance `dst_fn` by `identify_mesh_casts` + s.t. all dimensions are orthogonal to each other in a metric space, where each + unit movement in a dimension counts as 1 hop. + + We also assume `dst_fn` is translationally invariant (i.e., ∀src, dst, + src', dst' ∈ space, if |src - dst| = |src' - dst'|, + dst_fn(src, dst) = dst_fn(src', dst'). + """ + # Makes mcns from { [data] -> [dst -> src] } to { [data -> src] -> [dst] } + potential_srcs, srcs = _per_src(mcns) + # Sources are part of the extents, so we union it with the destinations. + # { [data -> src] -> [spacetime] } + casting_extents: isl.Map = srcs.union(potential_srcs) + + # Projects away all dimensions but one to find their extent for hypercube. + dims: int = potential_srcs.range_tuple_dim() + # Creates a mask of what to project out. + project_out_mask: list[bool] = [True] * dims + dim_extents: list[isl.PwAff] = [None] * dims + + # Gets the extents of all dimensions + for noc_dim in range(dims): + # Project out all the dimensions of the output besides noc_dim. + project_out_mask[noc_dim] = False + # { [spacetime] -> [dimension] } + extent_mapper: isl.Map = dim_projector_mask( + casting_extents.range().get_space(), project_out_mask + ).reverse() + dim_extent_space: isl.Map = casting_extents.apply_range(extent_mapper) + project_out_mask[noc_dim] = True + + # Finds max(noc_dim) - min(noc_dim) for each [data -> src] + max_extent: isl.PwAff = dim_extent_space.dim_max(0) + min_extent: isl.PwAff = dim_extent_space.dim_min(0) + + # Subtracts the max from the min to get the extent per [data -> src] + dim_extents[noc_dim] = max_extent.sub(min_extent).coalesce() + + return dim_extents diff --git a/accelforge/model/_looptree/reuse/isl/spatial.py b/accelforge/model/_looptree/reuse/isl/spatial.py index a278c377..7c6ad5a5 100644 --- a/accelforge/model/_looptree/reuse/isl/spatial.py +++ b/accelforge/model/_looptree/reuse/isl/spatial.py @@ -4,11 +4,14 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional +from typing import Optional import islpy as isl from accelforge.frontend.mapping import MappingNode +from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import ( + EdgePressure, +) from accelforge.model._looptree.reuse.isl.isl_functions import ( insert_equal_dims_map, reorder_projector, @@ -22,14 +25,6 @@ TaggedMap, ) -if TYPE_CHECKING: - # Note: guarded to avoid a runtime import cycle -- distributed_buffers.py - # imports TransferInfo from this module, so importing EdgePressure back - # here at runtime would form a loop. - from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( - EdgePressure, - ) - class Transfers(TaggedMap): """Transfers between regions in spacetime.""" @@ -68,7 +63,7 @@ class TransferInfo: # so this addition does not disturb any existing positional or keyword # construction site (`grep`-verified: every `TransferInfo(...)` call in the # tree already uses keyword arguments). - edge_pressure: Optional["EdgePressure"] = None + edge_pressure: Optional[EdgePressure] = None """Per-directed-edge load backing `hops`, for models that define one. Populated by `XYRoutingMulticastModel` (mesh links) and diff --git a/notebooks/astrasim2_correlation/distributed_models_demo.ipynb b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb new file mode 100644 index 00000000..e2165373 --- /dev/null +++ b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb @@ -0,0 +1,920 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c733530b", + "metadata": {}, + "source": [ + "# Distributed Transfer Models: Fully-Connected, Star, and XY Routing --- Demo & Review Guide\n", + "\n", + "This notebook demonstrates the distributed ISL transfer models in\n", + "`accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py`:\n", + "`FullyConnectedMulticastModel`, `StarMulticastModel`, `XYRoutingMulticastModel`,\n", + "and the `EdgePressure` per-link load abstraction they share with the older\n", + "`HypercubeMulticastModel`. It is a companion to\n", + "`notebooks/astrasim2_correlation/correlation.ipynb` (same conventions: one-hot\n", + "GPU coordinates, `isl.Map.read_from_str`, `model.apply(0, Fill(...), Occupancy(...))`).\n", + "\n", + "## What's new\n", + "\n", + "- **`FullyConnectedMulticastModel`** --- 1 hop per fabric-crossing delivery,\n", + " regardless of distance (the \"flat full-mesh\" view of a fully-connected fabric).\n", + "- **`StarMulticastModel`** --- the *spokes* realization of the same fabric\n", + " (`src -> switch -> dst`); exposes per-spoke `EdgePressure` where\n", + " `FullyConnectedMulticastModel` sees only a flat crossing count.\n", + "- **`XYRoutingMulticastModel`** --- dimension-order routing on a 2-D mesh, with\n", + " a full per-directed-mesh-link `EdgePressure` decomposition.\n", + "- **`EdgePressure`** --- `{ edge -> #multicast-trees-crossing-it }`, with\n", + " `.total()`, `.bottleneck()`, and `.eval_edge(name, coords)`, threaded through\n", + " `TransferInfo.edge_pressure` (`accelforge/model/_looptree/reuse/isl/spatial.py`).\n", + "- The `fulfilled_fill` / `unfulfilled_fill` partition (`_covered_fills`) and the\n", + " generic node-tuple lookup (`_mesh_node_tuple`) that every model above now\n", + " shares, replacing ad hoc per-model logic.\n", + "\n", + "## The two commits under review\n", + "\n", + "The code this notebook exercises arrived across two commits on `rengz-correl`:\n", + "\n", + "- **`142722f5`** --- *\"LLM implemented code from pseudocode but it needs\n", + " verification\"*: introduces `EdgePressure`, `StarMulticastModel`, and the\n", + " directed-mesh-link decomposition inside\n", + " `XYRoutingMulticastModel._directed_mesh_links`.\n", + "- **`81be62a4`** --- *\"isl distribuffers refactor\"*: consolidates the\n", + " `_eval_const` / `_const_pwq` / `_covered_fills` / `_mesh_node_tuple` helpers,\n", + " fixes a `fulfilled_fill`/`unfulfilled_fill` bug (every fill used to be\n", + " reported fulfilled unconditionally, even when no source held its datum), and\n", + " threads `EdgePressure` through `TransferInfo.edge_pressure` so `hops` and the\n", + " per-link decomposition come from one `identify_mesh_casts` call instead of\n", + " two independently computed paths.\n", + "\n", + "`HypercubeMulticastModel`, `identify_mesh_casts`, and `calculate_extents_per_dim`\n", + "predate both commits (they are the pre-existing, already-reviewed baseline) and\n", + "are not exercised here for that reason.\n", + "\n", + "## How to use this notebook as a review guide\n", + "\n", + "Each numbered section states which functions it exercises and cites their\n", + "current `file:line` ranges, recomputed against this working tree. Two passes\n", + "followed the two commits above: a documentation/comment style pass (comments\n", + "and docstrings only, verified AST-equivalent to `HEAD` with docstrings\n", + "stripped), and then a structural `/simplify` module split that moved\n", + "`EdgePressure` and its scalar-extraction helpers into `edge_pressure.py` and\n", + "`identify_mesh_casts` and its helpers into `mesh_casts.py`, leaving\n", + "`distributed_buffers.py` with a new `MulticastModel` base class (the shared\n", + "`__init__`/`apply` shape every model below now uses) and each model's own\n", + "cost kernel. Both passes are behavior-preserving -- verified against the\n", + "unmodified test suite and the asserted numbers below -- so the citations\n", + "reflect current file locations and line numbers, not the original commit\n", + "diffs. Every number asserted in a code cell below (`56`, `64`, `448`, ...) was\n", + "independently re-verified against this exact code before being written here.\n", + "Section 6 collects the section -> function -> file:line-range -> commit\n", + "mapping into one table so a reviewer can walk the diff systematically.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e3d6abf5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:20.903056Z", + "iopub.status.busy": "2026-07-06T20:54:20.902882Z", + "iopub.status.idle": "2026-07-06T20:54:22.202475Z", + "shell.execute_reply": "2026-07-06T20:54:22.201741Z" + } + }, + "outputs": [], + "source": [ + "import islpy as isl\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import (\n", + " FullyConnectedMulticastModel,\n", + " StarMulticastModel,\n", + " XYRoutingMulticastModel,\n", + " _eval_const,\n", + ")\n", + "from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import (\n", + " Fill,\n", + " Occupancy,\n", + " SpatialTag,\n", + ")\n", + "\n", + "CTX = isl.DEFAULT_CONTEXT\n" + ] + }, + { + "cell_type": "markdown", + "id": "031ba8c0", + "metadata": {}, + "source": [ + "## 2. Fully-connected vs. star: 8-GPU one-hot all-to-all\n", + "\n", + "GPU $i$ sits at one-hot coordinate $e_i$ of an 8-dimensional `noc` space (the\n", + "same encoding as `correlation.ipynb`). Two workloads distinguish the two models:\n", + "\n", + "- **all-to-all (unicast)**: `data[s, d]` is a unique chunk from GPU $s$ to GPU\n", + " $d$ --- every delivery has exactly one source and one destination.\n", + "- **broadcast**: GPU $g$ holds one private chunk `data[g]`, requested by every\n", + " other GPU --- a genuine multicast (one producer, $N-1$ consumers, fan-out at\n", + " the switch).\n", + "\n", + "`FullyConnectedMulticastModel` costs every fabric-crossing delivery at 1 hop\n", + "regardless of the payload pattern, so **both** workloads give\n", + "$N(N-1) = 56$ hops. `StarMulticastModel` is the *spokes* realization of the\n", + "same fabric --- a delivery routes `src -> switch -> dst` --- and its per-spoke\n", + "`EdgePressure` **is** pattern-sensitive: only the broadcast pattern exposes the\n", + "switch's fan-out (egress load $1$, not $N-1$), because the all-to-all pattern\n", + "has no shared payload to fan out in the first place.\n", + "\n", + "**Code exercised**: `MulticastModel.apply` (the shared `apply` every model\n", + "below now uses; `distributed_buffers.py:97-147`) together with\n", + "`FullyConnectedMulticastModel._transfer_cost`/`_cost_fully_connected`\n", + "(`distributed_buffers.py:265-284`, commit `81be62a4`); `StarMulticastModel`\n", + "(`distributed_buffers.py:523-599`, commit `142722f5`; its `hops`/\n", + "`edge_pressure` aggregation is now the shared `MulticastModel.apply` above,\n", + "fed by `StarMulticastModel._transfer_cost`, `distributed_buffers.py:546-547`);\n", + "`EdgePressure` (`edge_pressure.py:21-133`, commit `142722f5`, moved out of\n", + "`distributed_buffers.py` by the `/simplify` module split).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cd3dd43b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:22.205911Z", + "iopub.status.busy": "2026-07-06T20:54:22.205521Z", + "iopub.status.idle": "2026-07-06T20:54:22.214578Z", + "shell.execute_reply": "2026-07-06T20:54:22.213790Z" + } + }, + "outputs": [], + "source": [ + "def onehot_constraints(prefix: str, n: int) -> str:\n", + " \"\"\"One-hot constraints over dims `{prefix}0..{prefix}{n-1}`. Reused from correlation.ipynb.\"\"\"\n", + " bounds = \" and \".join(f\"0 <= {prefix}{i} <= 1\" for i in range(n))\n", + " hot = \" + \".join(f\"{prefix}{i}\" for i in range(n)) + \" = 1\"\n", + " return f\"{bounds} and {hot}\"\n", + "\n", + "\n", + "def linear_id(prefix: str, n: int) -> str:\n", + " \"\"\"Affine recovery of the GPU id from a one-hot vector: id = sum i*g_i. Reused from correlation.ipynb.\"\"\"\n", + " return \" + \".join(f\"{i}*{prefix}{i}\" for i in range(1, n))\n", + "\n", + "\n", + "def onehot_dist_fn(n: int) -> isl.Map:\n", + " \"\"\"Unit-cost distance function on the one-hot `noc` space: 0 hops if same\n", + " GPU, 1 hop otherwise. Same shape as correlation.ipynb's `all_to_all_maps`.\n", + " \"\"\"\n", + " xd = \", \".join(f\"xd{i}\" for i in range(n))\n", + " xs = \", \".join(f\"xs{i}\" for i in range(n))\n", + " same = \" and \".join(f\"xd{i} = xs{i}\" for i in range(n))\n", + " diff = \" or \".join(f\"(xd{i} < xs{i}) or (xd{i} > xs{i})\" for i in range(n))\n", + " return isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ [noc[{xd}] -> noc[{xs}]] -> hops[0] : {same}; \"\n", + " f\" [noc[{xd}] -> noc[{xs}]] -> hops[1] : {diff} }}\",\n", + " )\n", + "\n", + "\n", + "def all_to_all_maps(n: int) -> tuple[isl.Map, isl.Map, isl.Map]:\n", + " \"\"\"Build (occupancy, fill, dist_fn) for the N-GPU unicast all-to-all:\n", + " data[s, d] is the unique chunk sent by GPU s to GPU d. Verbatim from\n", + " notebooks/astrasim2_correlation/correlation.ipynb.\n", + " \"\"\"\n", + " gs = \", \".join(f\"gs{i}\" for i in range(n))\n", + " gd = \", \".join(f\"gd{i}\" for i in range(n))\n", + " occ = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gs}] -> data[s, d] : {onehot_constraints('gs', n)} \"\n", + " f\"and s = {linear_id('gs', n)} and 0 <= d < {n} }}\",\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gd}] -> data[s, d] : {onehot_constraints('gd', n)} \"\n", + " f\"and d = {linear_id('gd', n)} and 0 <= s < {n} }}\",\n", + " )\n", + " return occ, fill, onehot_dist_fn(n)\n", + "\n", + "\n", + "def broadcast_maps(n: int) -> tuple[isl.Map, isl.Map, isl.Map]:\n", + " \"\"\"Build (occupancy, fill, dist_fn) for the N-GPU one-hot *broadcast*\n", + " pattern: GPU g holds one private chunk data[g], requested by every other\n", + " GPU (one producer, N-1 consumers, fan-out at the switch). Same one-hot\n", + " node encoding as `all_to_all_maps`, different fill/occ.\n", + " \"\"\"\n", + " gs = \", \".join(f\"gs{i}\" for i in range(n))\n", + " gd = \", \".join(f\"gd{i}\" for i in range(n))\n", + " occ = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gs}] -> data[d] : {onehot_constraints('gs', n)} \"\n", + " f\"and d = {linear_id('gs', n)} }}\",\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gd}] -> data[d] : {onehot_constraints('gd', n)} \"\n", + " f\"and 0 <= d < {n} and d != {linear_id('gd', n)} }}\",\n", + " )\n", + " return occ, fill, onehot_dist_fn(n)\n", + "\n", + "\n", + "N = 8\n", + "tags = [SpatialTag(i, 0) for i in range(N)]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "99287f8f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:22.216427Z", + "iopub.status.busy": "2026-07-06T20:54:22.216263Z", + "iopub.status.idle": "2026-07-06T20:54:23.258718Z", + "shell.execute_reply": "2026-07-06T20:54:23.257609Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FullyConnectedMulticastModel (all-to-all unicast): 56 hops\n" + ] + } + ], + "source": [ + "# All-to-all (unicast) leg: reproduces correlation.ipynb's FullyConnectedMulticastModel\n", + "# result (56 hops = 8x7 fabric-crossing chunks) -- pattern-invariant, since FC only\n", + "# counts (dst, data) deliveries that cross the fabric, not whether a payload is shared.\n", + "occ_map, fill_map, dist_fn = all_to_all_maps(N)\n", + "fc_info = FullyConnectedMulticastModel(dist_fn).apply(\n", + " 0, Fill(tags, fill_map), Occupancy(tags, occ_map)\n", + ")\n", + "fc_hops = _eval_const(fc_info.hops)\n", + "assert fc_hops == N * (N - 1) == 56, fc_hops\n", + "print(f\"FullyConnectedMulticastModel (all-to-all unicast): {fc_hops} hops\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "da1b83a5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:23.262837Z", + "iopub.status.busy": "2026-07-06T20:54:23.262627Z", + "iopub.status.idle": "2026-07-06T20:54:25.768785Z", + "shell.execute_reply": "2026-07-06T20:54:25.768287Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "StarMulticastModel (broadcast): 64 hops = 8 injections + 56 deliveries\n", + " spoke_in per node : 7 (every node receives N-1 = 7)\n", + " spoke_out per node: 1 (every node sources 1 -- fan-out happens at the switch)\n", + " bottleneck spoke : 7\n", + " invariant: sum(spoke_in) = 56 == FullyConnected crossings = 56\n" + ] + } + ], + "source": [ + "# Broadcast leg: each GPU's chunk is requested by every other GPU. This is what\n", + "# exposes StarMulticastModel's spoke asymmetry -- the switch fans a shared\n", + "# chunk out from a single egress -- which the unicast pattern above cannot show\n", + "# (there, every delivery is already a distinct chunk with no sharing to fan out).\n", + "def onehot_coords(g: int, n: int) -> list[int]:\n", + " \"\"\"One-hot coordinate vector for GPU `g` among `n` GPUs.\"\"\"\n", + " return [1 if i == g else 0 for i in range(n)]\n", + "\n", + "\n", + "occ_b, fill_b, dist_fn_b = broadcast_maps(N)\n", + "fill_bc = Fill(tags, fill_b)\n", + "occ_bc = Occupancy(tags, occ_b)\n", + "\n", + "fc_info_b = FullyConnectedMulticastModel(dist_fn_b).apply(0, fill_bc, occ_bc)\n", + "fc_hops_b = _eval_const(fc_info_b.hops)\n", + "assert fc_hops_b == 56, fc_hops_b\n", + "\n", + "star_info = StarMulticastModel(dist_fn_b).apply(0, fill_bc, occ_bc)\n", + "star_pressure = star_info.edge_pressure\n", + "star_hops = _eval_const(star_info.hops)\n", + "\n", + "spoke_in = [star_pressure.eval_edge(\"spoke_in\", onehot_coords(g, N)) for g in range(N)]\n", + "spoke_out = [star_pressure.eval_edge(\"spoke_out\", onehot_coords(g, N)) for g in range(N)]\n", + "\n", + "assert all(v == N - 1 for v in spoke_in), spoke_in\n", + "assert all(v == 1 for v in spoke_out), spoke_out\n", + "assert star_pressure.bottleneck() == N - 1 == 7\n", + "assert star_hops == N + N * (N - 1) == 64\n", + "\n", + "total_ingress = sum(spoke_in)\n", + "assert total_ingress == fc_hops_b == 56, (total_ingress, fc_hops_b)\n", + "\n", + "print(f\"StarMulticastModel (broadcast): {star_hops} hops = {N} injections + {N * (N - 1)} deliveries\")\n", + "print(f\" spoke_in per node : {spoke_in[0]} (every node receives N-1 = {N - 1})\")\n", + "print(f\" spoke_out per node: {spoke_out[0]} (every node sources 1 -- fan-out happens at the switch)\")\n", + "print(f\" bottleneck spoke : {star_pressure.bottleneck()}\")\n", + "print(f\" invariant: sum(spoke_in) = {total_ingress} == FullyConnected crossings = {fc_hops_b}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "4524391d", + "metadata": {}, + "source": [ + "## 3. XY (dimension-order) routing on a 2-D mesh\n", + "\n", + "XY routing routes every packet along $x$ first, then $y$, so a multicast from\n", + "one source is a rigid tree: an X segment along the source row out to every\n", + "destination column, then an independent Y segment down each column from the\n", + "source row. Two geometries from\n", + "`tests/not_working/distribuffers/xy_routing/test_cases.yaml` (already\n", + "oracle-verified there; re-verified here):\n", + "\n", + "- **Case B** --- source $(1,0)$ casting to $(0,2)$ and $(2,2)$: a single tree,\n", + " $6$ hops, every used link carries load $1$ (no sharing).\n", + "- **Case F** --- an $8 \\times 8$ *column flood*: datum $(d_0, d_1)$ lives at\n", + " node $(d_0, d_1)$ and is requested by every node in column $x = d_0$. Each\n", + " column's vertical links are shared by up to 7 overlapping trees: $448$ total\n", + " hops, bottleneck $7$, `yedge_u[0,6] = 7`, `yedge_d[0,1] = 6`.\n", + "\n", + "**Code exercised**: `MulticastModel.apply` (shared `apply`;\n", + "`distributed_buffers.py:97-147`) together with\n", + "`XYRoutingMulticastModel._transfer_cost`\n", + "(`distributed_buffers.py:414-415`, commit `81be62a4`); `_directed_mesh_links`\n", + "(`distributed_buffers.py:417-515`, commit `142722f5`, touched again in\n", + "`81be62a4`); `EdgePressure.bottleneck` / `EdgePressure.eval_edge`\n", + "(`edge_pressure.py:89-133`, commit `142722f5`, moved out of\n", + "`distributed_buffers.py` by the `/simplify` module split).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d6c4b869", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:25.770948Z", + "iopub.status.busy": "2026-07-06T20:54:25.770573Z", + "iopub.status.idle": "2026-07-06T20:54:25.791882Z", + "shell.execute_reply": "2026-07-06T20:54:25.791233Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Case B: hops = 6, every used edge load 1 (single tree, no sharing)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
edgeload
0xedge_r[1,0]1
1xedge_l[0,0]1
2yedge_u[0,0]1
3yedge_u[0,1]1
4yedge_u[2,0]1
5yedge_u[2,1]1
\n", + "
" + ], + "text/plain": [ + " edge load\n", + "0 xedge_r[1,0] 1\n", + "1 xedge_l[0,0] 1\n", + "2 yedge_u[0,0] 1\n", + "3 yedge_u[0,1] 1\n", + "4 yedge_u[2,0] 1\n", + "5 yedge_u[2,1] 1" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "manhattan_2d = isl.Map.read_from_str(CTX, \"\"\"\n", + "{\n", + " [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + (yd - ys)] :\n", + " xd >= xs and yd >= ys;\n", + " [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] :\n", + " xd < xs and yd < ys;\n", + " [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + (yd - ys)] :\n", + " xd < xs and yd >= ys;\n", + " [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + -(yd - ys)] :\n", + " xd >= xs and yd < ys\n", + "}\n", + "\"\"\")\n", + "xy_tags = [SpatialTag(0, 0), SpatialTag(1, 0)]\n", + "\n", + "# Case B: source (1,0) casting to (0,2) and (2,2).\n", + "occ_case_b = isl.Map.read_from_str(CTX, \"{ noc[x, y] -> data[c] : c = 0 and x = 1 and y = 0 }\")\n", + "fill_case_b = isl.Map.read_from_str(\n", + " CTX, \"{ noc[x, y] -> data[c] : c = 0 and ((x = 0 and y = 2) or (x = 2 and y = 2)) }\"\n", + ")\n", + "info_b = XYRoutingMulticastModel(manhattan_2d).apply(\n", + " 0, Fill(xy_tags, fill_case_b), Occupancy(xy_tags, occ_case_b)\n", + ")\n", + "pressure_b = info_b.edge_pressure\n", + "hops_b = _eval_const(info_b.hops)\n", + "assert hops_b == 6, hops_b\n", + "\n", + "# The 6 links this single tree crosses: X segment (row 0, split at source\n", + "# column xs=1) + Y segment down column 0 and column 2 (both split at ys=0,\n", + "# both fully upward since dst row 2 > source row 0).\n", + "edges_b = pd.DataFrame(\n", + " [\n", + " {\"edge\": \"xedge_r[1,0]\", \"load\": pressure_b.eval_edge(\"xedge_r\", [1, 0])},\n", + " {\"edge\": \"xedge_l[0,0]\", \"load\": pressure_b.eval_edge(\"xedge_l\", [0, 0])},\n", + " {\"edge\": \"yedge_u[0,0]\", \"load\": pressure_b.eval_edge(\"yedge_u\", [0, 0])},\n", + " {\"edge\": \"yedge_u[0,1]\", \"load\": pressure_b.eval_edge(\"yedge_u\", [0, 1])},\n", + " {\"edge\": \"yedge_u[2,0]\", \"load\": pressure_b.eval_edge(\"yedge_u\", [2, 0])},\n", + " {\"edge\": \"yedge_u[2,1]\", \"load\": pressure_b.eval_edge(\"yedge_u\", [2, 1])},\n", + " ]\n", + ")\n", + "assert (edges_b[\"load\"] == 1).all(), edges_b\n", + "assert pressure_b.bottleneck() == 1\n", + "print(f\"Case B: hops = {hops_b}, every used edge load 1 (single tree, no sharing)\")\n", + "edges_b\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8a0aa2e7", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:25.796321Z", + "iopub.status.busy": "2026-07-06T20:54:25.796094Z", + "iopub.status.idle": "2026-07-06T20:54:25.814246Z", + "shell.execute_reply": "2026-07-06T20:54:25.813660Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Case F: hops = 448, bottleneck = 7\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tyedge_u[0,t]yedge_d[0,t]
0017
1126
2235
3344
4453
5562
6671
\n", + "
" + ], + "text/plain": [ + " t yedge_u[0,t] yedge_d[0,t]\n", + "0 0 1 7\n", + "1 1 2 6\n", + "2 2 3 5\n", + "3 3 4 4\n", + "4 4 5 3\n", + "5 5 6 2\n", + "6 6 7 1" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Case F: 8x8 mesh, datum (d0,d1) held at node (d0,d1), requested by every\n", + "# node in column x=d0 -- a column flood.\n", + "occ_case_f = isl.Map.read_from_str(\n", + " CTX,\n", + " \"{ noc[xs, ys] -> data[d0, d1] : d0 = xs and d1 = ys and 0 <= xs < 8 and 0 <= ys < 8 }\",\n", + ")\n", + "fill_case_f = isl.Map.read_from_str(\n", + " CTX,\n", + " \"{ noc[xd, yd] -> data[d0, d1] : d0 = xd and 0 <= d1 < 8 and 0 <= xd < 8 and 0 <= yd < 8 }\",\n", + ")\n", + "info_f = XYRoutingMulticastModel(manhattan_2d).apply(\n", + " 0, Fill(xy_tags, fill_case_f), Occupancy(xy_tags, occ_case_f)\n", + ")\n", + "pressure_f = info_f.edge_pressure\n", + "hops_f = _eval_const(info_f.hops)\n", + "assert hops_f == 448, hops_f\n", + "assert pressure_f.bottleneck() == 7\n", + "assert pressure_f.eval_edge(\"yedge_u\", [0, 6]) == 7\n", + "assert pressure_f.eval_edge(\"yedge_d\", [0, 1]) == 6\n", + "\n", + "# Column 0's vertical links: yedge_u[0,t] carries data with d1 <= t (load t+1);\n", + "# yedge_d[0,t] carries data with d1 > t (load 7-t).\n", + "column0 = pd.DataFrame(\n", + " [\n", + " {\n", + " \"t\": t,\n", + " \"yedge_u[0,t]\": pressure_f.eval_edge(\"yedge_u\", [0, t]),\n", + " \"yedge_d[0,t]\": pressure_f.eval_edge(\"yedge_d\", [0, t]),\n", + " }\n", + " for t in range(7)\n", + " ]\n", + ")\n", + "print(f\"Case F: hops = {hops_f}, bottleneck = {pressure_f.bottleneck()}\")\n", + "column0\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4b1bc6a2", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:25.816247Z", + "iopub.status.busy": "2026-07-06T20:54:25.816074Z", + "iopub.status.idle": "2026-07-06T20:54:25.944569Z", + "shell.execute_reply": "2026-07-06T20:54:25.943840Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg8AAAGJCAYAAAANJND6AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbrVJREFUeJzt3XdYFNf7NvB76b0ooKAIKoqogFijqIi9t1iiSQSxxNh7iV81aoIl9l5ij0Zj7z2CvYuxoCJiDSpREbCgsuf9w5f5uS7oDuyyi9yf69pL9+zMmefMsrPPnjlzRiGEECAiIiLSkJG+AyAiIqLchckDERERycLkgYiIiGRh8kBERESyMHkgIiIiWZg8EBERkSxMHoiIiEgWJg9EREQkC5MHIiIikoXJA+lNaGgoPD099bJthUKBn3/+Oce3GxERAYVCgYiIiBzf9sdq1aqFWrVq6TuMPGv58uVQKBS4ffu2Tur/+eefoVAoPrtcRp/D7Hw+FAoFevfunaV1NXX69GmYmZnhzp07Ot1Odq1atQqlSpWCqakpHBwcAOj3c/fx+7pgwQIUKVIEqampsuvK1clDbGwsfvjhBxQrVgwWFhaws7NDYGAgZs6ciVevXuk7PDXpH+aMHgsWLNB3eES5yoMHD9CuXTs4ODjAzs4OLVq0wK1bt/Qdlprw8HBs2bJF32F8UUaOHIkOHTrAw8NDpfyvv/7CV199BQcHB+TPnx9BQUHYuXNnlrcTExODb775BoULF4aVlRVKlSqFcePG4eXLl59d99q1awgNDUXx4sWxePFiLFq0KMtx6EpoaCjevHmDhQsXyl7XRAfx5IidO3eibdu2MDc3R6dOnVC2bFm8efMGR48exZAhQ3DlyhWDfLMAYP78+bCxsVEpq1Klip6iIcp9UlJSEBwcjOfPn+Onn36Cqakppk+fjqCgIERFRSF//vz6DlESHh6ONm3aoGXLlirl33//Pb755huYm5vrJ7BPePXqFUxMDPPrISoqCgcOHMDx48dVymfPno2+ffuiSZMmmDhxIl6/fo3ly5ejadOm2LhxI1q3bi1rO/fu3UPlypVhb2+P3r17I1++fDhx4gTGjBmDc+fOYevWrZ9cPyIiAkqlEjNnzoSXl5fsduYECwsLhISEYNq0aejTp49GPVXpDPOv4zPi4uLwzTffwMPDA3///TdcXV2l13r16oWbN29mK9vUtTZt2sDJyUnfYWjkxYsXsLa21ncYRCrmzZuHmJgYnD59GpUqVQIANGrUCGXLlsXUqVMRHh6u1/iEEHj9+jUsLS0zXcbY2BjGxsY5GJXmLCws9B1CppYtW4YiRYrgq6++UimfPXs2KlWqhO3bt0tfgmFhYShUqBBWrFghO3lYtWoVEhMTcfToUZQpUwYA0L17dyiVSqxcuRLPnj2Do6Njpus/fvwYAKTTFYaqXbt2mDx5Mg4dOoTatWtrvF6uPG0xefJkpKSkYMmSJSqJQzovLy/069dPer5s2TLUrl0bLi4uMDc3R+nSpTF//ny19c6ePYsGDRrAyckJlpaWKFq0KMLCwlSWUSqVmDFjBsqUKQMLCwsUKFAAP/zwA549e6aVtsXGxiI2Nvazy6WfLz18+DB++OEH5M+fH3Z2dujUqVOGsezevRs1atSAtbU1bG1t0aRJE1y5ckVlmdDQUNjY2CA2NhaNGzeGra0tvv3220/GoUm9ALBlyxaULVsWFhYWKFu2LDZv3pxhfU+ePMH3338POzs7ODg4ICQkBBcvXoRCocDy5ctVlr127RratGmDfPnywcLCAhUrVsS2bds+s+cyd+HCBTRq1Ah2dnawsbFBnTp1cPLkSZVlnj59isGDB8PX1xc2Njaws7NDo0aNcPHiRbX67t+/j5YtW8La2houLi4YMGBAls4tfij9l4yvry8sLCzg7OyMhg0b4uzZs9Iy7969w/jx41G8eHGYm5vD09MTP/3002e3ndk5+IzGadSqVQtly5bFP//8g6CgIFhZWcHLywsbNmwAAERGRqJKlSqwtLSEt7c3Dhw4oFJn+im8mzdvIjQ0FA4ODrC3t0fnzp016hLesGEDKlWqJCUOAFCqVCnUqVMHf/311yfXLVu2LIKDg9XKlUolChUqhDZt2qiUafJ59/T0RNOmTbF3715UrFgRlpaWWLhwIRQKBV68eIEVK1ZIpyhDQ0MBZL6/d+/ejaCgINja2sLOzg6VKlXCmjVrpNePHDmCtm3bokiRIjA3N4e7uzsGDBig1VO1H58bz+779csvv8DIyAizZ8+WymbPno0yZcrAysoKjo6OqFixoko7M7NlyxbUrl1b7VdyUlISXFxcVMrTP8vpSZwQAsHBwXB2dpa+3AHgzZs38PX1RfHixfHixQupPgAoUKCAynZcXV1hZGQEMzOzTGP09PTEmDFjAADOzs6fHUPy+PFjdOnSBQUKFICFhQX8/f2xYsUKteVevHiBQYMGwd3dHebm5vD29saUKVPw8c2xU1NTMWDAADg7O8PW1hbNmzfH/fv3M9x2hQoVkC9fvs/2pKgRuVChQoVEsWLFNF6+UqVKIjQ0VEyfPl3Mnj1b1K9fXwAQc+bMkZZ59OiRcHR0FCVLlhS//fabWLx4sRg5cqTw8fFRqatr167CxMREdOvWTSxYsEAMGzZMWFtbi0qVKok3b958Mo4xY8YIAOL69esiISFBejx9+lRaxsPDQ3h4eHy2TcuWLRMAhK+vr6hRo4aYNWuW6NWrlzAyMhI1a9YUSqVSWnblypVCoVCIhg0bitmzZ4tJkyYJT09P4eDgIOLi4qTlQkJChLm5uShevLgICQkRCxYsECtXrsw0Bk3r3bt3rzAyMhJly5YV06ZNEyNHjhT29vaiTJkyKm1NS0sTVatWFcbGxqJ3795izpw5ol69esLf318AEMuWLZOWvXz5srC3txelS5cWkyZNEnPmzBE1a9YUCoVCbNq06bP7D4AYM2aMSn3W1tbC1dVVjB8/XkycOFEULVpUmJubi5MnT0rLnTlzRhQvXlwMHz5cLFy4UIwbN04UKlRI2NvbiwcPHkjLvXz5UpQsWVJYWFiIoUOHihkzZogKFSoIPz8/AUAcOnToszFmJDQ0VAAQjRo1EjNmzBBTpkwRLVq0ELNnz5aWCQkJEQBEmzZtxNy5c0WnTp0EANGyZUuVuoKCgkRQUJD0PP1v6sP3TgghDh06pBZzUFCQcHNzE+7u7mLIkCFi9uzZonTp0sLY2FisXbtWFCxYUPz8889ixowZ0v5JSkqS1k//LAQEBIjWrVuLefPmia5duwoAYujQoZ/cB2lpacLc3Fz8+OOPaq/973//EwBUtvWxcePGCSMjIxEfH69SHhkZKQCI9evXS2Waft49PDyEl5eXcHR0FMOHDxcLFiwQhw4dEqtWrRLm5uaiRo0aYtWqVWLVqlXi+PHjme7vZcuWCYVCIcqWLSt+/fVXMXfuXNG1a1fx/fffS8v06dNHNG7cWISHh4uFCxeKLl26CGNjY9GmTRuV9qTv488JCQlRO+Z8/PmQ834BEL169ZKejxw5UigUCrFo0SKpbNGiRdLf6MKFC8XMmTNFly5dRN++fT8Z6/379wUAMWvWLLXX2rdvL4yNjcWsWbNEXFyciI6OFj179hSWlpbSPhdCiFu3bgkbGxvRqlUrqWz48OFCoVCIyMhIqWz37t0CgGjevLm4cOGCuHv3rli7dq2ws7MT/fv3/2ScmzdvFq1atRIAxPz588WqVavExYsXhRDqn7uXL18KHx8fYWpqKgYMGCBmzZolatSoIQCIGTNmSMsplUpRu3ZtoVAoRNeuXcWcOXNEs2bNBAC1eL777jsBQHTs2FHMmTNHtG7dWjr2fPi+pqtbt66oUKHCJ9v0sVyXPDx//lwAEC1atNB4nZcvX6qVNWjQQCUB2bx5swAgzpw5k2k9R44cEQDE6tWrVcr37NmTYfnH0j+AHz8+/ODKTR4qVKigchCbPHmyACC2bt0qhBAiOTlZODg4iG7duqms//DhQ2Fvb69Snv6lM3z48M9uX0695cqVE66uriIxMVEq27dvn1rbN27cqPaBSUtLE7Vr11ZLHurUqSN8fX3F69evpTKlUimqVasmSpQo8dn4P/4QtWzZUpiZmYnY2Fip7N9//xW2traiZs2aUtnr169FWlqaSl1xcXHC3NxcjBs3TiqbMWOGACD++usvqezFixfCy8sry8nD33//LQBkeIBNTxajoqIEANG1a1eV1wcPHiwAiL///lsqy27yAECsWbNGKrt27ZoAIIyMjFQSrr1796q9f+mfhbCwMJVttWrVSuTPn/+T+yEhIUEAUNnf6ebOnSsAiGvXrmW6/vXr1wUAlYRLCCF69uwpbGxspOOFnM+7h4eHACD27Nmjtj1ra2sREhKiVv7x/k5MTBS2traiSpUq4tWrVyrLfvhjIKPj2YQJE4RCoRB37tyRynSRPGjyfn2YPAwaNEgYGRmJ5cuXqyzTokULUaZMmc/G9rEDBw4IAGL79u1qrz169EjUqVNH5djq5OSkkjikW7hwoQAg/vjjD3Hy5ElhbGycYUIwfvx4YWlpqVLnyJEjNYo1fZ8lJCSolH/8uUs/Vvzxxx9S2Zs3b0TVqlWFjY2NlAhv2bJFABC//PKLSn1t2rQRCoVC3Lx5Uwjxf8eAnj17qizXsWPHTJOH7t27C0tLS43alS7XnbZI70qytbXVeJ0Pzzs+f/4c//33H4KCgnDr1i08f/4cwP+dl9qxYwfevn2bYT3r16+Hvb096tWrh//++096VKhQATY2Njh06JBG8WzcuBH79++XHqtXr5Zeu337tqxLt7p37w5TU1Pp+Y8//ggTExPs2rULALB//34kJiaiQ4cOKjEbGxujSpUqGcb8448/fna7mtYbHx+PqKgohISEwN7eXlq/Xr16KF26tEqde/bsgampKbp16yaVGRkZoVevXirLPX36FH///TfatWuH5ORkadtPnjxBgwYNEBMTgwcPHmiw995LS0vDvn370LJlSxQrVkwqd3V1RceOHXH06FHp787c3BxGRkbSek+ePIGNjQ28vb1x/vx5ad1du3bB1dVVpQvcysoK3bt31ziuj23cuBEKhULqDv1Qeldt+vs+cOBAldcHDRoEAFodC2RjY4NvvvlGeu7t7Q0HBwf4+PioDABO/39GV0L06NFD5XmNGjXw5MkTaX9nJL17PqOBhunn6j/VhV+yZEmUK1cO69atk8rS0tKwYcMGNGvWTDpeyP28Fy1aFA0aNMh0u5+zf/9+JCcnY/jw4WpjDj7siv/wePbixQv8999/qFatGoQQuHDhQpa3rwlN3y8hBHr37o2ZM2fijz/+QEhIiMrrDg4OuH//Ps6cOSNr+0+ePAGADMcaWFlZwdvbGyEhIVi/fj2WLl0KV1dXtG7dGjdv3lRZtnv37mjQoAH69OmD77//HsWLF89wnIynpydq1qyJRYsWYePGjQgLC0N4eDjmzJkjK+5P2bVrFwoWLIgOHTpIZaampujbty9SUlIQGRkpLWdsbIy+ffuqrD9o0CAIIbB7925pOQBqy/Xv3z/TGBwdHfHq1SuNTkGly3UDJu3s7AAAycnJGq9z7NgxjBkzBidOnFDbOc+fP4e9vT2CgoLw9ddfY+zYsZg+fTpq1aqFli1bomPHjtJBKiYmBs+fP4eLi0uG2/nwHNqn1KxZU2sDJkuUKKHy3MbGBq6urlICEhMTAwCZDoRJ35/pTExMULhwYel5SkoKUlJSpOfGxsZwdnbWuN7067A/jhOA2hfunTt34OrqCisrK5XlPh6pfPPmTQghMGrUKIwaNSrD7T9+/BiFChXK8LWPJSQk4OXLl/D29lZ7zcfHB0qlEvfu3UOZMmWkMQfz5s1DXFwc0tLSpGU/HOF/584deHl5qZ2XzWgbmoqNjYWbmxvy5cuX6TJ37tyBkZGR2j4rWLAgHBwctHpdfOHChdXaZ29vD3d3d7UyABmOxSlSpIjK8/QvhWfPnqn9baZL//LMaAzH69evVZbJTPv27fHTTz/hwYMHKFSoECIiIvD48WO0b99eWkbu571o0aKf3ObnpI91Klu27CeXu3v3LkaPHo1t27ap7dP0H0O6oun7tXLlSqSkpGD+/PkqX4rphg0bhgMHDqBy5crw8vJC/fr10bFjRwQGBmoUh/joHD8AtG3bFiYmJti+fbtU1qJFC5QoUQIjR45USRYBYMmSJShevDhiYmJw/Phxtb+ZtWvXonv37rhx44Z0TGzdujWUSiWGDRuGDh06aOWqnjt37qBEiRLSj5J0Pj4+0uvp/7q5uan9cM5oOSMjIxQvXlxluU8de9L35xd9tYWdnR3c3Nxw+fJljZaPjY1FnTp1UKpUKUybNg3u7u4wMzPDrl27MH36dCiVSgDvd9qGDRtw8uRJbN++HXv37kVYWBimTp2KkydPwsbGBkqlEi4uLio9BR9ydnbWWju1Jb19q1atQsGCBdVe//hyrA9/WQPAlClTMHbsWOm5h4cHbt++LbtebUrf9uDBgzP9paerS6PCw8MxatQohIWFYfz48ciXLx+MjIzQv39/KS5DIOcg8Ll1PkyQPpTZlQKZlWd0wJezbLp8+fLB3Nwc8fHxaq+ll7m5uWW6PvA+eRgxYgTWr1+P/v3746+//oK9vT0aNmwoLSP38/65hEUb0tLSUK9ePTx9+hTDhg1DqVKlYG1tjQcPHiA0NFTnf4Oavl+BgYGIiorCnDlz0K5dO7WE18fHB9evX8eOHTuwZ88ebNy4EfPmzcPo0aNVjjcfS/+y/jhpunXrFvbs2aN2eX6+fPlQvXp1HDt2TK2uiIgIKQG9dOkSqlatqvL6vHnzEBAQoPJjCgCaN2+O5cuX48KFC6hbt26mseYmz549g5WVlay/4VyXPABA06ZNsWjRIpw4cULtDf/Y9u3bkZqaim3btqlkzZmdYvjqq6/w1Vdf4ddff8WaNWvw7bffYu3atejatSuKFy+OAwcOIDAwMEcOFJqIiYlRGTmekpKC+Ph4NG7cGACk7NPFxSVLf+idOnVC9erVpefp7da03vRJXNJ7Kj50/fp1tWUPHTqEly9fqvQ+fNzlmH5qwdTUVCsfXmdnZ1hZWanFA7y/osPIyEj6Nb1hwwYEBwdjyZIlKsslJiaq9CZ5eHjg8uXLEEKofClntA1NFS9eHHv37sXTp08z7X3w8PCAUqlETEyM9IsEAB49eoTExES1SXU+lP4rMjExUaXc0GbxMzIygq+vr8oVJulOnTqFYsWKffa0ZtGiRVG5cmWsW7cOvXv3xqZNm9CyZUuVUyHa+rxrmsilf6YuX76cafJ76dIl3LhxAytWrECnTp2k8v3792c5Pl3w8vLC5MmTUatWLTRs2BAHDx5Ue0+sra3Rvn17tG/fHm/evEHr1q3x66+/YsSIEZleKlqqVCkA7y/X/9CjR48AZJzovn37Fu/evVMpi4+PR58+fVC/fn2YmZlJP0Q+/Hw8evQow9Mj6ae1P64zqzw8PPDPP/9AqVSq/HC7du2a9Hr6vwcOHEBycrLKvsxoOaVSidjYWJXehk8de+Li4lSOF5rIdWMeAGDo0KGwtrZG165dpT+aD8XGxmLmzJkA/i9T/jAzfv78OZYtW6ayzrNnz9Sy53LlygH4v+7Rdu3aIS0tDePHj1fb5rt379QOulmh6aWa6RYtWqQyRmP+/Pl49+4dGjVqBABo0KAB7OzsEB4enuFYjoSEhE/WX6xYMdStW1d6pHcralqvq6srypUrhxUrVqh0qe7fvx9Xr15VWadBgwZ4+/YtFi9eLJUplUrMnTtXZTkXFxfUqlULCxcuzPDX5+fa9DFjY2PUr18fW7duVRlv8ujRI6xZswbVq1eXumSNjY3V/k7Wr1+vNsaicePG+Pfff6VLFwHg5cuX2Zq47Ouvv4YQIsNfZukxpSeNM2bMUHl92rRpAIAmTZpkWn/6l9fhw4elsrS0NIOcbK1NmzY4c+aMSgJx/fp1/P3332jbtq1GdbRv3x4nT57E0qVL8d9//6mcsgC093m3trbWaNn69evD1tYWEyZMkE6/pEt/fzM6ngkhpOOdIfHz88OuXbsQHR2NZs2aqYxDSR+7kM7MzAylS5eGECLTMWcAUKhQIbi7u6sljl5eXjAyMsK6detU9s39+/dx5MgRBAQEqCzfrVs3KJVKLFmyBIsWLYKJiQm6dOmism7JkiVx4cIF3LhxQ2XdP//8E0ZGRvDz89N8Z3xC48aN8fDhQ5XTKu/evcPs2bNhY2ODoKAgabm0tDS18RbTp0+HQqGQjvnp/86aNUtluY+PCR86f/48qlWrJivuXNnzULx4caxZswbt27eHj4+PygyTx48fx/r166VrqdMzy2bNmuGHH35ASkoKFi9eDBcXF5UvnhUrVmDevHlo1aoVihcvjuTkZCxevBh2dnbSATkoKAg//PADJkyYgKioKNSvXx+mpqaIiYnB+vXrMXPmTJUBcllRp04dANB40OSbN29Qp04dtGvXDtevX8e8efNQvXp1NG/eHMD70zzz58/H999/j/Lly+Obb76Bs7Mz7t69i507dyIwMDBLg3/k1DthwgQ0adIE1atXR1hYGJ4+fSpd4/3heIqWLVuicuXKGDRoEG7evIlSpUph27ZtePr0KQDVX3Bz585F9erV4evri27duqFYsWJ49OgRTpw4gfv372c478Kn/PLLL9i/fz+qV6+Onj17wsTEBAsXLkRqaiomT54sLde0aVOMGzcOnTt3RrVq1XDp0iWsXr1aZaAl8P7gNGfOHHTq1Annzp2Dq6srVq1apTaeA3jffRocHIwxY8Z88lrw4OBgfP/995g1axZiYmLQsGFDKJVKHDlyBMHBwejduzf8/f0REhKCRYsWITExEUFBQTh9+jRWrFiBli1bZji/QboyZcrgq6++wogRI6TejbVr12rtF5Y29ezZE4sXL0aTJk0wePBgmJqaYtq0aShQoIA0OPRz2rVrh8GDB2Pw4MHIly+fWi+Wtj7vFSpUwIEDBzBt2jS4ubmhaNGiGc4oa2dnh+nTp6Nr166oVKkSOnbsCEdHR1y8eBEvX77EihUrUKpUKRQvXhyDBw/GgwcPYGdnh40bN2ptnhlt++qrr7B161Y0btwYbdq0wZYtW2Bqaor69eujYMGCCAwMRIECBRAdHY05c+agSZMmn+01atGiBTZv3qzSq+fs7IywsDD8/vvvqFOnDlq3bo3k5GTMmzcPr169wogRI6T1ly1bhp07d2L58uXSKYnZs2fju+++w/z589GzZ08AwJAhQ6R5bHr37o38+fNjx44d2L17N7p27frZU2Oa6t69OxYuXIjQ0FCcO3cOnp6e2LBhA44dO4YZM2ZI+6NZs2YIDg7GyJEjcfv2bfj7+2Pfvn3YunUr+vfvLyX/5cqVQ4cOHTBv3jw8f/4c1apVw8GDB9V6cNOdO3cOT58+RYsWLeQFLuvaDANz48YN0a1bN+Hp6SnMzMyEra2tCAwMFLNnz1a5hG/btm3Cz89PWFhYCE9PTzFp0iSxdOlSlcukzp8/Lzp06CCKFCkizM3NhYuLi2jatKk4e/as2nYXLVokKlSoICwtLYWtra3w9fUVQ4cOFf/+++8n483s0p0Pyb1UMzIyUnTv3l04OjoKGxsb8e2334onT56oLX/o0CHRoEEDYW9vLywsLETx4sVFaGioSvtCQkKEtbX1Z7ctt14h3l+G6ePjI8zNzUXp0qXFpk2bMrxELCEhQXTs2FHY2toKe3t7ERoaKo4dOyYAiLVr16osGxsbKzp16iQKFiwoTE1NRaFChUTTpk3Fhg0bPhs3Mrhk6fz586JBgwbCxsZGWFlZieDgYLXLvF6/fi0GDRokXF1dhaWlpQgMDBQnTpxQu/xKCCHu3LkjmjdvLqysrISTk5Po16+fdJnfh5c9bt++XQAQCxYs+Gzc7969E7/99psoVaqUMDMzE87OzqJRo0bi3Llz0jJv374VY8eOFUWLFhWmpqbC3d1djBgxQuUzIYT6JWNCvN+ndevWFebm5qJAgQLip59+Evv378/wUs2MLrXz8PAQTZo0USvHR9f+Z/ZZyOxy0Yzcu3dPtGnTRtjZ2QkbGxvRtGlTERMT89n1PhQYGJjhpa0f0uTznlm7hXh/CWvNmjWlS/7SL9vMrK3btm0T1apVE5aWlsLOzk5UrlxZ/Pnnn9LrV69eFXXr1hU2NjbCyclJdOvWTVy8eDHTy2E/R86lmpq8Xx+/10IIsXXrVmFiYiLat28v0tLSxMKFC0XNmjVF/vz5pbllhgwZIp4/f/7ZeM+fPy8AiCNHjqiUv337VsyePVuUK1dO2NjYCBsbGxEcHKxyefK9e/eEvb29aNasmVq9rVq1EtbW1uLWrVtS2alTp0SjRo2kY0zJkiXFr7/+Kt6+ffvZODW9VFOI95eZdu7cWTg5OQkzMzPh6+ur8l6mS05OFgMGDBBubm7C1NRUlChRQvz2228ql/IKIcSrV69E3759Rf78+YW1tbVo1qyZuHfvXobHvWHDhokiRYqo1fE5CiE+MTKJDNby5cvRuXNnnDlzBhUrVtR3ODq1ZcsWtGrVCkePHtV4NHZuMnToUPz555+4efOmQd7ngMjQ1KlTB25ubli1apW+Q8nVUlNT4enpieHDh6vMyqyJXDnmgb5cH1+fn5aWhtmzZ8POzg7ly5fXU1S6dejQIYwaNYqJA5GGwsPDsW7dOoMbzJvbLFu2DKampmrzd2giV455oC9Xnz598OrVK1StWhWpqanYtGkTjh8/jvDwcIO5wkXb5E6UQ5TXValSBW/evNF3GLlejx49spQ4AEweyMDUrl0bU6dOxY4dO/D69Wt4eXlh9uzZ6N27t75DIyKi/49jHoiIiEgWjnkgIiIiWZg8EBERkSy5esyDUqnEv//+C1tb2yzN5U9ERJRXCSGQnJwMNzc3tRtzfU6uTh7+/fdftTv4ERERkebu3bundgOwz8nVyUP6tJ337t3L9Pa9REREpC4pKQnu7u6fnRI8I7k6eUg/VWFnZ8fkgYiIKAuyctqfAyaJiIhIFiYPREREJAuTByIiIpIlV495oLxBCIF3794hLS1N36EQ5SmmpqYwNjbWdxhkgJg8kEF78+YN4uPj8fLlS32HQpTnKBQKFC5cGDY2NvoOhQwMkwcyWEqlEnFxcTA2NoabmxvMzMw4GRhRDhFCICEhAffv30eJEiXYA0EqmDyQwXrz5g2USiXc3d1hZWWl73CI8hxnZ2fcvn0bb9++ZfJAKvQ6YNLT0xMKhULt0atXL32GRQZG7rSpRKQd7OmjzOi15+HMmTMqg+AuX76MevXqoW3btnqMioiIiD5Fr8mDs7OzyvOJEyeiePHiCAoK0lNERERE9DkGM+bhzZs3+OOPPzBw4MBMu8pSU1ORmpoqPU9KSgLwfmCdUqnMkTgp5yiVSgghpEdeZ2RkhE2bNqFly5b6DkWSfkrJ3t4ez54903r9wcHBiIyMBACcP38e5cqVy3TZgwcPok+fPrh06dIXcX5++fLlGDBggLRfFyxYgF27dmHbtm05FkP6Z4/H2C9Tdt5Tg0ketmzZgsTERISGhma6zIQJEzB27Fi18oSEBLx+/VqH0WnZmva6rb/jOt3Wn0Pevn0LpVKJd+/e4d27d1J5iVH7cjSOmPH1c3R7n5KWlqayLwzB77//joYNG6rEFRkZiSFDhuDq1atwd3fHiBEj0KlTp0zrWLlyJQYNGoSEhASV8nXr1uHWrVuoVq2a2t/Bx4YOHYrhw4dL84LkdukH9vS2dOrUCb/88gsiIiJQvXr1HInh3bt3UCqVePLkCUxNTXNkm7rUZcUZnda/JKSSTuvXtuTk5CyvazDJw5IlS9CoUSO4ubllusyIESMwcOBA6Xn6HcGcnZ1z142xkv7Rbf0uLrqtP4e8fv0aycnJMDExgYmJ/v5U9bntjxkbGxtUPACQL18+lc9tXFwcWrRogR9++AGrV6/GwYMH8cMPP6BQoUJo0KBBhnWk92B83DYXFxdpjo9P/R0cPXoUt27dQrt27Qxu/3zOmzdvYGZmplb+8T4xMTFBhw4dMG/ePNSqVStHYjMxMYGRkRHy588PCwuLHNmmLkU/0+0AUJdcduzNzntqEMPY79y5gwMHDqBr166fXM7c3Fy6g+aHd9I0MjLKXQ8odfvQd/u0+MjoapycllEMmT1WrVoFJycnvHnzRqW8VatW6NSpk/R827ZtqFChAiwtLVG8eHGMGzcOaWlp0us3b95EUFAQLC0tUaZMGRw4cEAtlhMnTiAgIACWlpaoVKkStm7dCiMjI1y8eFFa5sqVK2jcuDFsbW1RsGBBdOrUCU+ePNGoLUWLFsXMmTNVygICAjB27FiV9+Lj9RYuXIiiRYti2rRpKF26NPr06YM2bdpgxowZGW4nMjISYWFheP78ufS+f7iNzLbz4WPdunWoV68eLC0tpbLOnTujVatWKssNGDAAwcHB0vPg4GD06dMHffr0gYODA5ydnTF69Ghpe3PnzoWvr6+0fPo+XrhwoVRWr149jBo1CgqFArdu3ULLli1RsGBB2NraonLlyjh48KDafv3ll18QEhICe3t7/PDDD1AoFFixYgU8PDxgbW2N1q1b4+nTp2rtbt68ObZt24bXr1/L+rvM7kPfxwFtPZRQ6PSh7/Zl5ZFVBpE8LFu2DC4uLmjSpIm+QyHKlrZt2yItLU3lvPTjx4+xc+dOhIWFAQCOHDmCTp06oV+/frh69SoWLlyI5cuX49dffwXwvru6devWMDMzw6lTp7BgwQIMGzZMZTtJSUlo1qwZfH19cf78eYwfP15tmcTERNSuXRsBAQE4e/Ys9uzZg0ePHqFdu3Y63QcnTpxA3bp1VcoaNGiAEydOZLh8tWrVMGPGDNjZ2SE+Ph7x8fEYPHiwrG0eOXIEFStWzFK8K1asgImJCU6fPo2ZM2di2rRp+P333wEAQUFBuHr1qnQ6JTIyEk5OToiIiADw/tTaiRMnpJ6AlJQUNG7cGAcPHsSFCxfQsGFDNGvWDHfv3lXZ5pQpU+Dv748LFy5g1KhROHXqFLp06YLevXsjKioKwcHB+OWXX9RirVixIt69e4dTp05lqa1E2qL3/j2lUolly5YhJCQk13U3En3M0tISHTt2xLJly6RLjv/44w8UKVJE+oIZO3Yshg8fjpCQEABAsWLFMH78eAwdOhRjxozBgQMHcO3aNezdu1c6HRAeHo5GjRpJ21mzZg0UCgUWL14MCwsLlC5dGg8ePEC3bt2kZebMmYOAgACEh4dLZUuXLoW7uztu3LiBkiVL6mQfPHz4EAUKFFApK1CgAJKSkvDq1StYWlqqvGZmZgZ7e3soFAoULFgwS9u8c+fOJ095foq7uzumT58OhUIBb29vXLp0CdOnT0e3bt1QtmxZ5MuXD5GRkWjTpg0iIiIwaNAgzJw5EwBw+vRpvH37FtWqVQMA+Pv7w9/fX6p7/Pjx2Lx5M7Zt24bevXtL5bVr18agQYOk56NGjULDhg0xdOhQAEDJkiVx/Phx7NmzRyVWKysr2Nvb486dO1lqK5G26L3n4cCBA7h79670q4wot+vWrRv27duHBw8eAHg/aj40NFTqfr948SLGjRsHGxsb6dGtWzfpHh7R0dFwd3dX+TKsWrWqyjauX78OPz8/lXOWlStXVlnm4sWLOHTokMp2SpUqBQCIjY3VSdv15dWrV1k+f/vVV1+pnA6rWrUqYmJipNNINWvWREREBBITE3H16lX07NkTqampuHbtGiIjI1GpUiVpBtSUlBQMHjwYPj4+cHBwgI2NDaKjo9V6Hj7uJYmOjkaVKlVUyj5+z9NZWlryXi+kd3r/qV+/fn1ehkdflICAAPj7+2PlypWoX78+rly5gp07d0qvp6SkYOzYsWjdurXautoclJaSkoJmzZph0qRJaq+5urp+dn0jIyO1z+bbt28/u17BggXx6NEjlbJHjx7Bzs5OrddBW5ycnNQuFc1q/B+rVasWFi1ahCNHjiAgIAB2dnZSQhEZGakyL83gwYOxf/9+TJkyBV5eXrC0tESbNm3w5s0blTqtra1lx5Hu6dOnanPkEOU0vScPRF+irl27YsaMGXjw4AHq1q0Ld3d36bXy5cvj+vXr8PLyynBdHx8f3Lt3D/Hx8dKX/MmTJ1WW8fb2xh9//IHU1FSYm5sDeD9j64fKly+PjRs3wtPTM0unBJ2dnREfHy89T0pKQlxc3GfXq1q1Knbt2qVStn///kx/SQPvT11k55brAQEBuHr1qkqZs7MzLl++rFIWFRWldsnhx+MHTp48qXIjqKCgIPTv3x/r16+XTj3VqlULBw4cwLFjx1ROPxw7dgyhoaFo1aoVgPcJ3O3btz8bv4+PT4ZxfCw2NhavX79GQEDAZ+sk0iW9n7Yg+hJ17NgR9+/fx+LFi9VOyY0ePRorV67E2LFjceXKFURHR2Pt2rX43//+BwCoW7cuSpYsiZCQEFy8eBFHjhzByJEj1epXKpXo3r07oqOjsXfvXkyZMgXA/92PoFevXnj69Ck6dOiAM2fOIDY2Fnv37kXnzp01+qKuXbs2Vq1ahSNHjuDSpUsICQnRaPKlHj164NatWxg6dCiuXbuGefPm4a+//sKAAQMyXcfT0xMpKSk4ePAg/vvvP9nd8g0aNMDRo0fV4j979ixWrlyJmJgYjBkzRi2ZAIC7d+9i4MCBuH79Ov7880/Mnj0b/fr1k1738/ODo6Mj1qxZo5I8bNmyBampqQgMDJSWLVGiBDZt2oSoqChcvHhRep8+p2/fvtizZw+mTJmCmJgYzJkzR228A/B+YGixYsVQvHhxTXcNkU7I+jmiVCoRGRmJI0eO4M6dO3j58iWcnZ0REBCg9uuKSFduTzT8q3Ls7e3x9ddfY+fOnWozQjZo0AA7duzAuHHjMGnSJJiamqJUqVLSpcpGRkbYvHkzunTpgsqVK8PT0xOzZs1Cw4YNpTrs7Oywfft2/PjjjyhXrhx8fX0xevRodOzYUTr14ebmhmPHjmHYsGGoX78+UlNT4eHhgYYNG2p0idaIESMQFxeHpk2bwt7eHuPHj9eo56Fo0aLYuXMnBgwYgJkzZ6Jw4cL4/fffVeZ4WL58OTp37iydVqhWrRp69OiB9u3b48mTJxgzZgx+/vnnz24r3bfffouhQ4fi+vXr8Pb2lvbzqFGjMHToULx+/RphYWHo1KkTLl26pLJup06d8OrVK1SuXBnGxsbo168funfvLr2uUChQo0YN7Ny5U5qcyc/PD3Z2dvD29lY5BTFt2jSEhYWhWrVqcHJywrBhw6SZcD/lq6++wuLFizFmzBiMHj0adevWxf/+9z+MHz9eZbk///xTZVAskb4ohAYDDl69eoWpU6di/vz5ePr0KcqVKwc3NzdYWlri6dOnuHz5Mv7991/Ur18fo0ePxldffZUTsSMpKQn29vZ4/vx57pok6md7Hdf/XLf155DXr18jLi4ORYsWzZUT1NSpUwdlypTBrFmzcmR7q1evRufOnfH8+XOdjS34mEKhwObNm2VPmT1mzBhERkZKlzx+yu3bt1G0aFFcuHDhk9NTDxkyBElJSVi4cKHGcdSqVQvlypXDjBkzNF5HX65cuYLatWvjxo0bsLfX8THk/8vtn8GPeQ7f+fmFsiE3/LD5UHa+QzXqeShZsiSqVq2KxYsXo169ehlOU3rnzh2sWbMG33zzDUaOHMnsmPKsZ8+eISIiAhEREZg3b57OtrNy5UoUK1YMhQoVwsWLFzFs2DC0a9cuxxKHdB06dED+/Plx//59jdfZvXs35syZ89nlGjVqhMOHD2tU58iRIzFv3jwolcpsTX5jqOLj47Fy5cocSxyIPkWj5GHfvn3w8fH55DIeHh4YMWIEBg8erHZZElFeEhAQgGfPnmHSpElSF7ouPHz4EKNHj8bDhw/h6uqKtm3bShNNfc7du3dRunTpTF+/evUqihQp8tl6YmJiAED2jahOnz6t0XK///47Xr16BQCfjcfBwQE//fSTrDhyk48n3iLSJ42Shw8Th7dv32Z6g5T//vsPTk5OHMxDeZomo+u1YejQodKkQnK5ubkhKirqk69rIrMrRrSlUKFCOq1fk9MmRKRO9vVb33zzDTZs2KB2j4FHjx6hTp06GY5mJiLDYmJiovMvfiL6csk+MXj37l21G1g9fPgQtWrVkmavIyIioi+X7ORh165dOH78uHRr7H///RdBQUHw9fXFX3/9pfUAiYiIyLDIPm3h7OyMffv2Sdc779ixA+XLl8fq1au/yBHOREREpCpL01O7u7tj//79qFGjBurVq4dVq1apjYEgIiKiL5NGyYOjo2OGycHLly+xfft25M+fXyp7+vSp9qIjIiIig6NR8pAbZl8j+tJldTZHbfp4RkZPT0/cuXMHwPvJsRwcHLS6vdDQUKxYsQIAPtv269evIygoCDExMbC1tc1wmeXLl6N///5ITEzUapy5wcdtX7BgAXbu3Int27frNzDKlTRKHkJCQnQdB5HmdD29t9r2vozpvnVl3Lhx6Natm8rMh//88w969eqFM2fOwNnZGX369PnknBQREREIDg5WS0BmzpyJiRMnanQL8REjRqBPnz6ZJg6kKiwsDOPHj8eRI0dQo0YNfYdDuUyWxjwolUrcvHkTjx8/VrtjXM2aNbUSGBHlDra2tihYsKD0PCkpCfXr10fdunWxYMECXLp0CWFhYXBwcFC54ZQm7O3tNZqO+e7du9ixYwdmz54tO/4vzZs3b2BmZvbZ5czMzNCxY0fMmjWLyQPJJvvyiJMnT8LLyws+Pj6oWbMmatWqJT2Cg4N1ESNRrrFy5Urkz58fqampKuUtW7bE999/Lz3funUrypcvDwsLCxQrVgxjx47Fu3fvpNdjYmJQs2ZNWFhYoHTp0ti/f7/ato4fP45y5crBwsICFStWxJYtW6BQKFRmjrx8+TIaNWoEGxsbFChQAN9//z3+++8/jdry4sULdOrUCTY2NnB1dcXUqVM1Wm/16tV48+YNli5dijJlyuCbb75B3759MW3atAyXv337tnTsSB9fFRoaqtG20v3111/w9/dXm5Fy+fLlKFKkCKysrNCqVSs8efJEbd358+ejePHiMDMzg7e3N1atWiW9NnjwYDRt2lR6PmPGDCgUCpXbZXt5eeH3338H8P40S8uWLTFlyhS4uroif/786NWrF96+fQsAmDNnDsqWLSutm/6eLViwQCpLv6MmAMTGxqJFixYoUKAAbGxsUKlSJRw4cEAlfk9PT4wfPx6dOnWCnZ2dlKBp0vZmzZph27Zt0hTgRJqSnTz06NEDFStWxOXLl/H06VM8e/ZMenCwJOV1bdu2RVpaGrZt2yaVPX78GDt37kRYWBgA4MiRI+jUqRP69euHq1evYuHChVi+fLl0XwqlUonWrVvDzMwMp06dwoIFCzBs2DCV7SQlJaFZs2bw9fXF+fPnMX78eLVlEhMTUbt2bQQEBODs2bPYs2cPHj16hHbt2mnUliFDhiAyMhJbt27Fvn37EBERgfPnz392vRMnTqBmzZoqv34bNGiA69ev49mzZ2rLu7u7Y+PGjQDej1uIj4/HzJkzNYox3ZEjR1CxYkWVslOnTqFLly7o3bs3oqKiEBwcjF9++UVlmc2bN6Nfv34YNGgQLl++jB9++AGdO3fGoUOHAABBQUE4evQo0tLSAACRkZFwcnKSprV+8OABYmNjUatWLanOQ4cOITY2FocOHcKKFSuwfPlyLF++XKrv6tWrSEhIyLC+t2/f4sSJE1J9KSkpaNy4MQ4ePIgLFy6gYcOGaNasmdr9g6ZMmQJ/f39cuHABo0aN0qjtAFCxYkW8e/cOp06dkrW/iWSftoiJicGGDRs4tS1RBiwtLdGxY0csW7YMbdu2BQD88ccfKFKkiPSFMHbsWAwfPlwaS1SsWDGMHz8eQ4cOxZgxY3DgwAFcu3YNe/fule4xER4ejkaNGknbWbNmDRQKBRYvXiz1Tjx48EDlbrZz5sxBQEAAwsPDpbKlS5fC3d0dN27cQMmSJTNtR0pKCpYsWYI//vgDderUAQCsWLEChQsX/uw+ePjwIYoWLapSVqBAAek1R0dHldeMjY2RL18+AICLi0uWBl3euXNHLXmYOXMmGjZsKI21KFmyJI4fP67SazBlyhSEhoaiZ8+eAICBAwfi5MmTmDJlCoKDg1GjRg0kJyfjwoULqFChAg4fPowhQ4Zgy5YtAN6P1ShUqJDK8dDR0RFz5syBsbExSpUqhSZNmuDgwYPo1q0bypYti3z58iEyMhJt2rRBREQEBg0aJCVLp0+fxtu3b1GtWjUAgL+/P/z9/aW6x48fj82bN2Pbtm3o3bu3VF67dm0MGjRIej5q1KjPth0ArKysYG9vLw16JdKU7J6HKlWq4ObNm7qIheiL0K1bN+zbtw8PHjwA8L77ODQ0VLrc+eLFixg3bhxsbGykR7du3RAfH4+XL18iOjoa7u7uKjenqlq1qso2rl+/Dj8/P1hYWEhllStXVlnm4sWLOHTokMp20qeQj42N/WQbYmNj8ebNG1SpUkUqy5cvn07vEpodr169UtkXABAdHa0SP6C+H6OjoxEYGKhSFhgYiOjoaADv79Tp7++PiIgIXLp0CWZmZujevTsuXLiAlJQUREZGIigoSGX9MmXKqNxl1NXVFY8fPwbw/oqZmjVrIiIiAomJibh69Sp69uyJ1NRUXLt2DZGRkahUqRKsrKwAvE/iBg8eDB8fHzg4OMDGxgbR0dFqPQ8fJ06atD2dpaUlXr58meFrRJmR3fPQp08fDBo0CA8fPoSvr6/aHTb9/Py0FhxRbhQQEAB/f3+sXLkS9evXx5UrV7Bz507p9ZSUFIwdOxatW7dWW/fjL8DsSElJQbNmzTBp0iS11zS5eiGrChYsiEePHqmUpT//cGClNjk5OWV4SkQbatWqhYiICJibmyMoKAj58uWDj48Pjh49isjISJVf/ADUjokKhUJlYHmtWrWwaNEiHDlyBAEBAbCzs5MSio+TkcGDB2P//v2YMmUKvLy8YGlpiTZt2uDNmzcq27C2ts5y+54+fQpnZ+csr095k+zk4euvvwYA6fwt8P7DIYSAQqGQzg0S5WVdu3bFjBkz8ODBA9StWxfu7u7Sa+XLl8f169czPfXn4+ODe/fuIT4+XvqSP3nypMoy3t7e+OOPP5Camgpzc3MAwJkzZ1SWKV++PDZu3AhPT0+YmMj7qBcvXhympqY4deoUihQpAuD9PA43btxQ+6X9sapVq2LkyJF4+/at9EW6f/9+eHt7q52ySJc+PiKrx4+AgABcvXpVpczHx0ftXP7H+9HHxwfHjh1TuRz92LFjKF26tPQ8KCgIS5cuhYmJCRo2bAjgfQLw559/4saNGyrjHTQRFBSE/v37Y/369dK6tWrVwoEDB3Ds2DGVZOTYsWMIDQ1Fq1atALxPCDW55bsmbQfe9zC9fv0aAQEBstpAJPu0RVxcnNrj1q1b0r9EBHTs2BH379/H4sWLVRJtABg9ejRWrlyJsWPH4sqVK4iOjsbatWulEfZ169ZFyZIlERISgosXL+LIkSMYOXKkWv1KpRLdu3dHdHQ09u7diylTpgCAdHqkV69eePr0KTp06IAzZ84gNjYWe/fuRefOnT/7JW1jY4MuXbpgyJAh+Pvvv3H58mWEhoZqdP+ajh07wszMDF26dMGVK1ewbt06zJw5U7qZXkY8PDygUCiwY8cOJCQkICUl5bPb+VCDBg1w4sQJlXb17dsXe/bswZQpUxATE4M5c+aonfMfMmQIli9fjvnz5yMmJgbTpk3Dpk2bMHjwYGmZmjVrIjk5GTt27FD5sl+9ejVcXV0/OXYkI35+fnB0dMSaNWtU6tuyZQtSU1NVTqOUKFECmzZtQlRUFC5evCi975+jSduB9wNNixUrhuLFi8tqA5HsngcPDw9dxEGkuVwwaZO9vT2+/vpr7Ny5U21WxAYNGmDHjh0YN24cJk2aBFNTU5QqVUq61b2RkRE2b96MLl26oHLlyvD09MSsWbOkX70AYGdnh+3bt+PHH39EuXLl4Ovri9GjR6Njx47SqQ83NzccO3YMw4YNQ/369ZGamgoPDw80bNhQoyTgt99+k0592NraYtCgQXj+/PP73t7eHvv27UOvXr1QoUIFODk5YfTo0SpzPKRPChUXFwdPT08UKlRIGkjauXNndOrUSbpCQRONGjWCiYkJDhw4gAYNGgAAvvrqKyxevBhjxozB6NGjpUsgx48fL63XsmVLzJw5E1OmTEG/fv1QtGhRLFu2TKU3wdHREb6+vnj06JE0ZqRmzZpQKpWf7YXJiEKhQI0aNbBz507pBoN+fn6ws7ODt7e3yimIadOmISwsDNWqVYOTkxOGDRuGpKSkz25Dk7YDwJ9//qkyyJZIUwohhPjcQtu2bUOjRo1gamqqcglaRpo3b6614D4nKSkJ9vb2eP78Oezs7HJsu9mm6xkSc8GXqyZev36NuLg4FC1aVKtjAXJKnTp1UKZMGcyaNStHtrd69Wp07twZz58/h6WlZY5s09PTE/3790f//v1lrbds2TKEh4fj6tWramMEMqLJ1Nxz587Ftm3bsHfvXlmx5FVXrlxB7dq1cePGjUwn4srtn8GPeQ7f+fmFsuH2xCY6rV/bsvMdqlHPQ8uWLfHw4UO4uLh88sPLMQ9E78cGREREICIiAvPmzdPZdlauXIlixYqhUKFCuHjxIoYNG4Z27drlWOKQbtiwYfjf//6HBw8eaDQbJADs2rUL4eHhn00cevTogT/++EOjOn/44QckJiYiOTmZU1RrID4+HitXrtT4PSP6kEbJw4fn2DQ530aUlwUEBODZs2eYNGmSTi9tfPjwIUaPHo2HDx/C1dUVbdu2lSaa+py7d++qDAr82NWrV6WBkp8SGRkpzZ4o5wt7/fr1Gi03btw4afzB564QMTExURsbQpmrW7euvkOgXCxL97YgosxpMhpeG4YOHfrJm019ipubm8o01hm9rgldj4FycXGBi4uLTrdBRPJplDzIOWfbt2/fLAdDRDnDxMSEs8QSUZZplDxMnz5do8oUCgWTB9I6Dcb0EpEO8LNHmdEoeYiLi9N1HERq0gfTvXz5MscHARIRpJksP5xumwgwgDEPDx48wLBhw7B79268fPkSXl5eWLZsmdpc7ZT3GBsbw8HBQbovgJWVlTQBEhHpllKpREJCAqysrGTPUEpfPr3+RTx79gyBgYEIDg7G7t274ezsjJiYmEynsKW8J/1eCOkJBBHlHCMjIxQpUoRJO6nRa/IwadIkuLu7Y9myZVLZx7fypbxNoVDA1dUVLi4u0iWBRJQzzMzMNJqNlPIevSYP27ZtQ4MGDdC2bVtERkaiUKFC6NmzZ6bTpaampiI1NVV6nj5Nq1KpzGXzT+j4w5ir9oVmFAqFdPMkIso5uevY+mlG0O0A0Ny2r7ITr16Th1u3bmH+/PkYOHAgfvrpJ5w5cwZ9+/aFmZmZyl3u0k2YMAFjx45VK09ISMDr169zImTtsNPxbcsNuYt/TXvd1d1xne7qJvqCdFlx5vMLZcOSkEo6rT+rfBx1mzzkttOrycnJWV43S8nDkSNHsHDhQsTGxmLDhg0oVKgQVq1ahaJFi0o3etGEUqlExYoVER4eDuD9zHyXL1/GggULMkweRowYoXJnvqSkJLi7u8PZ2Tl33dsi6R/d1m/Ik+rosu2G3G4iAxL9TLdjGAx1Yq+82u7MZOd+JbKTh40bN+L777/Ht99+iwsXLkinEZ4/f47w8HDs2rVL47pcXV3Vpsj18fHBxo0bM1ze3Nwc5ubmauVGRka57Lycjru2DHpf6LDtBt1uIsOhhG6/RA31eJxX252Z7MQre81ffvkFCxYswOLFi1VuahMYGIjz58/LqiswMBDXr19XKbtx4wZv+01ERGTAZCcP169fR82aNdXK7e3tkZiYKKuuAQMG4OTJkwgPD8fNmzexZs0aLFq0CL169ZIbFhEREeUQ2clDwYIFcfPmTbXyo0ePolixYrLqqlSpEjZv3ow///wTZcuWxfjx4zFjxgx8++23csMiIiKiHCJ7zEO3bt3Qr18/LF26FAqFAv/++y9OnDiBwYMHY9SoUbIDaNq0KZo2bSp7PSIiItIP2cnD8OHDoVQqUadOHbx8+RI1a9aEubk5Bg8ejD59+ugiRiIiIjIgspMHhUKBkSNHYsiQIbh58yZSUlJQunRp2NjY6CI+IiIiMjBZniTKzMxM7TJLIiIi+vLJTh5evHiBiRMn4uDBg3j8+LHa9Ja3bt3SWnBERERkeGQnD127dkVkZCS+//57uLq68m5rREREeYzs5GH37t3YuXMnAgMDdREPERERGTjZ8zw4OjoiX758uoiFiIiIcgHZycP48eMxevRovHz5UhfxEBERkYHT6LRFQECAytiGmzdvokCBAvD09FS5vwUA2fe3ICIiotxFo+ShZcuWOg6DiIiIcguNkocxY8boOg4iIiLKJWSPeShWrBiePHmiVp6YmCj7xlhERESU+8hOHm7fvo20tDS18tTUVNy/f18rQREREZHh0nieh23btkn/37t3L+zt7aXnaWlpOHjwIIoWLard6IiIiMjgaJw8pA+aVCgUCAkJUXnN1NQUnp6emDp1qlaDIyIiIsOjcfKQfg+LokWL4syZM3ByctJZUERERGS4ZE9PHRcXp4s4iIiIKJeQPWCSiIiI8jYmD0RERCQLkwciIiKShckDERERySJ7wGRSUlKG5QqFAubm5jAzM8t2UERERGS4ZCcPDg4OKnfY/FjhwoURGhqKMWPGwMiIHRtERERfGtnJw/LlyzFy5EiEhoaicuXKAIDTp09jxYoV+N///oeEhARMmTIF5ubm+Omnn7QeMBEREemX7ORhxYoVmDp1Ktq1ayeVNWvWDL6+vli4cCEOHjyIIkWK4Ndff2XyQERE9AWSfV7h+PHjCAgIUCsPCAjAiRMnAADVq1fH3bt3sx8dERERGRzZyYO7uzuWLFmiVr5kyRK4u7sDAJ48eQJHR8fsR0dEREQGR/ZpiylTpqBt27bYvXs3KlWqBAA4e/Ysrl27hg0bNgAAzpw5g/bt22s3UiIiIjIIspOH5s2b49q1a1i4cCFu3LgBAGjUqBG2bNkCT09PAMCPP/6o1SCJiIjIcMhOHoD3d9acOHGitmMhIiKiXCBLyUNiYiJOnz6Nx48fS7fqTtepUyetBEZERESGSXbysH37dnz77bdISUmBnZ2dyoRRCoVCVvLw888/Y+zYsSpl3t7euHbtmtywiIiIKIfITh4GDRqEsLAwhIeHw8rKKtsBlClTBgcOHPi/gEyy1BlCREREOUT2N/WDBw/Qt29frSQOwPtkoWDBglqpi4iIiHRPdvLQoEEDnD17FsWKFdNKADExMXBzc4OFhQWqVq2KCRMmoEiRIhkum5qaitTUVOl5+k26lEql2tgLw6bje34Y9L7QYdsNut1EhsMIQqf1G+rxOK+2OzPZiVd28tCkSRMMGTIEV69eha+vL0xNTVVeb968ucZ1ValSBcuXL4e3tzfi4+MxduxY1KhRA5cvX4atra3a8hMmTFAbIwEACQkJeP36tdym6I+dn27rf/xYt/Vnhy7bbsjtXqPjeU86rtNt/V+gLivO6LT+JSGVdFp/dvg46vZL9LGBfhbzarszk5ycnOV1FUIIWXvzU3fKVCgUSEtLy3IwiYmJ8PDwwLRp09ClSxe11zPqeXB3d8ezZ89gZ2eX5e3muHH5dVv/6Ce6rT87dNn2vNpuwLDbbqC8ftql0/pvhjfWaf3ZkVfbnlfbnZmkpCQ4Ojri+fPnsr9DZfc86LJbxsHBASVLlsTNmzczfN3c3Bzm5uZq5UZGRrns9t867toy6H2hw7bn1XYDBt52w6SE4vMLZYMhH5Pyatvzarszk514DaqlKSkpiI2Nhaurq75DISIiokxo1PMwa9YsdO/eHRYWFpg1a9Ynl+3bt6/GGx88eDCaNWsGDw8P/PvvvxgzZgyMjY3RoUMHjesgIiKinKVR8jB9+nR8++23sLCwwPTp0zNdTqFQyEoe7t+/jw4dOuDJkydwdnZG9erVcfLkSTg7O2tcBxEREeUsjZKHuLi4DP+fXWvXrtVaXURERJQzsj3mIS0tDVFRUXj27Jk24iEiIiIDJzt56N+/P5YsWQLgfeJQs2ZNlC9fHu7u7oiIiNB2fERERGRgZCcPGzZsgL+/P4D3N8m6ffs2rl27hgEDBmDkyJFaD5CIiIgMi+zk4b///pPuRbFr1y60bdsWJUuWRFhYGC5duqT1AImIiMiwyE4eChQogKtXryItLQ179uxBvXr1AAAvX76EsbGx1gMkIiIiwyJ7hsnOnTujXbt2cHV1hUKhQN26dQEAp06dQqlSpbQeIBERERkW2cnDzz//jLJly+LevXto27atNF20sbExhg8frvUAiYiIyLDITh4AoE2bNirPExMTERISopWAiIiIyLDJHvMwadIkrFv3f7f/bdeuHfLnz4/ChQvjn3/+0WpwREREZHhkJw8LFiyAu7s7AGD//v3Yv38/du/ejYYNG2Lw4MFaD5CIiIgMi+zTFg8fPpSShx07dqBdu3aoX78+PD09UaVKFa0HSERERIZFds+Do6Mj7t27BwDYs2ePdLWFEAJpaWnajY6IiIgMjuyeh9atW6Njx44oUaIEnjx5gkaNGgEALly4AC8vL60HSERERIZFdvIwffp0eHp64t69e5g8eTJsbGwAAPHx8ejZs6fWAyQiIiLDIjt5MDU1zXBg5IABA7QSEBERERm2LM3zEBsbixkzZiA6OhoAULp0afTv3x/FihXTanBERERkeGQPmNy7dy9Kly6N06dPw8/PD35+fjh16hRKly6N/fv36yJGIiIiMiCyex6GDx+OAQMGYOLEiWrlw4YNk26URURERF8m2T0P0dHR6NKli1p5WFgYrl69qpWgiIiIyHDJTh6cnZ0RFRWlVh4VFQUXFxdtxEREREQGTPZpi27duqF79+64desWqlWrBgA4duwYJk2ahIEDB2o9QCIiIjIsspOHUaNGwdbWFlOnTsWIESMAAG5ubvj555/Rt29frQdIREREhkVW8vDu3TusWbMGHTt2xIABA5CcnAwAsLW11UlwREREZHhkjXkwMTFBjx498Pr1awDvkwYmDkRERHmL7AGTlStXxoULF3QRCxEREeUCssc89OzZE4MGDcL9+/dRoUIFWFtbq7zu5+enteCIiIjI8MhOHr755hsAUBkcqVAoIISAQqHgbbmJiIi+cLKTh7i4OF3EQURERLmE7OTBw8NDF3EQERFRLiF7wOSECROwdOlStfKlS5di0qRJWgmKiIiIDJfs5GHhwoUoVaqUWnmZMmWwYMECrQRFREREhkt28vDw4UO4urqqlTs7OyM+Pj7LgUycOBEKhQL9+/fPch1ERESke7KTB3d3dxw7dkyt/NixY3Bzc8tSEGfOnMHChQt5mScREVEuIDt56NatG/r3749ly5bhzp07uHPnDpYuXYoBAwagW7dusgNISUnBt99+i8WLF8PR0VH2+kRERJSzZF9tMWTIEDx58gQ9e/bEmzdvAAAWFhYYNmyYdKMsOXr16oUmTZqgbt26+OWXXz65bGpqKlJTU6XnSUlJAAClUgmlUil72/ojO2eTx6D3hQ7bnlfbDRh42w2TEYRO6zfkY1JebXtebXdmshOv7ORBoVBg0qRJGDVqFKKjo2FpaYkSJUrA3Nxc9sbXrl2L8+fP48yZMxotP2HCBIwdO1atPCEhQbrfRq5gp+PTM48f67b+7NBl2/Nqu4Fstb3LCs0+f1m1xHSK7irvuC7Lq/o46vaL5LEB/z3m1bbn1XZnJv3mllkhO3lIZ2Njg0qVKmV5w/fu3UO/fv2wf/9+WFhYaLTOiBEjMHDgQOl5UlIS3N3d4ezsDDs7uyzHkuOS/tFt/S4uuq0/O3TZ9rzabiBbbY9+ptBiIOpcLAzzPdd5uw347zGvtj2vtjszmn73ZiTLyUN2nTt3Do8fP0b58uWlsrS0NBw+fBhz5sxBamoqjI2NVdYxNzfPsIfDyMgIRkY67hbWKh13bRn0vtBh2/Nqu4FstV0J3R5QjQz0Pdd5uw347zGvtj2vtjsz2YlXb8lDnTp1cOnSJZWyzp07o1SpUhg2bJha4kBERESGQW/Jg62tLcqWLatSZm1tjfz586uVExERkeHIXX0sREREpHeyk4cVK1Zg586d0vOhQ4fCwcEB1apVw507d7IVTEREBGbMmJGtOoiIiEi3ZCcP4eHhsLS0BACcOHECc+fOxeTJk+Hk5IQBAwZoPUAiIiIyLLLHPNy7dw9eXl4AgC1btuDrr79G9+7dERgYiFq1amk7PiIiIjIwsnsebGxs8OTJEwDAvn37UK9ePQDvrxd99eqVdqMjIiIigyO756FevXro2rUrAgICcOPGDTRu3BgAcOXKFXh6emo7PiIiIjIwsnse5s6di6pVqyIhIQEbN25E/vz5Abyf9KlDhw5aD5CIiIgMi+yeBwcHB8yZM0etPKN7ThAREdGXJ0vzPBw5cgTfffcdqlWrhgcPHgAAVq1ahaNHj2o1OCIiIjI8spOHjRs3okGDBrC0tMT58+elW2Q/f/4c4eHhWg+QiIiIDIvs5OGXX37BggULsHjxYpiamkrlgYGBOH/+vFaDIyIiIsMjO3m4fv06atasqVZub2+PxMREbcREREREBkx28lCwYEHcvHlTrfzo0aMoVqyYVoIiIiIiwyU7eejWrRv69euHU6dOQaFQ4N9//8Xq1asxePBg/Pjjj7qIkYiIiAyI7Es1hw8fDqVSiTp16uDly5eoWbMmzM3NMXjwYPTp00cXMRIREZEBkZ08KBQKjBw5EkOGDMHNmzeRkpKC0qVLw8bGRhfxERERkYHJ0jwPAHD37l3cu3cPvr6+sLGxgRBCm3ERERGRgZKdPDx58gR16tRByZIl0bhxY8THxwMAunTpgkGDBmk9QCIiIjIsspOHAQMGwNTUFHfv3oWVlZVU3r59e+zZs0erwREREZHhkT3mYd++fdi7dy8KFy6sUl6iRAncuXNHa4ERERGRYZLd8/DixQuVHod0T58+hbm5uVaCIiIiIsMlO3moUaMGVq5cKT1XKBRQKpWYPHkygoODtRocERERGR7Zpy0mT56MOnXq4OzZs3jz5g2GDh2KK1eu4OnTpzh27JguYiQiIiIDIrvnoWzZsrhx4waqV6+OFi1a4MWLF2jdujUuXLiA4sWL6yJGIiIiMiCyeh7evn2Lhg0bYsGCBRg5cqSuYiIiIiIDJqvnwdTUFP/884+uYiEiIqJcQPZpi++++w5LlizRRSxERESUC8geMPnu3TssXboUBw4cQIUKFWBtba3y+rRp07QWHBERERke2cnD5cuXUb58eQDAjRs3tB4QERERGTbZycOhQ4d0EQcRERHlErLHPISFhSE5OVmt/MWLFwgLC9NKUERERGS4ZCcPK1aswKtXr9TKX716pTLzJBEREX2ZND5tkZSUBCEEhBBITk6GhYWF9FpaWhp27doFFxcXnQRJREREhkPjngcHBwfky5cPCoUCJUuWhKOjo/RwcnJCWFgYevXqJWvj8+fPh5+fH+zs7GBnZ4eqVati9+7dshtBREREOUfjnodDhw5BCIHatWtj48aNyJcvn/SamZkZPDw84ObmJmvjhQsXxsSJE1GiRAkIIbBixQq0aNECFy5cQJkyZWTVRURERDlD4+QhKCgIABAXF4ciRYpAoVBke+PNmjVTef7rr79i/vz5OHnyJJMHIiIiAyX7Uk0PDw9dxIG0tDSsX78eL168QNWqVTNcJjU1FampqdLzpKQkAIBSqYRSqdRJXLohe5yqPAa9L3TY9rzabiBbbTeC0GIg6pQG+p7rvN0G/PeYV9ueV9udmezEKzt50LZLly6hatWqeP36NWxsbLB582aULl06w2UnTJiAsWPHqpUnJCTg9evXug5Ve+z8dFv/48e6rT87dNn2bLS7y4ozWgxE3RIDfs99HHV7QH1sapjvuc7bPSNYp/Wj47osr6rzthvoMSivtjszGU27oCm9Jw/e3t6IiorC8+fPsWHDBoSEhCAyMjLDBGLEiBEYOHCg9DwpKQnu7u5wdnaGnZ1dToadPUk6vrmYIV/1osu2Z6Pd0c+yfxruU1wsDPc9z9Vtz6vtBgy77QZ6DMqr7c7Mh1dNyqX35MHMzAxeXl4AgAoVKuDMmTOYOXMmFi5cqLasubk5zM3N1cqNjIxgZKTjbmGt0nHXlkHvCx22PRvtVkK3BxUjA37Pc3Xb82q7AcNuu4Eeg/JquzOTnXhlr1m7dm0kJiaqlSclJaF27dpZDiSdUqlUGddAREREhkV2z0NERATevHmjVv769WscOXJEVl0jRoxAo0aNUKRIESQnJ2PNmjWIiIjA3r175YZFREREOUTj5OGff/7v/N3Vq1fx8OFD6XlaWhr27NmDQoUKydr448eP0alTJ8THx8Pe3h5+fn7Yu3cv6tWrJ6seIiIiyjkaJw/lypWDQqGAQqHI8PSEpaUlZs+eLWvjS5YskbU8ERER6Z/GyUNcXByEEChWrBhOnz4NZ2dn6TUzMzO4uLjA2NhYJ0ESERGR4dA4eUifHCq3TYJBRERE2pWlW3Lv3LlTej506FA4ODigWrVquHPnjlaDIyIiIsMjO3kIDw+HpaUlAODEiROYM2cOJk+eDCcnJwwYMEDrARIREZFhkX2p5r1796RJnbZs2YI2bdqge/fuCAwMRK1atbQdHxERERkY2T0PNjY2ePLkCQBg37590mWVFhYWePXqlXajIyIiIoMju+ehXr166Nq1KwICAnDjxg00btwYAHDlyhV4enpqOz4iIiIyMLJ7HubOnYuqVasiISEBGzduRP78+QEA586dQ4cOHbQeIBERERkW2T0PDg4OmDNnjlp5RrfKJiIioi9Plu+q+fLlS9y9e1ftPhd+fn7ZDoqIiIgMl+zkISEhAaGhodizZ0+Gr6elpWU7KCIiIjJcssc89O/fH8+fP8epU6dgaWmJPXv2YMWKFShRogS2bdumixiJiIjIgMjuefj777+xdetWVKxYEUZGRvDw8EC9evVgZ2eHCRMmoEmTJrqIk4iIiAyE7J6HFy9ewMXFBQDg6OiIhIQEAICvry/Onz+v3eiIiIjI4MhOHry9vXH9+nUAgL+/PxYuXIgHDx5gwYIFcHV11XqAREREZFhkn7bo168f4uPjAQBjxoxBw4YNsXr1apiZmWH58uXajo+IiIgMjOzk4bvvvpP+X6FCBdy5cwfXrl1DkSJF4OTkpNXgiIiIyPBkeZ6HdFZWVihfvrw2YiEiIqJcQPaYh6+//hqTJk1SK588eTLatm2rlaCIiIjIcMlOHg4fPizdDOtDjRo1wuHDh7USFBERERku2clDSkoKzMzM1MpNTU2RlJSklaCIiIjIcMlOHnx9fbFu3Tq18rVr16J06dJaCYqIiIgMl+wBk6NGjULr1q0RGxuL2rVrAwAOHjyIP//8E+vXr9d6gERERGRYZCcPzZo1w5YtWxAeHo4NGzbA0tISfn5+OHDgAIKCgnQRIxERERmQLF2q2aRJE97DgoiIKI+SPeaBiIiI8jaNeh7y5cuHGzduwMnJCY6OjlAoFJku+/TpU60FR0RERIZHo+Rh+vTpsLW1lf7/qeSBiIiIvmwaJQ8hISHS/0NDQ3UVCxEREeUCssc8GBsb4/Hjx2rlT548gbGxsVaCIiIiIsMlO3kQQmRYnpqamuHMk0RERPRl0fhSzVmzZgEAFAoFfv/9d9jY2EivpaWl4fDhwyhVqpSsjU+YMAGbNm3CtWvXYGlpiWrVqmHSpEnw9vaWVQ8RERHlHI2Th+nTpwN43/OwYMEClVMUZmZm8PT0xIIFC2RtPDIyEr169UKlSpXw7t07/PTTT6hfvz6uXr0Ka2trWXURERFRztA4eYiLiwMABAcHY9OmTXB0dMz2xvfs2aPyfPny5XBxccG5c+dQs2bNbNdPRERE2id7hslDhw7pIg4AwPPnzwG8n1ciI6mpqUhNTZWep9/FU6lUQqlU6iwu7dPx3FwGvS902PZstNsIGY/l0RalAb/nubrtebXdgGG33UCPQXm13ZnJTrwaJQ8DBw7UuMJp06ZlKRClUon+/fsjMDAQZcuWzXCZCRMmYOzYsWrlCQkJeP36dZa2qxd2frqtP4OrYTTVZcUZLQaiboku256Ndvs46vag8tjUcN/zXN32vNpuwLDbPiNYd5V3VL+rs6Z03u5svCf6kJycnOV1NUoeLly4oFFl2Zk8qlevXrh8+TKOHj2a6TIjRoxQSWSSkpLg7u4OZ2dn2NnZZXnbOS7pH93W7+KS5VWjn+l2AjAXCx22Pa+2G8i7bc+r7QbybtsNud3ZiE0fLCwssryuRsmDLk9VAEDv3r2xY8cOHD58GIULF850OXNzc5ibm6uVGxkZwcgoN92mQ8ddW9nYF0ro9sNlpMu259V2A3m37Xm13UDebbshtztXfQ9lL94s3VVTW4QQ6NOnDzZv3oyIiAgULVpUn+EQERGRBmQnD8HBwZ88PfH3339rXFevXr2wZs0abN26Fba2tnj48CEAwN7eHpaWlnJDIyIiohwgO3koV66cyvO3b98iKioKly9fVrkHhibmz58PAKhVq5ZK+bJly3gPDSIiIgMlO3lInyzqYz///DNSUlJk1ZXZVNdERERkuLQ2uuO7777D0qVLtVUdERERGSitJQ8nTpzI1mUfRERElDvIPm3RunVrledCCMTHx+Ps2bMYNWqU1gIjIiIiwyQ7ebC3t1d5bmRkBG9vb4wbNw7169fXWmBERERkmGQnD8uWLdNFHERERJRLZGuSqJSUFLUba+SqaaKJiIhINtkDJuPi4tCkSRNYW1vD3t4ejo6OcHR0hIODg1Zu001ERESGTXbPw3fffQchBJYuXYoCBQpk62ZYRERElPvITh4uXryIc+fOwdvbWxfxEBERkYGTfdqiUqVKuHfvni5iISIiolxAds/D77//jh49euDBgwcoW7YsTE1NVV738/PTWnBERERkeGQnDwkJCYiNjUXnzp2lMoVCASEEFAoF0tLStBogERERGRbZyUNYWBgCAgLw559/csAkERFRHiQ7ebhz5w62bdsGLy8vXcRDREREBk72gMnatWvj4sWLuoiFiIiIcgHZPQ/NmjXDgAEDcOnSJfj6+qoNmGzevLnWgiMiIiLDIzt56NGjBwBg3Lhxaq9xwCQREdGXT3by8PG9LIiIiChvkT3mgYiIiPI2jZKHtWvXalzhvXv3cOzYsSwHRERERIZNo+Rh/vz58PHxweTJkxEdHa32+vPnz7Fr1y507NgR5cuXx5MnT7QeKBERERkGjcY8REZGYtu2bZg9ezZGjBgBa2trFChQABYWFnj27BkePnwIJycnhIaG4vLlyyhQoICu4yYiIiI90XjAZPPmzdG8eXP8999/OHr0KO7cuYNXr17ByckJAQEBCAgIgJERh1AQERF96WRfbeHk5ISWLVvqIBQiIiLKDdhVQERERLIweSAiIiJZmDwQERGRLEweiIiISBYmD0RERCSLRldbDBw4UOMKp02bluVgiIiIyPBplDxcuHBB5fn58+fx7t07eHt7AwBu3LgBY2NjVKhQQfsREhERkUHR6LTFoUOHpEezZs0QFBSE+/fv4/z58zh//jzu3buH4OBgNGnSRNbGDx8+jGbNmsHNzQ0KhQJbtmzJShuIiIgoB8ke8zB16lRMmDABjo6OUpmjoyN++eUXTJ06VVZdL168gL+/P+bOnSs3DCIiItIT2TNMJiUlISEhQa08ISEBycnJsupq1KgRGjVqJDcEIiIi0iPZyUOrVq3QuXNnTJ06FZUrVwYAnDp1CkOGDEHr1q21HuCHUlNTkZqaKj1PSkoCACiVSiiVSp1uW7t0fJFLNvaFEYQWA1Gn1GXb82q7gbzb9rzabiDvtt2Q252rvoeyF6/s5GHBggUYPHgwOnbsiLdv376vxMQEXbp0wW+//ZblQDQxYcIEjB07Vq08ISEBr1+/1tp2uqw4o7W6MrLEzk+n9ePx4yyv6uOo2w/XY1Mdtj2vthvIu23Pq+0G8m7bDbndM4J1Wj86rtNqdXLPFnxIdvJgZWWFefPm4bfffkNsbCwAoHjx4rC2ts5yEJoaMWKEymWjSUlJcHd3h7OzM+zs7LS2nehnCq3VlREXi390Wj9cXLK8aq5ue15tN5B3255X2w3k3bbn1XYD2Wp7RiwsLLK8ruzkIZ21tTX8/HScWX/E3Nwc5ubmauVGRkZavR24Err9AzOCjru2srEvcnXb82q7gbzb9rzabiDvtj2vthvIVtszri7r9WUpeTh79iz++usv3L17F2/evFF5bdOmTVkOhoiIiAyf7LRj7dq1qFatGqKjo7F582a8ffsWV65cwd9//w17e3tZdaWkpCAqKgpRUVEAgLi4OERFReHu3btywyIiIqIcIjt5CA8Px/Tp07F9+3aYmZlh5syZuHbtGtq1a4ciRYrIquvs2bMICAhAQEAAgPfTYAcEBGD06NFywyIiIqIcIvu0RWxsrDSTpJmZGV68eAGFQoEBAwagdu3aGV4NkZlatWpBCN2OfiUiIiLtkt3z4OjoKF3eUahQIVy+fBkAkJiYiJcvX2o3OiIiIjI4snseatasif3798PX1xdt27ZFv3798Pfff2P//v2oU6eOLmIkIiIiAyI7eZgzZ440IdPIkSNhamqK48eP4+uvv8b//vc/rQdIREREhkV28pAvXz7p/0ZGRhg+fLhWAyIiIiLDlqV5HtLS0rBlyxZER0cDAMqUKYPmzZvD2NhYq8ERERGR4ZGdPNy8eRNNmjTB/fv34e3tDeD9PSfc3d2xc+dOFC9eXOtBEhERkeGQfbVF3759UaxYMdy7dw/nz5/H+fPncffuXRQtWhR9+/bVRYxERERkQGT3PERGRuLkyZMqYx/y58+PiRMnIjAwUKvBERERkeGR3fNgbm6e4W08U1JSYGZmppWgiIiIyHDJTh6aNm2K7t2749SpUxBCQAiBkydPokePHmjevLkuYiQiIiIDIjt5mDVrFooXL46qVavCwsICFhYWCAwMhJeXF2bOnKmLGImIiMiAyB7z4ODggK1btyImJgbXrl0DAPj4+MDLy0vrwREREZHhydI8DwBQokQJlChRQpuxEBERUS6gUfIwcOBAjSucNm1aloMhIiIiw6dR8nDhwgWNKlMoFNkKhoiIiAyfRsnDoUOHdB0HERER5RKyr7YgIiKivI3JAxEREcnC5IGIiIhkYfJAREREsjB5ICIiIlmYPBAREZEsTB6IiIhIFiYPREREJAuTByIiIpKFyQMRERHJwuSBiIiIZGHyQERERLIweSAiIiJZmDwQERGRLEweiIiISBaDSB7mzp0LT09PWFhYoEqVKjh9+rS+QyIiIqJM6D15WLduHQYOHIgxY8bg/Pnz8Pf3R4MGDfD48WN9h0ZEREQZ0HvyMG3aNHTr1g2dO3dG6dKlsWDBAlhZWWHp0qX6Do2IiIgyYKLPjb958wbnzp3DiBEjpDIjIyPUrVsXJ06cUFs+NTUVqamp0vPnz58DABITE6FUKrUXWOoL7dWVgUSFQqf1IzEx6+vm5rbn1XYDebftebXdQN5te15tN5C9tmcgKSkJACCEkL+y0KMHDx4IAOL48eMq5UOGDBGVK1dWW37MmDECAB988MEHH3zwoaXHvXv3ZH9/67XnQa4RI0Zg4MCB0nOlUomnT58if/78UOg649OSpKQkuLu74969e7Czs9N3ODkqr7Y9r7YbyLttz6vtBvJu23Nju4UQSE5Ohpubm+x19Zo8ODk5wdjYGI8ePVIpf/ToEQoWLKi2vLm5OczNzVXKHBwcdBmiztjZ2eWaPzBty6ttz6vtBvJu2/Nqu4G82/bc1m57e/ssrafXAZNmZmaoUKECDh48KJUplUocPHgQVatW1WNkRERElBm9n7YYOHAgQkJCULFiRVSuXBkzZszAixcv0LlzZ32HRkRERBnQe/LQvn17JCQkYPTo0Xj48CHKlSuHPXv2oECBAvoOTSfMzc0xZswYtdMveUFebXtebTeQd9ueV9sN5N2257V2K4TIyjUaRERElFfpfZIoIiIiyl2YPBAREZEsTB6IiIhIFiYPREREJAuThxyWF28/fvjwYTRr1gxubm5QKBTYsmWLvkPKERMmTEClSpVga2sLFxcXtGzZEtevX9d3WDo3f/58+Pn5SZPlVK1aFbt379Z3WDlu4sSJUCgU6N+/v75D0bmff/4ZCoVC5VGqVCl9h5VjHjx4gO+++w758+eHpaUlfH19cfbsWX2HpVNMHnJQXr39+IsXL+Dv74+5c+fqO5QcFRkZiV69euHkyZPYv38/3r59i/r16+PFC93enEffChcujIkTJ+LcuXM4e/YsateujRYtWuDKlSv6Di3HnDlzBgsXLoSfn5++Q8kxZcqUQXx8vPQ4evSovkPKEc+ePUNgYCBMTU2xe/duXL16FVOnToWjo6O+Q9OtrN3SirKicuXKolevXtLztLQ04ebmJiZMmKDHqHIWALF582Z9h6EXjx8/FgBEZGSkvkPJcY6OjuL333/Xdxg5Ijk5WZQoUULs379fBAUFiX79+uk7JJ0bM2aM8Pf313cYejFs2DBRvXp1fYeR49jzkEPSbz9et25dqexTtx+nL0/6LeTz5cun50hyTlpaGtauXYsXL17kmSnne/XqhSZNmqh81vOCmJgYuLm5oVixYvj2229x9+5dfYeUI7Zt24aKFSuibdu2cHFxQUBAABYvXqzvsHSOyUMO+e+//5CWlqY2c2aBAgXw8OFDPUVFOUWpVKJ///4IDAxE2bJl9R2Ozl26dAk2NjYwNzdHjx49sHnzZpQuXVrfYenc2rVrcf78eUyYMEHfoeSoKlWqYPny5dizZw/mz5+PuLg41KhRA8nJyfoOTedu3bqF+fPno0SJEti7dy9+/PFH9O3bFytWrNB3aDql9+mpifKCXr164fLly3nmPLC3tzeioqLw/PlzbNiwASEhIYiMjPyiE4h79+6hX79+2L9/PywsLPQdTo5q1KiR9H8/Pz9UqVIFHh4e+Ouvv9ClSxc9RqZ7SqUSFStWRHh4OAAgICAAly9fxoIFCxASEqLn6HSHPQ85RO7tx+nL0bt3b+zYsQOHDh1C4cKF9R1OjjAzM4OXlxcqVKiACRMmwN/fHzNnztR3WDp17tw5PH78GOXLl4eJiQlMTEwQGRmJWbNmwcTEBGlpafoOMcc4ODigZMmSuHnzpr5D0TlXV1e1pNjHx+eLP23D5CGH8PbjeY8QAr1798bmzZvx999/o2jRovoOSW+USiVSU1P1HYZO1alTB5cuXUJUVJT0qFixIr799ltERUXB2NhY3yHmmJSUFMTGxsLV1VXfoehcYGCg2iXYN27cgIeHh54iyhk8bZGD8urtx1NSUlR+gcTFxSEqKgr58uVDkSJF9BiZbvXq1Qtr1qzB1q1bYWtrK41tsbe3h6WlpZ6j050RI0agUaNGKFKkCJKTk7FmzRpERERg7969+g5Np2xtbdXGs1hbWyN//vxf/DiXwYMHo1mzZvDw8MC///6LMWPGwNjYGB06dNB3aDo3YMAAVKtWDeHh4WjXrh1Onz6NRYsWYdGiRfoOTbf0fblHXjN79mxRpEgRYWZmJipXrixOnjyp75B07tChQwKA2iMkJETfoelURm0GIJYtW6bv0HQqLCxMeHh4CDMzM+Hs7Czq1Kkj9u3bp++w9CKvXKrZvn174erqKszMzEShQoVE+/btxc2bN/UdVo7Zvn27KFu2rDA3NxelSpUSixYt0ndIOsdbchMREZEsHPNAREREsjB5ICIiIlmYPBAREZEsTB6IiIhIFiYPREREJAuTByIiIpKFyQMRERHJwuSBiIiIZGHyQETZolAosGXLlmzVsXz5cjg4OGglHiLSPd7bgoiyJT4+Ho6OjvoO47NCQ0ORmJiY7USHiJg8EH2x3rx5AzMzM51vh7eUJ8p7eNqC6AtRq1Yt9O7dG/3794eTkxMaNGgAAIiMjETlypVhbm4OV1dXDB8+HO/evQMA7NixAw4ODkhLSwMAREVFQaFQYPjw4VK9Xbt2xXfffZfpdj88bXH79m0oFAps2rQJwcHBsLKygr+/P06cOKGyzvLly1GkSBFYWVmhVatWePLkiVq9W7duRfny5WFhYYFixYph7NixUtzjxo2Dm5ubynpNmjRBcHAwlEqlWl0///wzVqxYga1bt0KhUEChUCAiIkKDvUpEGdL3nbmISDuCgoKEjY2NGDJkiLh27Zq4du2auH//vrCyshI9e/YU0dHRYvPmzcLJyUmMGTNGCCFEYmKiMDIyEmfOnBFCCDFjxgzh5OQkqlSpItXr5eUlFi9enOl2AYjNmzcLIYSIi4sTAESpUqXEjh07xPXr10WbNm2Eh4eHePv2rRBCiJMnTwojIyMxadIkcf36dTFz5kzh4OAg7O3tpToPHz4s7OzsxPLly0VsbKzYt2+f8PT0FD///LMQQoh3796JqlWripYtWwohhJgzZ45wcHAQd+7cyTDG5ORk0a5dO9GwYUMRHx8v4uPjRWpqapb2MxEJweSB6AsRFBQkAgICVMp++ukn4e3tLZRKpVQ2d+5cYWNjI9LS0oQQQpQvX1789ttvQgghWrZsKX799VdhZmYmkpOTxf379wUAcePGjUy3m1Hy8Pvvv0uvX7lyRQAQ0dHRQgghOnToIBo3bqxSR/v27VWShzp16ojw8HCVZVatWiVcXV2l57GxscLW1lYMGzZMWFpaitWrV39y/4SEhIgWLVp8chki0gxPWxB9QSpUqKDyPDo6GlWrVoVCoZDKAgMDkZKSgvv37wMAgoKCEBERASEEjhw5gtatW8PHxwdHjx5FZGQk3NzcUKJECVlx+Pn5Sf93dXUFADx+/FiKqUqVKirLV61aVeX5xYsXMW7cONjY2EiPbt26IT4+Hi9fvgQAFCtWDFOmTMGkSZPQvHlzdOzYUVaMRJR1HDBJ9AWxtraWvU6tWrWwdOlSXLx4EaampihVqhRq1aqFiIgIPHv2DEFBQbLrNDU1lf6fnrhkNBYhMykpKRg7dixat26t9pqFhYX0/8OHD8PY2Bi3b9/Gu3fvYGLCQxpRTmDPA9EXzMfHBydOnIAQQio7duwYbG1tUbhwYQBAjRo1kJycjOnTp0uJQnryEBERgVq1amk9plOnTqmUnTx5UuV5+fLlcf36dXh5eak9jIzeH7bWrVuHTZs2ISIiAnfv3sX48eM/uV0zMzNpYCgRZQ+TB6IvWM+ePXHv3j306dMH165dw9atWzFmzBgMHDhQ+hJ2dHSEn58fVq9eLSUKNWvWxPnz53Hjxo0s9Tx8St++fbFnzx5MmTIFMTExmDNnDvbs2aOyzOjRo7Fy5UqMHTsWV65cQXR0NNauXYv//e9/AID79+/jxx9/xKRJk1C9enUsW7YM4eHhaknIhzw9PfHPP//g+vXr+O+///D27VuttosoL2HyQPQFK1SoEHbt2oXTp0/D398fPXr0QJcuXaQv4XRBQUFIS0uTkod8+fKhdOnSKFiwILy9vbUa01dffYXFixdj5syZ8Pf3x759+9TiadCgAXbs2IF9+/ahUqVK+OqrrzB9+nR4eHhACIHQ0FBUrlwZvXv3lpb/8ccf8d133yElJSXD7Xbr1g3e3t6oWLEinJ2dcezYMa22iygvUYgP+zOJiIiIPoM9D0RERCQLkwciIiKShckDERERycLkgYiIiGRh8kBERESyMHkgIiIiWZg8EBERkSxMHoiIiEgWJg9EREQkC5MHIiIikoXJAxEREcny/wDUAhaeyDc90gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(6, 4))\n", + "width = 0.38\n", + "x = column0[\"t\"]\n", + "ax.bar(x - width / 2, column0[\"yedge_u[0,t]\"], width, label=\"yedge_u[0,t] (upward)\", color=\"tab:blue\")\n", + "ax.bar(x + width / 2, column0[\"yedge_d[0,t]\"], width, label=\"yedge_d[0,t] (downward)\", color=\"tab:orange\")\n", + "ax.set_xlabel(\"row index t\")\n", + "ax.set_ylabel(\"load (multicast trees crossing the link)\")\n", + "ax.set_title(\"Case F: per-edge load, column 0 vertical links (8x8 flood)\")\n", + "ax.set_xticks(list(x))\n", + "ax.grid(True, axis=\"y\", alpha=0.3)\n", + "ax.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "markdown", + "id": "9a7106d7", + "metadata": {}, + "source": [ + "## 4. Fulfilled vs. unfulfilled fills: the `_covered_fills` partition\n", + "\n", + "Node 0 holds only `data[0]`; node 1 requests both `data[0]` and `data[1]`.\n", + "`data[1]` has no source anywhere, so `identify_mesh_casts` never emits an\n", + "entry for it -- `_covered_fills` reshapes the matched multicast networks back\n", + "into a `{ dst -> data }` map so `apply` can split `fills` into\n", + "`fulfilled_fill` / `unfulfilled_fill` by `intersect` / `subtract`, and `hops`\n", + "counts only the deliverable fill (`data[0]`), not the undeliverable one.\n", + "Before `81be62a4` this partition did not exist: every fill was reported\n", + "`fulfilled_fill` unconditionally, with `unfulfilled_fill` always empty\n", + "(`fills.map_.subtract(fills.map_)`).\n", + "\n", + "**Code exercised**: `_covered_fills`\n", + "(`mesh_casts.py:114-137`, commit `81be62a4`, moved out of\n", + "`distributed_buffers.py` by the `/simplify` module split); the partition\n", + "usage itself is now the shared `MulticastModel.apply`\n", + "(`distributed_buffers.py:97-147`, commit `81be62a4`, folded from\n", + "`FullyConnectedMulticastModel.apply` and the other three models' identical\n", + "`apply` tails during the module split).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2527b397", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:25.946179Z", + "iopub.status.busy": "2026-07-06T20:54:25.946030Z", + "iopub.status.idle": "2026-07-06T20:54:25.952729Z", + "shell.execute_reply": "2026-07-06T20:54:25.951389Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fulfilled_fill.map_ : { noc[g = 1] -> data[d = 0] }\n", + "unfulfilled_fill.map_: { noc[g = 1] -> data[d = 1] }\n", + "hops : 1\n", + "\n", + "fulfilled_fill and unfulfilled_fill partition fills exactly;\n", + "hops (1) counts only the deliverable data[0], not the undeliverable data[1].\n" + ] + } + ], + "source": [ + "partition_tags = [SpatialTag(0, 0)]\n", + "occ_p = isl.Map.read_from_str(CTX, \"{ noc[g] -> data[d] : g = 0 and d = 0 }\")\n", + "fill_p = isl.Map.read_from_str(CTX, \"{ noc[g] -> data[d] : g = 1 and (d = 0 or d = 1) }\")\n", + "dist_fn_1d = isl.Map.read_from_str(\n", + " CTX,\n", + " \"{ [noc[gd] -> noc[gs]] -> hops[gd - gs] : gd >= gs;\"\n", + " \" [noc[gd] -> noc[gs]] -> hops[gs - gd] : gd < gs }\",\n", + ")\n", + "info_p = FullyConnectedMulticastModel(dist_fn_1d).apply(\n", + " 0, Fill(partition_tags, fill_p), Occupancy(partition_tags, occ_p)\n", + ")\n", + "\n", + "print(\"fulfilled_fill.map_ :\", info_p.fulfilled_fill.map_)\n", + "print(\"unfulfilled_fill.map_:\", info_p.unfulfilled_fill.map_)\n", + "print(\"hops :\", _eval_const(info_p.hops))\n", + "\n", + "fulfilled = info_p.fulfilled_fill.map_\n", + "unfulfilled = info_p.unfulfilled_fill.map_\n", + "assert fulfilled.intersect(unfulfilled).is_empty(), \"fulfilled/unfulfilled overlap\"\n", + "assert fulfilled.union(unfulfilled).is_equal(fill_p), \"fulfilled + unfulfilled != fills\"\n", + "assert _eval_const(info_p.hops) == 1, \"hops must count only the deliverable fill\"\n", + "print()\n", + "print(\"fulfilled_fill and unfulfilled_fill partition fills exactly;\")\n", + "print(\"hops (1) counts only the deliverable data[0], not the undeliverable data[1].\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "36a00572", + "metadata": {}, + "source": [ + "## 5. Tuple-name genericity: `_mesh_node_tuple`\n", + "\n", + "`_directed_mesh_links` and `_spoke_loads` used to hardcode the node tuple name\n", + "`'noc'` into every ISL map string they built. `_mesh_node_tuple` instead reads\n", + "the name (and dimensionality) off the caller's maps, so any spacetime tuple\n", + "name works. Below, the identical Case B geometry from Section 3 is run twice\n", + "--- once with the node tuple named `noc[x, y]`, once `pe[x, y]` --- and gives\n", + "identical hops and bottleneck.\n", + "\n", + "**Code exercised**: `_mesh_node_tuple`\n", + "(`mesh_casts.py:140-164`, commit `81be62a4`, moved out of\n", + "`distributed_buffers.py` by the `/simplify` module split); its caller\n", + "`XYRoutingMulticastModel._directed_mesh_links`\n", + "(`distributed_buffers.py:417-515`, commit `142722f5`, touched again in\n", + "`81be62a4`).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "f621fc81", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-06T20:54:25.955168Z", + "iopub.status.busy": "2026-07-06T20:54:25.955010Z", + "iopub.status.idle": "2026-07-06T20:54:25.969709Z", + "shell.execute_reply": "2026-07-06T20:54:25.968875Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "noc[x, y]: hops = 6, bottleneck = 1\n", + "pe[x, y]: hops = 6, bottleneck = 1\n", + "\n", + "Identical numbers under both tuple names -- _mesh_node_tuple reads the name/dims generically.\n" + ] + } + ], + "source": [ + "def xy_case_b(name: str) -> tuple[isl.Map, isl.Map, isl.Map]:\n", + " \"\"\"Case B geometry (source (1,0) -> (0,2),(2,2)) under an arbitrary node\n", + " tuple `name`, to exercise `_mesh_node_tuple`'s generic name/dims lookup.\n", + " \"\"\"\n", + " occ = isl.Map.read_from_str(\n", + " CTX, f\"{{ {name}[x, y] -> data[c] : c = 0 and x = 1 and y = 0 }}\"\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ {name}[x, y] -> data[c] : c = 0 and \"\n", + " f\"((x = 0 and y = 2) or (x = 2 and y = 2)) }}\",\n", + " )\n", + " dist_fn = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ [{name}[xd, yd] -> {name}[xs, ys]] -> hops[(xd - xs) + (yd - ys)] :\"\n", + " f\" xd >= xs and yd >= ys;\"\n", + " f\" [{name}[xd, yd] -> {name}[xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] :\"\n", + " f\" xd < xs and yd < ys;\"\n", + " f\" [{name}[xd, yd] -> {name}[xs, ys]] -> hops[-(xd - xs) + (yd - ys)] :\"\n", + " f\" xd < xs and yd >= ys;\"\n", + " f\" [{name}[xd, yd] -> {name}[xs, ys]] -> hops[(xd - xs) + -(yd - ys)] :\"\n", + " f\" xd >= xs and yd < ys }}\",\n", + " )\n", + " return occ, fill, dist_fn\n", + "\n", + "\n", + "results = {}\n", + "for name in (\"noc\", \"pe\"):\n", + " occ_n, fill_n, dist_fn_n = xy_case_b(name)\n", + " info_n = XYRoutingMulticastModel(dist_fn_n).apply(\n", + " 0, Fill(xy_tags, fill_n), Occupancy(xy_tags, occ_n)\n", + " )\n", + " results[name] = (_eval_const(info_n.hops), info_n.edge_pressure.bottleneck())\n", + " print(f\"{name}[x, y]: hops = {results[name][0]}, bottleneck = {results[name][1]}\")\n", + "\n", + "assert results[\"noc\"] == results[\"pe\"] == (6, 1), results\n", + "print()\n", + "print(\"Identical numbers under both tuple names -- _mesh_node_tuple reads the name/dims generically.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "270244d3", + "metadata": {}, + "source": [ + "## 6. Review checklist\n", + "\n", + "Which lines are new (and need a reviewer's attention) vs. which lines predate\n", + "this work, mapped by demo section. `HypercubeMulticastModel`,\n", + "`identify_mesh_casts`, and `calculate_extents_per_dim` are the pre-existing\n", + "baseline and are intentionally absent from this table (and from this notebook).\n", + "\n", + "| Section | Function(s) exercised | File | Lines | Commit |\n", + "|---|---|---|---|---|\n", + "| 2. FC vs. star | `EdgePressure` (class, `total`, `bottleneck`, `eval_edge`) | `edge_pressure.py` | 21-133 | `142722f5` |\n", + "| 2. FC vs. star | `MulticastModel.apply` (shared `apply`, all four models) | `distributed_buffers.py` | 97-147 | `81be62a4` |\n", + "| 2. FC vs. star | `FullyConnectedMulticastModel._transfer_cost` / `_cost_fully_connected` | `distributed_buffers.py` | 265-284 | `81be62a4` |\n", + "| 2. FC vs. star | `StarMulticastModel` (class, `_spoke_loads`) | `distributed_buffers.py` | 523-599 | `142722f5` |\n", + "| 2. FC vs. star | `StarMulticastModel._transfer_cost` (hops/edge_pressure aggregation, feeds the shared `apply`) | `distributed_buffers.py` | 546-547 | `81be62a4` |\n", + "| 2. FC vs. star | `TransferInfo.edge_pressure` field | `spatial.py` | 66-72 | `81be62a4` |\n", + "| 3. XY routing | `MulticastModel.apply` (shared `apply`) | `distributed_buffers.py` | 97-147 | `81be62a4` |\n", + "| 3. XY routing | `XYRoutingMulticastModel._transfer_cost` | `distributed_buffers.py` | 414-415 | `81be62a4` |\n", + "| 3. XY routing | `XYRoutingMulticastModel._directed_mesh_links` | `distributed_buffers.py` | 417-515 | `142722f5` (+`81be62a4`) |\n", + "| 3. XY routing | `EdgePressure.bottleneck` / `EdgePressure.eval_edge` | `edge_pressure.py` | 89-133 | `142722f5` |\n", + "| 4. Partition | `_covered_fills` | `mesh_casts.py` | 114-137 | `81be62a4` |\n", + "| 4. Partition | `MulticastModel.apply` (partition usage, shared) | `distributed_buffers.py` | 97-147 | `81be62a4` |\n", + "| 5. Tuple genericity | `_mesh_node_tuple` | `mesh_casts.py` | 140-164 | `81be62a4` |\n", + "| 5. Tuple genericity | `XYRoutingMulticastModel._directed_mesh_links` (caller) | `distributed_buffers.py` | 417-515 | `142722f5` (+`81be62a4`) |\n", + "| (all sections) | `_eval_const`, `_const_pwq` (shared scalar-extraction helpers) | `edge_pressure.py` | 159-201 | `81be62a4` |\n", + "| 2, 3 | `_edge_pressure_from_links` | `edge_pressure.py` | 230-251 | `142722f5` |\n", + "\n", + "Notes on mixed attribution: `XYRoutingMulticastModel.apply` and\n", + "`StarMulticastModel.apply` were first written in `de426ae0`/`142722f5` and had\n", + "their `hops`/`edge_pressure` derivation refactored to a single\n", + "`identify_mesh_casts` call in `81be62a4` (rows above cite `81be62a4` for that\n", + "reason). A later, behavior-preserving `/simplify` module split folded every\n", + "model's `apply` into one `MulticastModel.apply` (`distributed_buffers.py:\n", + "97-147`) and moved `EdgePressure`/`identify_mesh_casts` and their helpers into\n", + "`edge_pressure.py`/`mesh_casts.py` respectively; the table above cites that\n", + "current location rather than `apply` on each subclass, since the per-model\n", + "`apply` no longer exists post-split -- only each model's `_transfer_cost` (the\n", + "one-line cost-kernel hook `apply` now calls) and cost-specific helper remain\n", + "on the subclass. `FullyConnectedMulticastModel`'s cost kernel\n", + "(`_cost_fully_connected`) is unchanged since `de426ae0` and predates the\n", + "reviewed range; only its fulfilled/unfulfilled partition usage (now inside\n", + "the shared `MulticastModel.apply`) is new in `81be62a4`.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "accelforge", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 96fc3954d02151ada9cfa5bb880cd12f07290618 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Mon, 6 Jul 2026 19:17:34 -0400 Subject: [PATCH 16/19] moved filed and fixed tiling issue --- .../_looptree/reuse/isl/distributed/README.md | 24 +- .../reuse/isl/mapping_to_isl/tiling.py | 3 +- .../distributed_models_demo.ipynb | 76 +-- .../fully_connected/test_cases.yaml | 0 .../distributed}/helpers.py | 11 +- .../isl/distributed/multicast/test_cases.yaml | 119 +++- .../distributed}/test_contract.py | 0 .../distributed}/test_edge_pressure.py | 0 .../distributed}/test_fully_connected.py | 0 .../distributed}/test_multicast.py | 0 .../distributed}/test_xy_routing.py | 0 .../distributed}/xy_routing/test_cases.yaml | 0 .../distribuffers => isl/mapper}/__init__.py | 0 .../mapper/configs/conv1d/conv1d.mapping.yaml | 0 .../configs/conv1d/conv1d.workload.yaml | 0 .../two_conv1d/two_conv1d.expected.yaml | 0 .../two_conv1d/two_conv1d.mapping.yaml | 0 .../two_conv1d/two_conv1d.workload.yaml | 0 .../isl/mapper/test_isl_functions.py | 0 .../isl/mapper/test_mapping_to_isl.py | 0 .../isl/mapper/test_spatial_reuse_analysis.py | 0 .../mapper/test_temporal_reuse_analysis.py | 0 .../distribuffers/multicast/test_cases.yaml | 599 ------------------ .../spec/binding/valid_bindings.yaml | 91 --- .../distribuffers/spec/distributed.yaml | 100 --- .../distribuffers/spec/logical_arch.yaml | 32 - .../distribuffers/spec/physical_arch.yaml | 69 -- .../not_working/distribuffers/test_binding.py | 49 -- tests/not_working/isl/mapper/__init__.py | 0 29 files changed, 176 insertions(+), 997 deletions(-) rename tests/{not_working/distribuffers => isl/distributed}/fully_connected/test_cases.yaml (100%) rename tests/{not_working/distribuffers => isl/distributed}/helpers.py (94%) rename tests/{not_working/distribuffers => isl/distributed}/test_contract.py (100%) rename tests/{not_working/distribuffers => isl/distributed}/test_edge_pressure.py (100%) rename tests/{not_working/distribuffers => isl/distributed}/test_fully_connected.py (100%) rename tests/{not_working/distribuffers => isl/distributed}/test_multicast.py (100%) rename tests/{not_working/distribuffers => isl/distributed}/test_xy_routing.py (100%) rename tests/{not_working/distribuffers => isl/distributed}/xy_routing/test_cases.yaml (100%) rename tests/{not_working/distribuffers => isl/mapper}/__init__.py (100%) rename tests/{not_working => }/isl/mapper/configs/conv1d/conv1d.mapping.yaml (100%) rename tests/{not_working => }/isl/mapper/configs/conv1d/conv1d.workload.yaml (100%) rename tests/{not_working => }/isl/mapper/configs/two_conv1d/two_conv1d.expected.yaml (100%) rename tests/{not_working => }/isl/mapper/configs/two_conv1d/two_conv1d.mapping.yaml (100%) rename tests/{not_working => }/isl/mapper/configs/two_conv1d/two_conv1d.workload.yaml (100%) rename tests/{not_working => }/isl/mapper/test_isl_functions.py (100%) rename tests/{not_working => }/isl/mapper/test_mapping_to_isl.py (100%) rename tests/{not_working => }/isl/mapper/test_spatial_reuse_analysis.py (100%) rename tests/{not_working => }/isl/mapper/test_temporal_reuse_analysis.py (100%) delete mode 100755 tests/not_working/distribuffers/multicast/test_cases.yaml delete mode 100755 tests/not_working/distribuffers/spec/binding/valid_bindings.yaml delete mode 100755 tests/not_working/distribuffers/spec/distributed.yaml delete mode 100755 tests/not_working/distribuffers/spec/logical_arch.yaml delete mode 100755 tests/not_working/distribuffers/spec/physical_arch.yaml delete mode 100755 tests/not_working/distribuffers/test_binding.py delete mode 100644 tests/not_working/isl/mapper/__init__.py diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index a3936169..cdd8c561 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -25,8 +25,8 @@ implementations: > **There is no registry or factory.** Models are constructed directly > (`HypercubeMulticastModel(dist_fn)`) and applied via `.apply(...)`. The only current usage is the -> test suite — [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py). -> The `not_working/` location signals these models are work-in-progress. +> test suite — [`tests/isl/distributed/test_multicast.py`](../../../../../../tests/isl/distributed/test_multicast.py). +> The suite lives at `tests/isl/distributed/` and runs as part of CI. > **Module layout.** This directory splits into three files: the four model classes above and > the shared `MulticastModel` base they all inherit (one `apply()`, one abstract `_transfer_cost` @@ -255,7 +255,7 @@ So these two models bracket the design space: **stateful + distance-aware** (hyp left to you: build the cost over the correct domain, and decide deliberately whether your topology defines a per-link decomposition or not. 5. **Add a test** mirroring - [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py): + [`tests/isl/distributed/test_multicast.py`](../../../../../../tests/isl/distributed/test_multicast.py): a YAML-driven gamut of `(dims, fill, occ, dist_fn, expected_hops)` cases. ### Copy-paste skeleton @@ -347,14 +347,14 @@ model incurs on a fully-connected fabric (each unicast costed as a `(1+1)(1+1) instead of a single crossing — see the discrepancy in §3). This example is exercised by a real test: -[`tests/not_working/distribuffers/test_fully_connected.py`](../../../../../../tests/not_working/distribuffers/test_fully_connected.py) +[`tests/isl/distributed/test_fully_connected.py`](../../../../../../tests/isl/distributed/test_fully_connected.py) with cases in -[`tests/not_working/distribuffers/fully_connected/test_cases.yaml`](../../../../../../tests/not_working/distribuffers/fully_connected/test_cases.yaml). +[`tests/isl/distributed/fully_connected/test_cases.yaml`](../../../../../../tests/isl/distributed/fully_connected/test_cases.yaml). Run it (m4 on `PATH` per the islpy-barvinok setup): ```bash PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ - tests/not_working/distribuffers/test_fully_connected.py -q + tests/isl/distributed/test_fully_connected.py -q ``` --- @@ -428,16 +428,16 @@ Notes / limitations: ### Tested -[`tests/not_working/distribuffers/test_xy_routing.py`](../../../../../../tests/not_working/distribuffers/test_xy_routing.py) +[`tests/isl/distributed/test_xy_routing.py`](../../../../../../tests/isl/distributed/test_xy_routing.py) with hand-derived cases in -[`tests/not_working/distribuffers/xy_routing/test_cases.yaml`](../../../../../../tests/not_working/distribuffers/xy_routing/test_cases.yaml) +[`tests/isl/distributed/xy_routing/test_cases.yaml`](../../../../../../tests/isl/distributed/xy_routing/test_cases.yaml) (there is **no XY oracle in the repo**, so the expected values are hand-derived and each case carries its geometry). Cases: unicast `4`, the `(1,0)` discriminator `6`, three-corner `6`, 1-D column `3`, replicated-source `4`, and an 8×8 scale case `448`. Run: ```bash PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ - tests/not_working/distribuffers/test_xy_routing.py -q + tests/isl/distributed/test_xy_routing.py -q ``` --- @@ -503,7 +503,7 @@ deliveries). ### Tested -[`tests/not_working/distribuffers/test_edge_pressure.py`](../../../../../../tests/not_working/distribuffers/test_edge_pressure.py) +[`tests/isl/distributed/test_edge_pressure.py`](../../../../../../tests/isl/distributed/test_edge_pressure.py) — the XY `Σ load == hops` invariant over A–F (with the pressure taken from `apply(...).edge_pressure`), F's bottleneck/edge loads (`7`, `yedge_u[0,6]=7`, `yedge_d[0,1]=6`), single-tree unit bottlenecks, and the star spoke loads / `Σ ingress == FC count` invariant for 4- @@ -512,7 +512,7 @@ and 8-GPU all-to-all (star `hops` = injections + deliveries: `16`/`64`, ingress ```bash PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ - tests/not_working/distribuffers/test_edge_pressure.py -q + tests/isl/distributed/test_edge_pressure.py -q ``` --- @@ -528,4 +528,4 @@ PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest \ [`edge_pressure.py`](edge_pressure.py) - Tagged-map / tag types: [`../mapping_to_isl/types.py`](../mapping_to_isl/types.py) - Example test harness: - [`tests/not_working/distribuffers/test_multicast.py`](../../../../../../tests/not_working/distribuffers/test_multicast.py) + [`tests/isl/distributed/test_multicast.py`](../../../../../../tests/isl/distributed/test_multicast.py) diff --git a/accelforge/model/_looptree/reuse/isl/mapping_to_isl/tiling.py b/accelforge/model/_looptree/reuse/isl/mapping_to_isl/tiling.py index 7da28fed..122d4219 100644 --- a/accelforge/model/_looptree/reuse/isl/mapping_to_isl/tiling.py +++ b/accelforge/model/_looptree/reuse/isl/mapping_to_isl/tiling.py @@ -345,8 +345,9 @@ def consumer_based_tile_shape_inference( # For each tensor read by this einsum, tile that tensor's producers. for tensor in workload.einsums[einsum].input_tensor_names: + # The einsums that write `tensor` -- its producers in this workload. producer_einsums: oset[EinsumName] = oset( - [e for e in workload.einsums[einsum].output_tensor_names] + [e.name for e in workload.einsums if tensor in e.output_tensor_names] ) if len(producer_einsums) > 1: raise NotImplementedError( diff --git a/notebooks/astrasim2_correlation/distributed_models_demo.ipynb b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb index e2165373..368c00b4 100644 --- a/notebooks/astrasim2_correlation/distributed_models_demo.ipynb +++ b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb @@ -77,10 +77,10 @@ "id": "e3d6abf5", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:20.903056Z", - "iopub.status.busy": "2026-07-06T20:54:20.902882Z", - "iopub.status.idle": "2026-07-06T20:54:22.202475Z", - "shell.execute_reply": "2026-07-06T20:54:22.201741Z" + "iopub.execute_input": "2026-07-06T22:33:22.330405Z", + "iopub.status.busy": "2026-07-06T22:33:22.330272Z", + "iopub.status.idle": "2026-07-06T22:33:23.645047Z", + "shell.execute_reply": "2026-07-06T22:33:23.644422Z" } }, "outputs": [], @@ -145,10 +145,10 @@ "id": "cd3dd43b", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:22.205911Z", - "iopub.status.busy": "2026-07-06T20:54:22.205521Z", - "iopub.status.idle": "2026-07-06T20:54:22.214578Z", - "shell.execute_reply": "2026-07-06T20:54:22.213790Z" + "iopub.execute_input": "2026-07-06T22:33:23.647609Z", + "iopub.status.busy": "2026-07-06T22:33:23.647337Z", + "iopub.status.idle": "2026-07-06T22:33:23.655758Z", + "shell.execute_reply": "2026-07-06T22:33:23.655106Z" } }, "outputs": [], @@ -231,10 +231,10 @@ "id": "99287f8f", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:22.216427Z", - "iopub.status.busy": "2026-07-06T20:54:22.216263Z", - "iopub.status.idle": "2026-07-06T20:54:23.258718Z", - "shell.execute_reply": "2026-07-06T20:54:23.257609Z" + "iopub.execute_input": "2026-07-06T22:33:23.658028Z", + "iopub.status.busy": "2026-07-06T22:33:23.657872Z", + "iopub.status.idle": "2026-07-06T22:33:24.595478Z", + "shell.execute_reply": "2026-07-06T22:33:24.594519Z" } }, "outputs": [ @@ -265,10 +265,10 @@ "id": "da1b83a5", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:23.262837Z", - "iopub.status.busy": "2026-07-06T20:54:23.262627Z", - "iopub.status.idle": "2026-07-06T20:54:25.768785Z", - "shell.execute_reply": "2026-07-06T20:54:25.768287Z" + "iopub.execute_input": "2026-07-06T22:33:24.598198Z", + "iopub.status.busy": "2026-07-06T22:33:24.597829Z", + "iopub.status.idle": "2026-07-06T22:33:26.845318Z", + "shell.execute_reply": "2026-07-06T22:33:26.844213Z" } }, "outputs": [ @@ -335,7 +335,7 @@ "one source is a rigid tree: an X segment along the source row out to every\n", "destination column, then an independent Y segment down each column from the\n", "source row. Two geometries from\n", - "`tests/not_working/distribuffers/xy_routing/test_cases.yaml` (already\n", + "`tests/isl/distributed/xy_routing/test_cases.yaml` (already\n", "oracle-verified there; re-verified here):\n", "\n", "- **Case B** --- source $(1,0)$ casting to $(0,2)$ and $(2,2)$: a single tree,\n", @@ -352,7 +352,7 @@ "(`distributed_buffers.py:417-515`, commit `142722f5`, touched again in\n", "`81be62a4`); `EdgePressure.bottleneck` / `EdgePressure.eval_edge`\n", "(`edge_pressure.py:89-133`, commit `142722f5`, moved out of\n", - "`distributed_buffers.py` by the `/simplify` module split).\n" + "`distributed_buffers.py` by the `/simplify` module split)." ] }, { @@ -361,10 +361,10 @@ "id": "d6c4b869", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:25.770948Z", - "iopub.status.busy": "2026-07-06T20:54:25.770573Z", - "iopub.status.idle": "2026-07-06T20:54:25.791882Z", - "shell.execute_reply": "2026-07-06T20:54:25.791233Z" + "iopub.execute_input": "2026-07-06T22:33:26.847779Z", + "iopub.status.busy": "2026-07-06T22:33:26.847607Z", + "iopub.status.idle": "2026-07-06T22:33:26.864876Z", + "shell.execute_reply": "2026-07-06T22:33:26.864027Z" } }, "outputs": [ @@ -502,10 +502,10 @@ "id": "8a0aa2e7", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:25.796321Z", - "iopub.status.busy": "2026-07-06T20:54:25.796094Z", - "iopub.status.idle": "2026-07-06T20:54:25.814246Z", - "shell.execute_reply": "2026-07-06T20:54:25.813660Z" + "iopub.execute_input": "2026-07-06T22:33:26.867413Z", + "iopub.status.busy": "2026-07-06T22:33:26.867246Z", + "iopub.status.idle": "2026-07-06T22:33:26.884336Z", + "shell.execute_reply": "2026-07-06T22:33:26.883695Z" } }, "outputs": [ @@ -648,10 +648,10 @@ "id": "4b1bc6a2", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:25.816247Z", - "iopub.status.busy": "2026-07-06T20:54:25.816074Z", - "iopub.status.idle": "2026-07-06T20:54:25.944569Z", - "shell.execute_reply": "2026-07-06T20:54:25.943840Z" + "iopub.execute_input": "2026-07-06T22:33:26.886357Z", + "iopub.status.busy": "2026-07-06T22:33:26.886187Z", + "iopub.status.idle": "2026-07-06T22:33:26.992550Z", + "shell.execute_reply": "2026-07-06T22:33:26.991894Z" } }, "outputs": [ @@ -713,10 +713,10 @@ "id": "2527b397", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:25.946179Z", - "iopub.status.busy": "2026-07-06T20:54:25.946030Z", - "iopub.status.idle": "2026-07-06T20:54:25.952729Z", - "shell.execute_reply": "2026-07-06T20:54:25.951389Z" + "iopub.execute_input": "2026-07-06T22:33:26.994366Z", + "iopub.status.busy": "2026-07-06T22:33:26.994211Z", + "iopub.status.idle": "2026-07-06T22:33:26.999684Z", + "shell.execute_reply": "2026-07-06T22:33:26.999149Z" } }, "outputs": [ @@ -788,10 +788,10 @@ "id": "f621fc81", "metadata": { "execution": { - "iopub.execute_input": "2026-07-06T20:54:25.955168Z", - "iopub.status.busy": "2026-07-06T20:54:25.955010Z", - "iopub.status.idle": "2026-07-06T20:54:25.969709Z", - "shell.execute_reply": "2026-07-06T20:54:25.968875Z" + "iopub.execute_input": "2026-07-06T22:33:27.001244Z", + "iopub.status.busy": "2026-07-06T22:33:27.001102Z", + "iopub.status.idle": "2026-07-06T22:33:27.014016Z", + "shell.execute_reply": "2026-07-06T22:33:27.013407Z" } }, "outputs": [ diff --git a/tests/not_working/distribuffers/fully_connected/test_cases.yaml b/tests/isl/distributed/fully_connected/test_cases.yaml similarity index 100% rename from tests/not_working/distribuffers/fully_connected/test_cases.yaml rename to tests/isl/distributed/fully_connected/test_cases.yaml diff --git a/tests/not_working/distribuffers/helpers.py b/tests/isl/distributed/helpers.py similarity index 94% rename from tests/not_working/distribuffers/helpers.py rename to tests/isl/distributed/helpers.py index a190c9dd..78de3e3d 100644 --- a/tests/not_working/distribuffers/helpers.py +++ b/tests/isl/distributed/helpers.py @@ -24,7 +24,7 @@ a local copy. That import only resolves if the repository root is on ``sys.path``, which is the case when the suite is invoked from the repo root as - PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest tests/not_working/distribuffers/ -q + PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest tests/isl/distributed/ -q (pytest inserts the current working directory / rootdir onto ``sys.path``). Running this module or its dependents from a different working directory, or @@ -58,10 +58,11 @@ ) from accelforge.model._looptree.reuse.isl.spatial import TransferInfo, TransferModel -# Design: `tests/not_working/distribuffers/util.py` used to be a byte-identical -# copy of `tests/isl/util.py`. Rather than keep two copies of `load_solutions` -# in sync by hand, this package now imports the canonical one directly -- see -# the "Import note" above for the run-from-repo-root requirement this implies. +# Design: this package (formerly `tests/not_working/distribuffers/`) used to +# carry a byte-identical copy of `tests/isl/util.py` as its own `util.py`. +# Rather than keep two copies of `load_solutions` in sync by hand, it imports +# the canonical one directly -- see the "Import note" above for the +# run-from-repo-root requirement this implies. from tests.isl.util import load_solutions diff --git a/tests/isl/distributed/multicast/test_cases.yaml b/tests/isl/distributed/multicast/test_cases.yaml index 32dbc8eb..00afc2e0 100755 --- a/tests/isl/distributed/multicast/test_cases.yaml +++ b/tests/isl/distributed/multicast/test_cases.yaml @@ -479,4 +479,121 @@ # dist_fn: *ring_dist_size_8 # expected: # latency: 1 -# total_hops: 4 \ No newline at end of file +# total_hops: 4 +################################################### +# 8-GPU fully-connected (NVLink/NVSwitch-style) # +# all-to-all, one-hot GPU encoding. # +# # +# GPU i sits at one-hot coordinate e_i so every # +# src!=dst cast has extent 1 along exactly the # +# src and dst dims: cost = (1+1)(1+1)-1 = 3, # +# uniform across all pairs (fully-connected). # +# Self-chunks never cross the fabric (cost 0). # +# dist_fn is unit-cost (matching only). # +################################################### + +# All-to-all over 8 GPUs: each GPU holds chunks data[self, d] and requests data[s, self]. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and 0 <= d < 8 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and 0 <= s < 8 + } + dims: &8d_onehot_spatial + - type: Spatial + spatial_dim: 0 + target: 0 + - type: Spatial + spatial_dim: 1 + target: 0 + - type: Spatial + spatial_dim: 2 + target: 0 + - type: Spatial + spatial_dim: 3 + target: 0 + - type: Spatial + spatial_dim: 4 + target: 0 + - type: Spatial + spatial_dim: 5 + target: 0 + - type: Spatial + spatial_dim: 6 + target: 0 + - type: Spatial + spatial_dim: 7 + target: 0 + dist_fn: &fully_connected_unit | + { + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[0] : + xd0 = xs0 and xd1 = xs1 and xd2 = xs2 and xd3 = xs3 and xd4 = xs4 and xd5 = xs5 and xd6 = xs6 and xd7 = xs7; + [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[1] : + (xd0 < xs0) or (xd0 > xs0) or (xd1 < xs1) or (xd1 > xs1) or (xd2 < xs2) or (xd2 > xs2) or (xd3 < xs3) or (xd3 > xs3) or (xd4 < xs4) or (xd4 > xs4) or (xd5 < xs5) or (xd5 > xs5) or (xd6 < xs6) or (xd6 > xs6) or (xd7 < xs7) or (xd7 > xs7) + } + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 168 + extent_DOR_hops: null + +# Single chunk GPU0 -> GPU3: one unicast cast, cost (1+1)(1+1)-1 = 3. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 0 and d = 3 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 0 and d = 3 + } + dims: *8d_onehot_spatial + + dist_fn: *fully_connected_unit + + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 3 + extent_DOR_hops: null + +# Self chunk GPU5 -> GPU5: never crosses the fabric, cost 0. +- occ: | + { + noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : + 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and + gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and + s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 5 and d = 5 + } + fill: | + { + noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : + 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and + gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and + d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 5 and d = 5 + } + dims: *8d_onehot_spatial + + dist_fn: *fully_connected_unit + + expected: + latency: null + total_hops: null + multicast_hops: null + hypercube_hops: 0 + extent_DOR_hops: null diff --git a/tests/not_working/distribuffers/test_contract.py b/tests/isl/distributed/test_contract.py similarity index 100% rename from tests/not_working/distribuffers/test_contract.py rename to tests/isl/distributed/test_contract.py diff --git a/tests/not_working/distribuffers/test_edge_pressure.py b/tests/isl/distributed/test_edge_pressure.py similarity index 100% rename from tests/not_working/distribuffers/test_edge_pressure.py rename to tests/isl/distributed/test_edge_pressure.py diff --git a/tests/not_working/distribuffers/test_fully_connected.py b/tests/isl/distributed/test_fully_connected.py similarity index 100% rename from tests/not_working/distribuffers/test_fully_connected.py rename to tests/isl/distributed/test_fully_connected.py diff --git a/tests/not_working/distribuffers/test_multicast.py b/tests/isl/distributed/test_multicast.py similarity index 100% rename from tests/not_working/distribuffers/test_multicast.py rename to tests/isl/distributed/test_multicast.py diff --git a/tests/not_working/distribuffers/test_xy_routing.py b/tests/isl/distributed/test_xy_routing.py similarity index 100% rename from tests/not_working/distribuffers/test_xy_routing.py rename to tests/isl/distributed/test_xy_routing.py diff --git a/tests/not_working/distribuffers/xy_routing/test_cases.yaml b/tests/isl/distributed/xy_routing/test_cases.yaml similarity index 100% rename from tests/not_working/distribuffers/xy_routing/test_cases.yaml rename to tests/isl/distributed/xy_routing/test_cases.yaml diff --git a/tests/not_working/distribuffers/__init__.py b/tests/isl/mapper/__init__.py similarity index 100% rename from tests/not_working/distribuffers/__init__.py rename to tests/isl/mapper/__init__.py diff --git a/tests/not_working/isl/mapper/configs/conv1d/conv1d.mapping.yaml b/tests/isl/mapper/configs/conv1d/conv1d.mapping.yaml similarity index 100% rename from tests/not_working/isl/mapper/configs/conv1d/conv1d.mapping.yaml rename to tests/isl/mapper/configs/conv1d/conv1d.mapping.yaml diff --git a/tests/not_working/isl/mapper/configs/conv1d/conv1d.workload.yaml b/tests/isl/mapper/configs/conv1d/conv1d.workload.yaml similarity index 100% rename from tests/not_working/isl/mapper/configs/conv1d/conv1d.workload.yaml rename to tests/isl/mapper/configs/conv1d/conv1d.workload.yaml diff --git a/tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.expected.yaml b/tests/isl/mapper/configs/two_conv1d/two_conv1d.expected.yaml similarity index 100% rename from tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.expected.yaml rename to tests/isl/mapper/configs/two_conv1d/two_conv1d.expected.yaml diff --git a/tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.mapping.yaml b/tests/isl/mapper/configs/two_conv1d/two_conv1d.mapping.yaml similarity index 100% rename from tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.mapping.yaml rename to tests/isl/mapper/configs/two_conv1d/two_conv1d.mapping.yaml diff --git a/tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.workload.yaml b/tests/isl/mapper/configs/two_conv1d/two_conv1d.workload.yaml similarity index 100% rename from tests/not_working/isl/mapper/configs/two_conv1d/two_conv1d.workload.yaml rename to tests/isl/mapper/configs/two_conv1d/two_conv1d.workload.yaml diff --git a/tests/not_working/isl/mapper/test_isl_functions.py b/tests/isl/mapper/test_isl_functions.py similarity index 100% rename from tests/not_working/isl/mapper/test_isl_functions.py rename to tests/isl/mapper/test_isl_functions.py diff --git a/tests/not_working/isl/mapper/test_mapping_to_isl.py b/tests/isl/mapper/test_mapping_to_isl.py similarity index 100% rename from tests/not_working/isl/mapper/test_mapping_to_isl.py rename to tests/isl/mapper/test_mapping_to_isl.py diff --git a/tests/not_working/isl/mapper/test_spatial_reuse_analysis.py b/tests/isl/mapper/test_spatial_reuse_analysis.py similarity index 100% rename from tests/not_working/isl/mapper/test_spatial_reuse_analysis.py rename to tests/isl/mapper/test_spatial_reuse_analysis.py diff --git a/tests/not_working/isl/mapper/test_temporal_reuse_analysis.py b/tests/isl/mapper/test_temporal_reuse_analysis.py similarity index 100% rename from tests/not_working/isl/mapper/test_temporal_reuse_analysis.py rename to tests/isl/mapper/test_temporal_reuse_analysis.py diff --git a/tests/not_working/distribuffers/multicast/test_cases.yaml b/tests/not_working/distribuffers/multicast/test_cases.yaml deleted file mode 100755 index 00afc2e0..00000000 --- a/tests/not_working/distribuffers/multicast/test_cases.yaml +++ /dev/null @@ -1,599 +0,0 @@ -- occ: "{ noc[xs, ys] -> data[d0, d1] : d0=xs and d1=ys and 0 <= xs < 8 and 0 <= ys < 8 }" - fill: "{ noc[xd, yd] -> data[d0, d1] : d0=xd and 0 <= d1 < 8 and 0 <= xd < 8 and 0 <= yd < 8 }" - dims: &2d_spatial - - type: Spatial - spatial_dim: 0 - target: 0 - - type: Spatial - spatial_dim: 1 - target: 0 - dist_fn: &2d_manhattan | - { - [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + (yd - ys)] : - xd >= xs and yd >= ys; - [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] : - xd < xs and yd < ys; - [noc[xd, yd] -> noc[xs, ys]] -> hops[-(xd - xs) + (yd - ys)] : - xd < xs and yd >= ys; - [noc[xd, yd] -> noc[xs, ys]] -> hops[(xd - xs) + -(yd - ys)] : - xd >= xs and yd < ys - } - expected: &equivalent_class_1 - latency: 7 - total_hops: 1344 - multicast_hops: 448 - hypercube_hops: 448 - extent_DOR_hops: 448 -- occ: "{ noc[xs, ys] -> data[d0, d1] : d0=xs and d1=ys and 0 <= xs < 8 and 0 <= ys < 8 }" - fill: "{ noc[xd, yd] -> data[d0, d1] : 0 <= d0 < 8 and d1=yd and 0 <= xd < 8 and 0 <= yd < 8 }" - dims: *2d_spatial - dist_fn: *2d_manhattan - expected: *equivalent_class_1 -- occ: "{ noc[xs, ys] -> data[d0, d1] : d0 = xs and 0 <= d1 < 8 and 0 <= xs < 8 and 0 <= ys < 8 }" - fill: "{ noc[xd, yd] -> data[d0, d1] : d0 = xd and d1 = yd and 0 <= xd < 8 and 0 <= yd < 8 }" - dims: *2d_spatial - dist_fn: *2d_manhattan - expected: &equivalent_class_2 - latency: 0 - total_hops: 0 - multicast_hops: 0 - hypercube_hops: 0 - extent_DOR_hops: 448 -- occ: "{ noc[xs, ys] -> data[d0, d1] : 0 <= d0 < 8 and d1 = ys and 0 <= xs < 8 and 0 <= ys < 8 }" - fill: "{ noc[xd, yd] -> data[d0, d1] : d0 = xd and d1 = yd and 0 <= xd < 8 and 0 <= yd < 8 }" - dims: *2d_spatial - dist_fn: *2d_manhattan - expected: *equivalent_class_2 - -####################### -# Temporal Test Cases # -####################### -##@section 1x1 dummy case to catch temporal integration issues. -# Edge to Src -- occ: &1x1_edge | - { - noc[tm, tn, xs, ys] -> A[m, k] : - 0 <= tm < 1 and 0 <= tn < 1 and - xs=0 and ys=0 and - 0 <= m < 1 and 0 <= k < 1 - } - fill: | - { - noc[tm, tn, xd, yd] -> A[m, k] : - 0 <= tm < 1 and 0 <= tn < 1 and tn = 0 and - 0 <= xd < 1 and 0 <= yd < 1 and - m = (1 * tm) + xd and 0 <= k < 1 - } - dims: &2t-dim_2s-dim - - type: Temporal - - type: Temporal - - type: Spatial - spatial_dim: 0 - target: 0 - - type: Spatial - spatial_dim: 1 - target: 0 - dist_fn: &2t-dim_2s-dim_manhattan | - { - [noc[tm, tn, xd, yd] -> noc[tm, tn, xs, ys]] -> hops[(xd - xs) + (yd - ys)] : - xd >= xs and yd >= ys; - [noc[tm, tn, xd, yd] -> noc[tm, tn, xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] : - xd < xs and yd < ys; - [noc[tm, tn, xd, yd] -> noc[tm, tn, xs, ys]] -> hops[-(xd - xs) + (yd - ys)] : - xd < xs and yd >= ys; - [noc[tm, tn, xd, yd] -> noc[tm, tn, xs, ys]] -> hops[(xd - xs) + -(yd - ys)] : - xd >= xs and yd < ys - } - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 0 - extent_DOR_hops: null -# Src to Dst -- occ: | - { - noc[tm, tn, tk, xs, ys] -> A[m, k] : - 0 <= tm < 1 and 0 <= tn < 1 and 0 <= tk < 1 and - 0 <= xs < 1 and 0 <= ys < 1 and - m= (1 * tm) + xs and 0 <= k < 1 - } - fill: &1x1_fill | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 1 and 0 <= tn < 1 and 0 <= tk < 1 and - 0 <= xd < 1 and 0 <= yd < 1 and - m= (1 * tm) + xd and k=tk - } - dims: &3t-dim_2s-dim - - type: Temporal - - type: Temporal - - type: Temporal - - type: Spatial - spatial_dim: 0 - target: 0 - - type: Spatial - spatial_dim: 1 - target: 0 - dist_fn: &3t-dim_2s-dim_manhattan | - { - [noc[tm, tn, tk, xd, yd] -> noc[tm, tn, tk, xs, ys]] -> hops[(xd - xs) + (yd - ys)] : - xd >= xs and yd >= ys; - [noc[tm, tn, tk, xd, yd] -> noc[tm, tn, tk, xs, ys]] -> hops[-(xd - xs) + -(yd - ys)] : - xd < xs and yd < ys; - [noc[tm, tn, tk, xd, yd] -> noc[tm, tn, tk, xs, ys]] -> hops[-(xd - xs) + (yd - ys)] : - xd < xs and yd >= ys; - [noc[tm, tn, tk, xd, yd] -> noc[tm, tn, tk, xs, ys]] -> hops[(xd - xs) + -(yd - ys)] : - xd >= xs and yd < ys - } - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 0 - extent_DOR_hops: null -##@section 8x8 case for a bigger chip. -##@brief Full duplication to make sure temporal handling is correct. -# Edge to Src -- occ: &8x8_edge | - { - noc[tm, tn, xs, ys] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and - xs=0 and ys=0 and - 0 <= m < 64 and 0 <= k < 64 - } - fill: | - { - noc[tm, tn, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and tn = 0 and - 0 <= xd < 8 and 0 <= yd < 8 and - m = (8 * tm) + xd and 0 <= k < 64 - } - dims: *2t-dim_2s-dim - dist_fn: *2t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 143360 - extent_DOR_hops: null -# Src to Dst -- occ: | - { - noc[tm, tn, tk, xs, ys] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and 0 <= tk < 64 and - 0 <= xs < 8 and 0 <= ys < 8 and - m = (8 * tm) + xs and 0 <= k < 64 - } - fill: &8x8_pe | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and 0 <= tk < 64 and - 0 <= xd < 8 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and k=tk - } - dims: *3t-dim_2s-dim - dist_fn: *3t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 0 - extent_DOR_hops: null -##@brief No duplication, one time step to ensure spatial handling is correct. -# Edge to Src, only 1 time step. -- occ: *8x8_edge - fill: | - { - noc[tm, tn, xd, yd] -> A[m, k] : - 0 <= tm < 8 and tm = 0 and 0 <= tn < 8 and tn = 0 and - 0 <= xd < 8 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - dims: *2t-dim_2s-dim - dist_fn: *2t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 9856 - extent_DOR_hops: null -# Src to Dst, only 1 time step. -- occ: | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and tm = 0 and 0 <= tn < 8 and tn = 0 and 0 <= tk < 64 and - 0 <= xd < 8 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - fill: | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and tm = 0 and 0 <= tn < 8 and tn = 0 and 0 <= tk < 64 and tk = 0 and - 0 <= xd < 8 and 0 <= yd < 8 and - m = (8 * tm) + xd and k=tk - } - dims: *3t-dim_2s-dim - dist_fn: *3t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 56 - extent_DOR_hops: null -##@brief No duplication, one spatial dimension to ensure temporal handling is correct. -##@note, tn = 0 here because otherwise we recast every tn, which is not fundamental to the problem. -# Edge to Src, only 1 spatial dimension. -- occ: *8x8_edge - fill: | - { - noc[tm, tn, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and tn = 0 and - 0 <= xd < 8 and xd = 0 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - dims: *2t-dim_2s-dim - dist_fn: *2t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 1792 - extent_DOR_hops: null -# Src to Dst, only 1 spatial dimension. -- occ: | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and 0 <= tk < 64 and - 0 <= xd < 8 and xd = 0 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - fill: | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and 0 <= tk < 64 and - 0 <= xd < 8 and xd = 0 and 0 <= yd < 8 and - m = (8 * tm) + xd and k=tk - } - dims: *3t-dim_2s-dim - dist_fn: *3t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 28672 - extent_DOR_hops: null -##@brief No duplication, full chip to ensure temporal and spatial handling is correct. -# Edge to Src, full chip. -- occ: *8x8_edge - fill: | - { - noc[tm, tn, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and tn = 0 and - 0 <= xd < 8 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - dims: *2t-dim_2s-dim - dist_fn: *2t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 78848 - extent_DOR_hops: null -# Src to Dst, full chip. -- occ: | - { - noc[tm, tn, tk, xd, yd] -> A[m, k] : - 0 <= tm < 8 and 0 <= tn < 8 and 0 <= tk < 64 and - 0 <= xd < 8 and 0 <= yd < 8 and - 0 <= m < 64 and m = (8 * tm) + xd and - 0 <= k < 64 and 0 = (yd - k) % 8 - } - fill: *8x8_pe - dims: *3t-dim_2s-dim - dist_fn: *3t-dim_2s-dim_manhattan - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 229376 - extent_DOR_hops: null -# - occ: "{ noc[xs, ys] -> data[d0, d1] : 0 <= d0 < 8 and 0 <= d1 < 8 and (xs=0 or 3<=xs<=4 or xs=7) and (ys=0 or 3<=ys<=4 or ys=7) }" -# fill: "{ noc[xd, yd] -> data[d0, d1] : d0 = xd and d1 = yd and 0 <= xd < 8 and 0 <= yd < 8 }" -# dims: *2d_spatial -# dist_fn: *2d_manhattan -# expected: -# latency: 2 -# total_hops: 64 -# multicast_hops: null -# - occ: "{ noc[xs, ys] -> data[d0, d1] : d0 = xs and d1 = ys and 0 <= xs < 8 and 0 <= ys < 8 }" -# fill: "{ noc[xd, yd] -> data[d0, d1] : 0 <= d0 < 8 and 0 <= d1 < 8 and (xd=0 or 3<=xd<=4 or xd=7) and (yd=0 or 3<=yd<=4 or yd=7) }" -# dist_fn: *2d_manhattan -# dims: *2d_spatial -# expected: -# latency: 14 -# total_hops: 5632 -# multicast_hops: null -# - occ: | -# { noc[xs, ys] -> data[d0, d1] : -# 0 <= d0 < 3 and 0 <= d1 < 3 and -# 0 <= xs < 9 and 0 <= ys < 9 and xs % 3 = 1 and ys % 3 = 1 -# } -# fill: "{ noc[xd, yd] -> data[d0, d1] : d0 = xd % 3 and d1 = yd % 3 and 0 <= xd < 9 and 0 <= yd < 9 }" -# dims: *2d_spatial -# dist_fn: *2d_manhattan -# expected: -# latency: 2 -# total_hops: 108 -# multicast_hops: null -# - occ: "{ noc[xs, ys] -> data[d0, d1] : d0 = xs % 3 and d1 = ys % 3 and 0 <= xs < 9 and 0 <= ys < 9 }" -# fill: | -# { noc[xd, yd] -> data[d0, d1] : -# 0 <= d0 < 3 and 0 <= d1 < 3 and -# 0 <= xd < 9 and 0 <= yd < 9 and xd % 3 = 1 and yd % 3 = 1 -# } -# dims: *2d_spatial -# dist_fn: *2d_manhattan -# expected: -# latency: 2 -# total_hops: 108 -# - occ: "{ [xs, ys] -> [d0] : d0=xs and 0 <= xs < 8 and ys = 0 }" -# fill: "{ [xd, yd] -> [d0] : d0=xd and 0 <= xd < 8 and 0 <= yd < 8 }" -# dims: *2d_spatial -# dist_fn: *2d_manhattan -# expected: -# latency: 7 -# total_hops: 224 -# - occ: "{ [xs, ys] -> [d0] : d0=xs and 0 <= xs < 8 and 0 <= ys < 8 }" -# fill: "{ [xd, yd] -> [d0] : d0=xd and 0 <= xd < 8 and yd = 0 }" -# dims: *2d_spatial -# dist_fn: *2d_manhattan -# expected: -# latency: 0 -# total_hops: 0 -# - occ: | -# { [xs, ys, z1s, z2s] -> [d0, d1, d2, d3] : -# d0 = xs and d1 = ys and d2 = z1s and d3 = z2s and -# 0 <= xs < 8 and 0 <= ys < 8 and 0 <= z1s < 8 and 0 <= z2s < 8 -# } -# fill: | -# { [xd, yd, z1d, z2d] -> [d0, d1, d2, d3] : -# 0 <= d0 < 8 and 0 <= d1 < 8 and 0 <= d2 < 8 and 0 <= d3 < 8 and -# xd=0 and yd=0 and z1d=0 and z2d=0 -# } -# dims: &4d_spatial -# - Spatial -# - 0 -# - 0 -# - Spatial -# - 1 -# - 0 -# - Spatial -# - 2 -# - 0 -# - Spatial -# - 3 -# - 0 -# dist_fn: &4d_manhattan | -# { -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + (yd - ys) + (z1d - z1s) + (z2d - z2s)] : -# (xd >= xs) and (yd >= ys) and (z1d >= z1s) and (z2d >= z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + (yd - ys) + (z1d - z1s) + (z2d - z2s)] : -# (xd < xs) and (yd >= ys) and (z1d >= z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + -(yd - ys) + (z1d - z1s) + (z2d - z2s)] : -# (xd < xs) and (yd < ys) and (z1d >= z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + -(yd - ys) + -(z1d - z1s) + (z2d - z2s)] : -# (xd < xs) and (yd < ys) and (z1d < z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + -(yd - ys) + -(z1d - z1s) + -(z2d - z2s)] : -# (xd < xs) and (yd < ys) and (z1d < z1s) and (z2d < z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + -(yd - ys) + (z1d - z1s) + (z2d - z2s)] : -# (xd >= xs) and (yd < ys) and (z1d >= z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + -(yd - ys) + -(z1d - z1s) + (z2d - z2s)] : -# (xd >= xs) and (yd < ys) and (z1d < z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + -(yd - ys) + -(z1d - z1s) + -(z2d - z2s)] : -# (xd >= xs) and (yd < ys) and (z1d < z1s) and (z2d < z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + (yd - ys) + -(z1d - z1s) + (z2d - z2s)] : -# (xd >= xs) and (yd >= ys) and (z1d < z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + (yd - ys) + -(z1d - z1s) + -(z2d - z2s)] : -# (xd >= xs) and (yd >= ys) and (z1d < z1s) and (z2d < z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + (yd - ys) + (z1d - z1s) + -(z2d - z2s)] : -# (xd >= xs) and (yd >= ys) and (z1d >= z1s) and (z2d < z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + (yd - ys) + -(z1d - z1s) + -(z2d - z2s)] : -# (xd < xs) and (yd >= ys) and (z1d < z1s) and (z2d < z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + -(yd - ys) + (z1d - z1s) + -(z2d - z2s)] : -# (xd < xs) and (yd < ys) and (z1d >= z1s) and (z2d < z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + (yd - ys) + (z1d - z1s) + -(z2d - z2s)] : -# (xd < xs) and (yd >= ys) and (z1d >= z1s) and (z2d < z2s); - -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [-(xd - xs) + (yd - ys) + -(z1d - z1s) + (z2d - z2s)] : -# (xd < xs) and (yd >= ys) and (z1d < z1s) and (z2d >= z2s); -# [[xd, yd, z1d, z2d] -> [xs, ys, z1s, z2s]] -> -# [(xd - xs) + -(yd - ys) + (z1d - z1s) + -(z2d - z2s)] : -# (xd >= xs) and (yd < ys) and (z1d >= z1s) and (z2d < z2s) -# } -# expected: -# latency: 28 -# total_hops: null -# - occ: | -# { [xs, ys, z1s, z2s] -> [d0, d1, d2, d3] : -# 0 <= d0 < 8 and 0 <= d1 < 8 and 0 <= d2 < 8 and 0 <= d3 < 8 and -# xs=0 and ys=0 and z1s=0 and z2s=0 -# } -# fill: | -# { [xd, yd, z1d, z2d] -> [d0, d1, d2, d3] : -# d0 = xd and d1 = yd and d2 = z1d and d3 = z2d and -# 0 <= xd < 8 and 0 <= yd < 8 and 0 <= z1d < 8 and 0 <= z2d < 8 -# } -# dims: *4d_spatial -# dist_fn: *4d_manhattan -# expected: -# latency: 28 -# total_hops: null -# - occ: "{ [xs] -> [d0] : 0 <= d0 < 8 and xs = 0 }" -# fill: "{ [xd] -> [d0] : d0 = xd and 0 <= xd < 8 }" -# dist_fn: &ring_dist_size_8 | -# { -# [[xd] -> [xs]] -> [(xd-xs) % 8] : -# (xd-xs)%8 <= (xs-xd)%8; -# [[xd] -> [xs]] -> [(xs-xd) % 8] : -# (xd-xs)%8 > (xs-xd)%8 -# } -# expected: -# latency: 4 -# total_hops: 16 -# - occ: "{ [xs] -> [d0] : 0 <= xs < 8 and d0 = xs }" -# fill: "{ [xd] -> [d0] : 0 <= d0 < 8 and xd = 0 }" -# dist_fn: *ring_dist_size_8 -# expected: -# latency: 4 -# total_hops: 16 -# - occ: "{ [xs] -> [d0] : xs <= d0 <= xs + 1 and xs % 2 = 0}" -# fill: "{ [xd] -> [d0] : d0 = xd and 0 <= xd < 8 }" -# dist_fn: *ring_dist_size_8 -# expected: -# latency: 1 -# total_hops: 4 -################################################### -# 8-GPU fully-connected (NVLink/NVSwitch-style) # -# all-to-all, one-hot GPU encoding. # -# # -# GPU i sits at one-hot coordinate e_i so every # -# src!=dst cast has extent 1 along exactly the # -# src and dst dims: cost = (1+1)(1+1)-1 = 3, # -# uniform across all pairs (fully-connected). # -# Self-chunks never cross the fabric (cost 0). # -# dist_fn is unit-cost (matching only). # -################################################### - -# All-to-all over 8 GPUs: each GPU holds chunks data[self, d] and requests data[s, self]. -- occ: | - { - noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : - 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and - gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and - s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and 0 <= d < 8 - } - fill: | - { - noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : - 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and - gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and - d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and 0 <= s < 8 - } - dims: &8d_onehot_spatial - - type: Spatial - spatial_dim: 0 - target: 0 - - type: Spatial - spatial_dim: 1 - target: 0 - - type: Spatial - spatial_dim: 2 - target: 0 - - type: Spatial - spatial_dim: 3 - target: 0 - - type: Spatial - spatial_dim: 4 - target: 0 - - type: Spatial - spatial_dim: 5 - target: 0 - - type: Spatial - spatial_dim: 6 - target: 0 - - type: Spatial - spatial_dim: 7 - target: 0 - dist_fn: &fully_connected_unit | - { - [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[0] : - xd0 = xs0 and xd1 = xs1 and xd2 = xs2 and xd3 = xs3 and xd4 = xs4 and xd5 = xs5 and xd6 = xs6 and xd7 = xs7; - [noc[xd0, xd1, xd2, xd3, xd4, xd5, xd6, xd7] -> noc[xs0, xs1, xs2, xs3, xs4, xs5, xs6, xs7]] -> hops[1] : - (xd0 < xs0) or (xd0 > xs0) or (xd1 < xs1) or (xd1 > xs1) or (xd2 < xs2) or (xd2 > xs2) or (xd3 < xs3) or (xd3 > xs3) or (xd4 < xs4) or (xd4 > xs4) or (xd5 < xs5) or (xd5 > xs5) or (xd6 < xs6) or (xd6 > xs6) or (xd7 < xs7) or (xd7 > xs7) - } - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 168 - extent_DOR_hops: null - -# Single chunk GPU0 -> GPU3: one unicast cast, cost (1+1)(1+1)-1 = 3. -- occ: | - { - noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : - 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and - gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and - s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 0 and d = 3 - } - fill: | - { - noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : - 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and - gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and - d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 0 and d = 3 - } - dims: *8d_onehot_spatial - - dist_fn: *fully_connected_unit - - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 3 - extent_DOR_hops: null - -# Self chunk GPU5 -> GPU5: never crosses the fabric, cost 0. -- occ: | - { - noc[gs0, gs1, gs2, gs3, gs4, gs5, gs6, gs7] -> data[s, d] : - 0 <= gs0 <= 1 and 0 <= gs1 <= 1 and 0 <= gs2 <= 1 and 0 <= gs3 <= 1 and 0 <= gs4 <= 1 and 0 <= gs5 <= 1 and 0 <= gs6 <= 1 and 0 <= gs7 <= 1 and - gs0 + gs1 + gs2 + gs3 + gs4 + gs5 + gs6 + gs7 = 1 and - s = 1*gs1 + 2*gs2 + 3*gs3 + 4*gs4 + 5*gs5 + 6*gs6 + 7*gs7 and s = 5 and d = 5 - } - fill: | - { - noc[gd0, gd1, gd2, gd3, gd4, gd5, gd6, gd7] -> data[s, d] : - 0 <= gd0 <= 1 and 0 <= gd1 <= 1 and 0 <= gd2 <= 1 and 0 <= gd3 <= 1 and 0 <= gd4 <= 1 and 0 <= gd5 <= 1 and 0 <= gd6 <= 1 and 0 <= gd7 <= 1 and - gd0 + gd1 + gd2 + gd3 + gd4 + gd5 + gd6 + gd7 = 1 and - d = 1*gd1 + 2*gd2 + 3*gd3 + 4*gd4 + 5*gd5 + 6*gd6 + 7*gd7 and s = 5 and d = 5 - } - dims: *8d_onehot_spatial - - dist_fn: *fully_connected_unit - - expected: - latency: null - total_hops: null - multicast_hops: null - hypercube_hops: 0 - extent_DOR_hops: null diff --git a/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml b/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml deleted file mode 100755 index 36dcf1b8..00000000 --- a/tests/not_working/distribuffers/spec/binding/valid_bindings.yaml +++ /dev/null @@ -1,91 +0,0 @@ -- binding: - version: 0.4 - nodes: - - logical: - name: ProcessingElement - dims: [i] - physical: - name: ProcessingElement - dims: [x, y] - relations: - tensorA: i = x + y * 2 # This is a dimension-major compression into the logical. It is bijective. - tensorB: i = x + y * 2 # This is a dimension-major compression into the logical. It is bijective. - - logical: - name: Scratchpad - dims: [x, y] - physical: - name: GLB - dims: [a, b] - relations: - tensorA: x = a and y = b - tensorB: x = b and y = a - solution: - version: 0.4 - nodes: - - - tensorA: | - { - l_ProcessingElement_dims[i] -> p_ProcessingElement_dims[x, y] : - i = x + 2y - } - tensorB: | - { - l_ProcessingElement_dims[i] -> p_ProcessingElement_dims[x, y] : - i = x + (y * 2) - } - - - tensorA: | - { - l_Scratchpad_dims[x, y] -> p_GLB_dims[ a, b ] : - x = a and y = b - } - tensorB: | - { - l_Scratchpad_dims[x, y] -> p_GLB_dims[ a, b ] : - x = b and y = a - } - -# NOTE: the block below predates the `Domain.dims` / `Binding.version` / -# `BindingNode.relations` schema in `accelforge/frontend/_binding.py` (it used -# `l_dims`/`p_dims`/`relation`, and node 3 even names its physical field -# `l_dims` instead of `p_dims`) and has no `solution` counterpart to check -# against, so `test_valid_bindings` never exercised it as a second spec entry. -# Commented out rather than "fixed" per the same reasoning already applied in -# `tests/isl/distributed/spec/binding/valid_bindings.yaml` (the canonical, -# passing twin of this fixture): deriving the right ISL solution strings for -# the floor-division compression relations (`i = i // 2`) and the bypass node -# (empty relations) is a separate piece of work from the D1-D6 network-model -# fixes this file accompanies, not a minimal, mechanical rename. -# - binding: -# version: 0.4 -# nodes: -# - logical: -# name: DRAM -# l_dims: [i] -# physical: -# name: DRAM -# p_dims: [i] -# relations: # Compression relation where less DRAM chips than planned. -# weights: i = i // 2 -# inputs: i = i // 2 -# outputs: i = i // 2 -# - logical: -# name: Scratchpad -# l_dims: [i] -# physical: -# name: GLB -# p_dims: [x, y, z] -# relations: # weight stationary relation -# weights: c = x and h = y and w = z -# inputs: i=x and i=y and i=z -# outputs: i=x and i=y and i=z -# - logical: -# name: PE_Buffer -# l_dims: [i] -# physical: -# name: GLB -# l_dims: [x, y, z] -# relations: # Some weird bypass shenanigans. -# weights: -# inputs: -# outputs: diff --git a/tests/not_working/distribuffers/spec/distributed.yaml b/tests/not_working/distribuffers/spec/distributed.yaml deleted file mode 100755 index 8f7267be..00000000 --- a/tests/not_working/distribuffers/spec/distributed.yaml +++ /dev/null @@ -1,100 +0,0 @@ -networks: - - name: L2 - topology: Mesh - dims: - - x - - y - constraints: - - 0 <= x < 2 - - 0 <= y < 3 - affine: | - { - [L2[x, y] -> L2[x', y']] -> hops[(x - x') + (y - y')] : - x >= x' and y >= y'; - [L2[x, y] -> L2[x', y']] -> hops[-(x - x') + -(y - y')] : - x < x' and y < y'; - [L2[x, y] -> L2[x', y']] -> hops[-(x - x') + (y - y')] : - x < x' and y >= y'; - [L2[x, y] -> L2[x', y']] -> hops[(x - x') + -(y - y')] : - x >= x' and y < y' - } - domain_restriction: | - { - [L2[x, y] -> L2[x', y']] -> hops[(x - x') + (y - y')] : - x >= x' and y >= y' and 0 <= x < 2 and 0 <= x' < 2 and 0 <= y < 3 and 0 <= y' < 3; - [L2[x, y] -> L2[x', y']] -> hops[-(x - x') + (y - y')] : - x < x' and y >= y' and 0 <= x < 2 and 0 <= x' < 2 and 0 <= y < 3 and 0 <= y' < 3; - [L2[x, y] -> L2[x', y']] -> hops[(x - x') + -(y - y')] : - x >= x' and y < y' and 0 <= x < 2 and 0 <= x' < 2 and 0 <= y < 3 and 0 <= y' < 3; - [L2[x, y] -> L2[x', y']] -> hops[-(x - x') + -(y - y')] : - x < x' and y < y' and 0 <= x < 2 and 0 <= x' < 2 and 0 <= y < 3 and 0 <= y' < 3 - } - - name: L1 - topology: Mesh - dims: - - x - - y - - z - constraints: - - 0 <= x < 2027 - - 0 <= y < 2029 - - 0 <= z < 17 - affine: | - { - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') + (y - y') + (z - z') ] : - x >= x' and y >= y' and z >= z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') + (y - y') - (z - z') ] : - x >= x' and y >= y' and z < z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') - (y - y') + (z - z') ] : - x >= x' and y < y' and z >= z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') + (y - y') + (z - z') ] : - x < x' and y >= y' and z >= z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') - (y - y') + (z - z') ] : - x < x' and y < y' and z >= z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') - (y - y') - (z - z') ] : - x >= x' and y < y' and z < z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') + (y - y') - (z - z') ] : - x < x' and y >= y' and z < z'; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') - (y - y') - (z - z') ] : - x < x' and y < y' and z < z' - } - domain_restriction: | - { - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') + (y - y') + (z - z') ] : - x >= x' and y >= y' and z >= z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') + (y - y') - (z - z') ] : - x >= x' and y >= y' and z < z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') - (y - y') + (z - z') ] : - x >= x' and y < y' and z >= z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') + (y - y') + (z - z') ] : - x < x' and y >= y' and z >= z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') - (y - y') + (z - z') ] : - x < x' and y < y' and z >= z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ (x - x') - (y - y') - (z - z') ] : - x >= x' and y < y' and z < z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') + (y - y') - (z - z') ] : - x < x' and y >= y' and z < z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17; - [L1[x, y, z] -> L1[x', y', z']] -> hops[ -(x - x') - (y - y') - (z - z') ] : - x < x' and y < y' and z < z' and - 0 <= x < 2027 and 0 <= y < 2029 and 0 <= z < 17 and - 0 <= x' < 2027 and 0 <= y' < 2029 and 0 <= z' < 17 - } -placement: - - level: DRAM - placement: "DRAM[i] -> L2[x, y] : i = x and y = 0" - - level: MAC - placement: "MAC[i] -> L1[x, y, z] : i = x and y = 0 and z = 0" \ No newline at end of file diff --git a/tests/not_working/distribuffers/spec/logical_arch.yaml b/tests/not_working/distribuffers/spec/logical_arch.yaml deleted file mode 100755 index 51053db7..00000000 --- a/tests/not_working/distribuffers/spec/logical_arch.yaml +++ /dev/null @@ -1,32 +0,0 @@ -arch: - nodes: - - !Component - name: DRAM # offchip DRAM is the source of all datatypes - class: DRAM # assume DRAM is large enough to store all the data, so no depth specification needed - attributes: - width: 64 # width in bits - - - !Container - name: ProcessingElement - spatial: {meshX: 1, meshY: 2} - - # registers for the mac unit - - !Component - name: weight_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: input_activation_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: output_activation_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: mac - class: mac_compute - attributes: {n_pipline_stages: 2} \ No newline at end of file diff --git a/tests/not_working/distribuffers/spec/physical_arch.yaml b/tests/not_working/distribuffers/spec/physical_arch.yaml deleted file mode 100755 index ab835911..00000000 --- a/tests/not_working/distribuffers/spec/physical_arch.yaml +++ /dev/null @@ -1,69 +0,0 @@ -arch: - name: "test_distributed_spec" - nodes: - - !Container - name: system_arch - attributes: - # Top-level attributes inherited by all components unless overridden - technology: "45nm" - cycle_period: 1e-9 - - - !Network - name: L2 - topology: Mesh - dims: - - x - constraints: - - 0 <= x < 2 - - !Network - name: L1 - topology: Mesh - dims: - - x - - y - constraints: - - 0 <= x < 1 - - -1 <= y <= 1 - port: | - {L2[x] -> L1[x', y'] : x = 1 and x' = 0 and y' = 0} - - - !Component - name: DRAM # offchip DRAM is the source of all datatypes - class: DRAM # assume DRAM is large enough to store all the data, so no depth specification needed - attributes: - width: 64 # width in bits - network: - level: L2 - placement: "{ DRAM[i] -> L2[i] | i = 1 }" - - - !Container - name: ProcessingElement - network: - level: L1 - placement: | - { - ProcessingElement[i, j] -> L1[x, y] : - j = y and x < 0 and i = -x - 1; - j = y and x > 0 and i = x; - } - - # registers for the mac unit - - !Component - name: weight_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: input_activation_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: output_activation_reg - class: reg_storage - attributes: {depth: 1, width: 8} - - - !Component - name: mac - class: mac_compute - attributes: {n_pipline_stages: 2} \ No newline at end of file diff --git a/tests/not_working/distribuffers/test_binding.py b/tests/not_working/distribuffers/test_binding.py deleted file mode 100755 index 528c93f4..00000000 --- a/tests/not_working/distribuffers/test_binding.py +++ /dev/null @@ -1,49 +0,0 @@ -from pathlib import Path -import unittest - -from typing import Dict, List -from islpy import DEFAULT_CONTEXT, Map -from ruamel.yaml import YAML - -from accelforge.frontend._binding import Binding, BindingNode - -TESTS_DIR = Path(__file__).parent / "spec" / "binding" -yaml = YAML(typ="safe") - - -class TestBindingMapper(unittest.TestCase): - def test_valid_bindings(self): - """ - Tests that the valid bindings translate into the appropriate - ISL strings. - """ - specs_file: str = TESTS_DIR / "valid_bindings.yaml" - with open(specs_file, mode="r", encoding="utf-8") as f: - specs: List = yaml.load(f) - - spec: Dict - for spec in specs: - binding: Binding = Binding.model_validate(spec["binding"]) - - soln: Dict = spec["solution"] - - soln_node: Dict[str, str] - binding_node: BindingNode - for soln_node, binding_node in zip(soln["nodes"], binding.nodes): - isl_relations: Dict[str, Map] = binding_node.isl_relations - assert soln_node.keys() == isl_relations.keys(), ( - "Not all isl_relations read in properly. Missing \n" - f"{set(soln_node.keys()).difference(isl_relations.keys())} " - "\nfrom isl_relations and \n" - f"{set(isl_relations.keys()).difference(soln_node.keys())} " - "\nfrom solutions.\n" - ) - - tensor: str - for tensor in soln_node: - soln_relation: Map = Map.read_from_str( - DEFAULT_CONTEXT, soln_node[tensor] - ) - assert soln_relation.is_equal( - isl_relations[tensor] - ), f"\n{soln_relation} != \n{isl_relations[tensor]}" diff --git a/tests/not_working/isl/mapper/__init__.py b/tests/not_working/isl/mapper/__init__.py deleted file mode 100644 index e69de29b..00000000 From 3cfba6e0839baee7dbf31310fdd2ee88f692a026 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Thu, 16 Jul 2026 14:23:07 -0400 Subject: [PATCH 17/19] NCCL coordination information --- .../astrasim2_correlation/correlation.ipynb | 60 + .../correlation/.gitignore | 4 + .../correlation/README.md | 188 ++ .../correlation/config.py | 666 +++++++ .../correlation/data/.gitkeep | 0 .../correlation/orchestrate.py | 935 +++++++++ .../correlation/parse_nccl.py | 483 +++++ .../correlation/provision.py | 895 +++++++++ .../correlation/run_profile.sh | 182 ++ .../correlation/setup_node.sh | 137 ++ .../correlation/teardown.py | 850 +++++++++ .../correlation/tests/__init__.py | 0 .../correlation/tests/test_orchestrate.py | 580 ++++++ .../correlation/tests/test_parse_nccl.py | 288 +++ .../tests/test_provision_teardown.py | 802 ++++++++ .../correlation/torus_bench/Makefile | 27 + .../correlation/torus_bench/torus_bench.cu | 1681 +++++++++++++++++ 17 files changed, 7778 insertions(+) create mode 100644 notebooks/astrasim2_correlation/correlation/.gitignore create mode 100644 notebooks/astrasim2_correlation/correlation/README.md create mode 100644 notebooks/astrasim2_correlation/correlation/config.py create mode 100644 notebooks/astrasim2_correlation/correlation/data/.gitkeep create mode 100644 notebooks/astrasim2_correlation/correlation/orchestrate.py create mode 100644 notebooks/astrasim2_correlation/correlation/parse_nccl.py create mode 100644 notebooks/astrasim2_correlation/correlation/provision.py create mode 100644 notebooks/astrasim2_correlation/correlation/run_profile.sh create mode 100644 notebooks/astrasim2_correlation/correlation/setup_node.sh create mode 100644 notebooks/astrasim2_correlation/correlation/teardown.py create mode 100644 notebooks/astrasim2_correlation/correlation/tests/__init__.py create mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py create mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py create mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py create mode 100644 notebooks/astrasim2_correlation/correlation/torus_bench/Makefile create mode 100644 notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index 06aa15c2..014fd35f 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -594,6 +594,66 @@ "ax.legend() # ASTRA-sim 2.0 and EC2 nccl-tests series overlay here later\n", "plt.show()" ] + }, + { + "cell_type": "markdown", + "id": "449a7991", + "source": "## Torus leg: 2×2×2 logical torus (3-cube) all-to-all\n\nGPU $r$ sits at binary coordinates $(g_0, g_1, g_2) \\in \\{0, 1\\}^3$ of a 3-dimensional `noc` space, one coordinate per torus dimension. Rank is recovered row-major with the **last** dimension fastest-varying, $r = 4g_0 + 2g_1 + g_2$ — matching the `torus_bench` benchmark's rank$\\leftrightarrow$coordinate convention.\n\nEvery dimension here has extent 2, so the $\\pm 1 \\pmod 2$ torus wraparound edge coincides with the only other value that coordinate can take: wraparound degenerates exactly to the hypercube edge, and `HypercubeMulticastModel` applies directly. Its precondition — a Manhattan, translation-invariant `dist_fn` — is satisfied by Hamming distance on the binary coordinates (ISL has no `!=`/`<>` operator, so per-axis inequality is written as the disjunction `(a < b or a > b)`).\n\nCost per $(s, d)$ pair at Hamming distance $k$ is $2^k - 1$ (the bounding-box multicast-tree cost `HypercubeMulticastModel` computes), summing to **152** hops over all 64 `data[s, d]` chunks. A dimension-ordered *minimal unicast* routing — one packet per hop instead of a shared multicast tree, which is what the empirical torus benchmark actually implements — would instead traverse $\\sum_{s,d} \\mathrm{hamming}(s, d) = $ **96** hops. The two series bracket real routing behavior; the gap between them is itself a correlation observable, quantifying how far a multicast-aware model diverges from point-to-point unicast.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "d166f6c4", + "source": "def binary_bounds(prefix: str, k: int) -> str:\n \"\"\"Bit constraints over dims ``{prefix}0..{prefix}{k-1}``.\"\"\"\n return \" and \".join(f\"0 <= {prefix}{i} <= 1\" for i in range(k))\n\n\ndef rank_expr(prefix: str, k: int) -> str:\n \"\"\"Affine recovery of the rank from binary torus coordinates.\n\n Row-major with the LAST coordinate fastest-varying:\n ``rank = sum_i 2**(k-1-i) * g_i``, matching the ``torus_bench``\n benchmark's rank<->coordinate convention (see the markdown above).\n \"\"\"\n return \" + \".join(f\"{2 ** (k - 1 - i)}*{prefix}{i}\" for i in range(k))\n\n\ndef torus_all_to_all_maps(\n dims: tuple[int, ...] = (2, 2, 2),\n) -> tuple[isl.Map, isl.Map, isl.Map]:\n \"\"\"Build (occupancy, fill, dist_fn) for a logical-torus all-to-all.\n\n Mirrors ``all_to_all_maps`` above but keys GPUs by binary torus\n coordinates instead of a one-hot id. ``data[s, d]`` is the chunk sent\n by GPU ``s`` to GPU ``d``; each GPU holds the chunks it sends (occ)\n and requests the chunks addressed to it (fill).\n\n Parameters\n ----------\n dims : tuple of int, default (2, 2, 2)\n Per-dimension extents of the logical torus. Every entry must\n equal 2 (see Raises); ``K = len(dims)`` sets the dimensionality\n of the ``noc`` space and the number of ``SpatialTag`` axes the\n caller must build to match.\n\n Returns\n -------\n occ : isl.Map\n ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n holds the chunk it is the source of, for every destination ``d``.\n fill : isl.Map\n ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n requests the chunk addressed to it, for every source ``s``.\n dist_fn : isl.Map\n Piecewise Hamming distance between two ``noc`` points, encoded as\n a disjunction over the ``2**K`` equal/differ patterns per axis.\n\n Raises\n ------\n ValueError\n If any entry of ``dims`` is not 2. General (non-power-of-two)\n torus extents need a genuine wraparound-aware Manhattan distance\n function, not the hypercube-degeneracy shortcut used here — see\n the markdown cell above for why extent 2 is special.\n \"\"\"\n if any(d != 2 for d in dims):\n raise ValueError(\n f\"torus_all_to_all_maps only supports extent-2 dims (hypercube \"\n f\"wraparound degeneracy), got dims={dims!r}; general torus \"\n f\"extents need a wraparound-aware Manhattan dist_fn\"\n )\n k = len(dims)\n n = 2 ** k\n\n gs = \", \".join(f\"gs{i}\" for i in range(k))\n gd = \", \".join(f\"gd{i}\" for i in range(k))\n occ = isl.Map.read_from_str(\n CTX,\n f\"{{ noc[{gs}] -> data[s, d] : {binary_bounds('gs', k)} \"\n f\"and s = {rank_expr('gs', k)} and 0 <= d < {n} }}\",\n )\n fill = isl.Map.read_from_str(\n CTX,\n f\"{{ noc[{gd}] -> data[s, d] : {binary_bounds('gd', k)} \"\n f\"and d = {rank_expr('gd', k)} and 0 <= s < {n} }}\",\n )\n\n # Piecewise Hamming distance: one disjunct per subset of axes that\n # differ (2**K sign patterns), cost = popcount of the subset. ISL has\n # no `!=`, so per-axis inequality is the disjunction (a < b or a > b).\n pieces = []\n for mask in range(2 ** k):\n dist = bin(mask).count(\"1\")\n conds = [\n f\"(gd{i} < gs{i} or gd{i} > gs{i})\" if mask & (1 << i) else f\"gd{i} = gs{i}\"\n for i in range(k)\n ]\n pieces.append(\n f\"[noc[{gd}] -> noc[{gs}]] -> hops[{dist}] : \" + \" and \".join(conds)\n )\n dist_fn = isl.Map.read_from_str(CTX, \"{ \" + \"; \".join(pieces) + \" }\")\n\n return occ, fill, dist_fn", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "256ea183", + "source": "dims_torus = (2, 2, 2)\nk = len(dims_torus)\nn_t = 2 ** k # = 8 GPUs, same physical count as the FC leg above\n\nocc_t, fill_t, dist_fn_t = torus_all_to_all_maps(dims_torus)\ntags_t = [SpatialTag(i, 0) for i in range(k)]\nmodel_t = HypercubeMulticastModel(dist_fn_t)\n\n\ndef torus_pair_hops(src: int, dst: int) -> int:\n \"\"\"Hops for a single (src, dst) chunk under the torus (Hamming) dist_fn.\"\"\"\n chunk = isl.Set.read_from_str(CTX, f\"{{ data[{src}, {dst}] }}\")\n info = model_t.apply(\n 0,\n Fill(tags_t, fill_t.intersect_range(chunk)),\n Occupancy(tags_t, occ_t.intersect_range(chunk)),\n )\n return eval_total(info.hops)\n\n\ndef hamming(a: int, b: int) -> int:\n \"\"\"Hamming distance between the binary representations of two ints.\"\"\"\n return bin(a ^ b).count(\"1\")\n\n\n# --- Run the tool over the full all-to-all and validate against the\n# verified reference numbers (see torus_hops_check.py pre-verification). ---\ninfo_t = model_t.apply(0, Fill(tags_t, fill_t), Occupancy(tags_t, occ_t))\ntorus_hops = eval_total(info_t.hops)\nassert torus_hops == 152, f\"model returned {torus_hops} total hops, expected 152\"\n\nprobes = {(0, 1): 1, (0, 3): 3, (0, 7): 7, (5, 5): 0}\nfor (src, dst), expected in probes.items():\n got = torus_pair_hops(src, dst)\n assert got == expected, f\"pair ({src}->{dst}) cost {got} != {expected}\"\n\n# Dimension-ordered minimal unicast routing (what the empirical torus\n# benchmark implements): one hop per differing bit, summed over every\n# ordered (s, d) pair. Computed programmatically -- not hardcoded -- so\n# it stays correct if dims_torus ever changes.\nminroute_hops = sum(hamming(s, d) for s in range(n_t) for d in range(n_t))\nassert minroute_hops == 96, f\"minroute_hops {minroute_hops} != 96\"\n\nprint(f\"ISL network model: {n_t}-GPU 2x2x2 logical torus all-to-all (binary coords)\")\nprint(f\" bounding-box multicast hops: {torus_hops} (Sigma 2^hamming(s,d) - 1)\")\nprint(f\" min-route unicast hops : {minroute_hops} (Sigma hamming(s,d))\")\nprint(f\" routing gap : {torus_hops - minroute_hops} hops \"\n f\"({torus_hops / minroute_hops:.2f}x)\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "8f3f4946", + "source": "# 8 nodes * 3 distinct neighbors/node. Extent-2 wraparound means the +1\n# and -1 neighbor along a dim coincide (mod 2), so each node has exactly\n# one neighbor per torus dimension, not two -- hence 3, not 6.\nDIRECTED_LINKS_TORUS = n_t * k\n\nrows_t = []\nmib = MIN_MIB\nwhile mib <= MAX_MIB:\n size = mib * (1 << 20)\n total = size * n_t if PER_RANK else size\n per_rank = total / n_t\n chunk_bytes = total / (n_t * n_t)\n # By symmetry, all-to-all traffic loads all 24 directed torus links\n # uniformly: total hop-bytes (chunk_bytes * hops, summed over the 64\n # chunks) divides evenly across the links. Two series bracket\n # routing: `torus_hops` (bounding-box multicast) vs. `minroute_hops`\n # (dimension-ordered unicast) -- see the markdown cell above.\n per_link_bytes_model = (torus_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n per_link_bytes_minroute = (minroute_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n model_latency_s = ALPHA_S + per_link_bytes_model / bw\n minroute_latency_s = ALPHA_S + per_link_bytes_minroute / bw\n # Closed-form cross-check: 152/(64*24) = 19/192, 96/(64*24) = 1/16.\n assert math.isclose(model_latency_s, ALPHA_S + total * 19 / (192 * bw), rel_tol=1e-12)\n assert math.isclose(minroute_latency_s, ALPHA_S + total / (16 * bw), rel_tol=1e-12)\n rows_t.append((n_t, int(total), int(per_rank), per_link_bytes_model,\n model_latency_s, minroute_latency_s))\n mib *= 2\n\nsweep_torus = pd.DataFrame(rows_t, columns=[\n \"nodes\", \"collective_size_bytes\", \"per_rank_bytes\", \"per_link_bytes_model\",\n \"model_latency_s\", \"minroute_latency_s\",\n])\nsweep_torus", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "5e209b22", + "source": "## Empirical overlay: EC2 nccl-tests + torus_bench\n\nCSV files under `correlation/data/` (e.g. `correlation/data//csv/*.csv`) are fetched by `correlation/orchestrate.py` (sibling infrastructure, not part of this notebook) and share a unified schema: `source, topology, dims, collective, size_bytes, count, dtype, time_us, algbw_GBps, busbw_GBps, wrong`.\n\nSize normalization to the total collective size $S$ used by `sweep` / `sweep_torus` above differs by source: for `source == \"torus_bench\"` rows, `size_bytes` is **already** the total $S$. For `source == \"nccl-tests\"` rows with `collective == \"alltoall\"`, `size_bytes` is the *per-rank* message size, so $S = N \\times \\texttt{size\\_bytes}$ — verify this against the nccl-tests version actually in use, since the size semantics nccl-tests reports differ per collective. Only `alltoall` is normalized and correlated against the model here; other collectives are inventoried below for later work.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "4715980a", + "source": "from pathlib import Path\n\n# Design: prefer the path relative to the notebook's own directory (the\n# normal case when the notebook is run in place), but fall back to the\n# repo-root-relative path so `jupyter execute` from a different cwd (e.g.\n# CI running from the repo root) still finds real data when it exists,\n# instead of silently taking the empty-data path below.\nDATA_DIR = (\n Path(\"correlation/data\") if Path(\"correlation\").is_dir()\n else Path(\"notebooks/astrasim2_correlation/correlation/data\")\n)\n\nUNIFIED_CSV_COLUMNS = [\n \"source\", \"topology\", \"dims\", \"collective\", \"size_bytes\", \"count\",\n \"dtype\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\", \"wrong\",\n]\n_NUMERIC_COLUMNS = [\"size_bytes\", \"count\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\"]\n\nfiles = sorted(DATA_DIR.glob(\"**/*.csv\"))\nif files:\n emp = pd.concat(\n [pd.read_csv(f, dtype={\"wrong\": str}) for f in files],\n ignore_index=True,\n )\n for col in _NUMERIC_COLUMNS:\n emp[col] = pd.to_numeric(emp[col], errors=\"coerce\")\nelse:\n # Design: keep the unified schema even when no files were found, so\n # downstream cells can reference emp[...] columns inside their\n # `if HAVE_EMPIRICAL:` guards without ever hitting a KeyError.\n emp = pd.DataFrame(columns=UNIFIED_CSV_COLUMNS)\n\nHAVE_EMPIRICAL = len(files) > 0 and not emp.empty\n\nif not HAVE_EMPIRICAL:\n print(\"no empirical data yet -- run correlation/orchestrate.py; \"\n \"overlay cells below will no-op\")\nelse:\n inventory = emp.groupby([\"topology\", \"collective\"]).agg(\n rows=(\"size_bytes\", \"count\"),\n min_size=(\"size_bytes\", \"min\"),\n max_size=(\"size_bytes\", \"max\"),\n )\n display(inventory)", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "a2e080ce", + "source": "fig, ax = plt.subplots(figsize=(7, 5))\n\nax.loglog(sweep[\"collective_size_bytes\"], sweep[\"model_latency_s\"] * 1e6, \"o-\",\n label=f\"FC model ({LINK_BW_GBPS:g} GB/s links)\")\nax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"model_latency_s\"] * 1e6, \"s-\",\n label=\"torus model, 152 hops (bounding-box multicast)\")\nax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"minroute_latency_s\"] * 1e6, \"s--\",\n label=\"torus min-route, 96 hops (dimension-ordered unicast)\")\n\nif HAVE_EMPIRICAL:\n alltoall = emp[emp[\"collective\"] == \"alltoall\"]\n fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n if not fc_rows.empty:\n # nccl-tests alltoall size_bytes is the PER-RANK message size (see\n # markdown above); scale to total S to match `sweep`'s convention.\n ax.scatter(fc_rows[\"size_bytes\"] * NODES, fc_rows[\"time_us\"],\n marker=\"x\", color=\"tab:red\", label=\"EC2 nccl-tests alltoall (fc)\")\n if not torus_rows.empty:\n # torus_bench size_bytes is ALREADY total collective size S.\n ax.scatter(torus_rows[\"size_bytes\"], torus_rows[\"time_us\"],\n marker=\"+\", color=\"tab:green\", label=\"EC2 torus_bench alltoall (torus)\")\n\nax.set_xlabel(\"total collective size (bytes)\")\nax.set_ylabel(\"all-to-all latency (µs)\")\nax.set_title(\"Fully-connected vs. 2x2x2 torus all-to-all: model vs. empirical\")\nax.grid(True, which=\"both\", alpha=0.3)\nax.legend()\nplt.show()", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "id": "a8ec5d63", + "source": "import numpy as np\n\n\ndef linear_fit(S: np.ndarray, t: np.ndarray) -> tuple[float, float, float]:\n \"\"\"Ordinary-least-squares fit of ``t = intercept + slope * S``.\n\n Parameters\n ----------\n S : np.ndarray\n Total collective size in bytes for each empirical sample.\n t : np.ndarray\n Measured latency in seconds for each empirical sample. Must be\n the same length as `S`.\n\n Returns\n -------\n intercept : float\n Fitted per-operation latency overhead (the model's alpha), seconds.\n slope : float\n Fitted dt/dS, seconds/byte.\n r_squared : float\n Coefficient of determination of the linear fit; NaN if `t` has\n zero total variance (R^2 is undefined in that degenerate case).\n \"\"\"\n slope, intercept = np.polyfit(S, t, 1)\n t_pred = slope * S + intercept\n ss_res = float(np.sum((t - t_pred) ** 2))\n ss_tot = float(np.sum((t - np.mean(t)) ** 2))\n r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else float(\"nan\")\n return float(intercept), float(slope), r_squared\n\n\ndef bw_from_slope(slope: float, slope_factor: float) -> float:\n \"\"\"Translate a fitted t-vs-S slope into an effective bandwidth, bytes/s.\n\n Given the model form ``t = alpha + S * slope_factor / BW``, the\n fitted slope is ``dt/dS = slope_factor / BW``, so\n ``BW = slope_factor / slope``.\n \"\"\"\n return slope_factor / slope\n\n\nif HAVE_EMPIRICAL and (emp[\"collective\"] == \"alltoall\").any():\n alltoall = emp[emp[\"collective\"] == \"alltoall\"].dropna(\n subset=[\"size_bytes\", \"time_us\"]\n )\n\n fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n if len(fc_rows) >= 2:\n # fc slope_factor = (N-1)/N^2 = 7/64 (N=8), from the closed form\n # in the \"Size and latency conventions\" cell above.\n S_fc = fc_rows[\"size_bytes\"].to_numpy(dtype=float) * NODES\n t_fc = fc_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n alpha_fc, slope_fc, r2_fc = linear_fit(S_fc, t_fc)\n bw_fc = bw_from_slope(slope_fc, 7 / 64)\n print(f\"FC calibration : alpha = {alpha_fc * 1e6:.3f} us, \"\n f\"BW_eff = {bw_fc / 1e9:.2f} GB/s, R^2 = {r2_fc:.4f}\")\n else:\n print(\"FC calibration : fewer than 2 alltoall rows, skipping fit\")\n\n torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n if len(torus_rows) >= 2:\n # torus slope_factor = hops/(64*24) = hops/1536; report BW under\n # both routing hypotheses (h=152 bounding-box, h=96 min-route)\n # from a single fit, since alpha/R^2 don't depend on slope_factor\n # -- only the slope-to-BW translation does.\n S_t = torus_rows[\"size_bytes\"].to_numpy(dtype=float)\n t_t = torus_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n alpha_t, slope_t, r2_t = linear_fit(S_t, t_t)\n bw_t_152 = bw_from_slope(slope_t, 152 / 1536)\n bw_t_96 = bw_from_slope(slope_t, 96 / 1536)\n print(f\"torus calibration : alpha = {alpha_t * 1e6:.3f} us, R^2 = {r2_t:.4f}\")\n print(f\" BW_eff @ h=152 (bounding-box model): {bw_t_152 / 1e9:.2f} GB/s\")\n print(f\" BW_eff @ h=96 (dimension-ordered) : {bw_t_96 / 1e9:.2f} GB/s\")\n else:\n print(\"torus calibration : fewer than 2 alltoall rows, skipping fit\")\nelse:\n print(\"calibration: no empirical alltoall rows yet\")", + "metadata": {}, + "execution_count": null, + "outputs": [] } ], "metadata": { diff --git a/notebooks/astrasim2_correlation/correlation/.gitignore b/notebooks/astrasim2_correlation/correlation/.gitignore new file mode 100644 index 00000000..6bddd09b --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/.gitignore @@ -0,0 +1,4 @@ +keys/ +.state/ +*.pem +logs/ diff --git a/notebooks/astrasim2_correlation/correlation/README.md b/notebooks/astrasim2_correlation/correlation/README.md new file mode 100644 index 00000000..4eb33486 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/README.md @@ -0,0 +1,188 @@ +# correlation/ -- AWS provisioning for NCCL profiling + +This directory is the **empirical leg** of the ISL-model correlation +study: it profiles real NCCL collective-communication performance on one +AWS `p5.48xlarge` instance (8x H100, NVSwitch), both in the GPUs' native +fully-connected (FC) topology and under a logical torus overlay, so those +measurements can be compared against the model's and ASTRA-sim's +predictions. + +## Prerequisites + +- **AWS credentials** available to boto3 via the standard mechanisms (env + vars `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN`, a + named profile via `AWS_PROFILE`, or an attached IAM role). This + directory never manages or stores credentials itself. +- **boto3** installed in whatever Python environment you run these + scripts from: `pip install boto3`. It is deliberately *not* a dependency + of the `accelforge` package -- only this notebook's provisioning corner + needs it. +- **Service Quotas.** `p5.48xlarge` requires 192 vCPUs of quota. Before + your first run, check the Service Quotas console for your target region + and confirm both: + - the on-demand P-instance quota ("Running On-Demand P instances" or + equivalent, depending on current AWS naming), and + - the corresponding Spot P-instance vCPU quota (if you intend to use + `--purchasing spot` or the default `spot-then-ondemand`), + + are each **>= 192 vCPUs**. Quota codes and exact names change over time + and vary by account/region presentation -- look them up in the console + rather than trusting a hardcoded code here. A quota that's too low + surfaces as an `InsufficientInstanceCapacity`-adjacent or limit-exceeded + error at launch time. + +## Cost warning + +`p5.48xlarge` on-demand pricing is roughly **$30-55/hr**, depending on +region and current AWS pricing -- **verify current pricing** at + before running anything. +A full profiling sweep (FC + torus, all collectives, the full message-size +range) is expected to take well under an hour, but see the dead-man-timer +note below: the default 120-minute timer is comfortably above that +estimate, but a large custom `--min-mib`/`--max-mib` range or collective +list can push a full sweep close to or past it. Spot pricing is +substantially cheaper when capacity is available, which is why +`spot-then-ondemand` is the default purchasing mode -- see +`provision.launch_instance`'s docstring for the exact fallback conditions. +Both `orchestrate.py` and `provision.py` print this same warning and +require an interactive `yes` confirmation before touching AWS (skippable +with `--yes`; not required for `--dry-run`). + +## Quickstart + +**`orchestrate.py` is the end-to-end path.** It provisions its own +instance, pushes the profiling scripts, runs the full FC and/or torus +sweep, fetches the results locally, and tears the instance down again -- +all in one command: + +```bash +# 1. Sanity-check permissions and request shape without launching anything +# (still creates a real SSH key pair + security group -- see the +# --dry-run note below): +python orchestrate.py --topology both --dry-run + +# 2. The real run: provision, profile both legs, fetch results, tear down. +python orchestrate.py --topology both +``` + +**Do not run `provision.py` before `orchestrate.py`.** `orchestrate.py` +provisions its *own* instance internally; running `provision.py` first +would launch a *second*, independent `p5.48xlarge` instance that nothing +in the `orchestrate.py` run above knows about or tears down, silently +doubling your bill. `provision.py`/`teardown.py` are for advanced, manual +control only -- see the section below. + +Every `Config` field (region, purchasing mode, topology, message-size +range, collectives, ...) is a CLI flag; run `python orchestrate.py --help` +for the full list, or pass `--config some.yaml` to load a batch of +overrides from YAML (CLI flags still win over anything in the YAML file). +Useful flags: + +- `--keep-alive`: skip `orchestrate.py`'s own teardown at the end, leaving + the instance running (its SSH command is printed) for manual + inspection. The on-instance dead-man timer still applies regardless. +- `--dead-man-minutes `: extend the dead-man timer past its 120-minute + default -- see the note below. +- `--ssh-cidr /32`: skip auto-detecting your IP for the SSH + security-group rule; required if IP auto-detection fails (see + `provision.caller_ip`'s docstring). + +### Fetched data layout + +Each leg's results land under `data///`: + +``` +data/// +├── csv/ # parsed, unified-schema CSVs consumed by the correlation notebook +├── raw/ # raw nccl-tests/torus_bench stdout logs, one per collective +└── metadata.txt # machine/software provenance (nvidia-smi topo, driver/NCCL versions, git rev) +``` + +`` is `fc` or `torus`. `data/` is the one subdirectory of this +project *not* gitignored -- fetched results are committed intentionally. + +## Advanced / manual control: `provision.py` + `teardown.py` + +`provision.py` and `teardown.py` are the individual "up" and "down" halves +`orchestrate.py` composes internally. Use them directly only if you need +manual control between provisioning and profiling (e.g. debugging the +instance by hand, or running a custom workload instead of +`run_profile.sh`) -- most users should use `orchestrate.py` above instead. + +```bash +# Provision one instance and leave it running (see the warning below): +python provision.py + +# ... do whatever manual work you need on the instance ... + +# Tear down that one run: +python teardown.py --run-id + +# Tear down everything this study has tagged, across every region this +# study's local state knows about: +python teardown.py --all + +# Also delete the SSH key pair (AWS-side) and local PEM: +python teardown.py --run-id --delete-key + +# Audit: confirm nothing is left running, without tearing anything down. +# Exits 0 ("no running instances") when clean, 1 with a table otherwise -- +# safe to use as a post-teardown check or a periodic cron/CI safety net. +python teardown.py --verify +``` + +**Warning: `provision.py` leaves the instance running with NO dead-man +timer until `setup_node.sh` is run on it.** The dead-man timer is armed +*by* `setup_node.sh` (a step `orchestrate.py` always runs for you, but +`provision.py` alone does not reach) -- so an instance provisioned via +`provision.py` and never followed up with `setup_node.sh` (or +`teardown.py`) will run, and bill, indefinitely with no automatic +backstop. If you provision manually, either run `setup_node.sh` on the +instance promptly or tear it down yourself as soon as you're done. + +`--dry-run` on **both** `provision.py` and `orchestrate.py` performs +`DryRun=True` authorization checks only and launches no instance -- but a +**real** SSH key pair and security group ARE still created in AWS either +way (there is no dry-run equivalent for those two calls). Clean them up +with: + +```bash +python teardown.py --run-id --delete-key +``` + +`teardown.py` discovers resources two ways and reconciles them: local +`.state/*.json` files written by `provision.py`/`orchestrate.py`, and a +live `describe_instances` search by `Project`/`RunId` tags. The tag search +is authoritative, so teardown still works even if a state file was lost or +a run was started from a different machine. `--region` defaults to +*resolving per run* rather than to a fixed region: an explicit `--region` +flag always wins, otherwise each run's own state file (if any) supplies +its region, otherwise `us-east-1` (`Config`'s default) is used -- so +`--all`/`--verify` correctly span every region this study's local state +knows about in one invocation, not just one hardcoded region. + +## Safety guardrails + +- **Dead-man timer -- armed by `setup_node.sh`, not at instance launch.** + Once `setup_node.sh` has run on the instance (always true for an + `orchestrate.py` run; not automatic for a manual `provision.py` one -- + see the warning above), an on-instance timer force-shuts-down the box + after `--dead-man-minutes` (default 120) regardless of whether + `teardown.py` was ever run -- a backstop against a forgotten or failed + teardown. Re-running `setup_node.sh` (e.g. via a second `orchestrate.py` + leg) pushes the deadline back rather than erroring or stacking. Note + that the 120-minute default can be tight for a long custom sweep -- + raise it with `--dead-man-minutes` if you expect to run past it. +- **`InstanceInitiatedShutdownBehavior=terminate`.** The instance is + launched so that an in-instance `shutdown` (including the dead-man + timer firing) *terminates* it rather than merely stopping it, so it + cannot be left billing in a stopped state. +- **Teardown-in-`finally`.** `orchestrate.py` calls teardown from a + `finally` block around the provisioning-through-profiling sequence, so a + mid-sweep crash or a failed leg still tears the instance down (unless + `--keep-alive` was passed). +- **PEM files are gitignored.** `keys/`, `.state/`, `*.pem`, and `logs/` + are all excluded (see `.gitignore`) -- private key material and + per-run state never get committed. `data/` (fetched profiling CSVs) is + the one subdirectory *not* ignored; those results are committed + intentionally. diff --git a/notebooks/astrasim2_correlation/correlation/config.py b/notebooks/astrasim2_correlation/correlation/config.py new file mode 100644 index 00000000..9fb675f0 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/config.py @@ -0,0 +1,666 @@ +"""Shared configuration for the NCCL profiling correlation-study AWS scripts. + +This module defines :class:`Config`, the single source of truth for every +tunable knob used by ``provision.py``, ``teardown.py``, and (per the plan) +the sibling ``orchestrate.py`` that a later work package will add. Keeping +the configuration in one frozen dataclass -- rather than threading loose +kwargs through each script -- means every script agrees on defaults, +validation, and CLI flag names without duplicating logic. + +Construction paths +------------------- +``Config`` instances can be built three ways, all of which funnel through +the same validation in :meth:`Config.__post_init__`: + +1. Directly, e.g. ``Config(region="us-west-2")`` -- handy for tests and for + any future caller that wants a config without touching argparse at all. +2. Via :meth:`Config.from_args`, which owns its own + :class:`argparse.ArgumentParser` end to end. +3. Via :meth:`Config.add_args` + :meth:`Config.from_parsed`, which lets a + *caller* (``provision.py``, ``teardown.py``) build one shared parser, + add its own script-specific flags (e.g. ``--dry-run``), and only then + hand the resulting namespace back to ``Config`` to extract just the + fields that belong to it. This is the path ``provision.py`` and + ``teardown.py`` actually use. + +Design: frozen dataclass +------------------------- +``Config`` is declared ``frozen=True`` so that once built it can be passed +into ``provision.py`` functions (``launch_instance``, ``write_state``, ...) +without any risk of one function's edits leaking into another's view of the +same run. Frozen dataclasses cannot assign to ``self.`` in the usual +way, so any post-construction normalization (parsing "2x2x2" into a tuple, +generating a run id, coercing str paths to :class:`pathlib.Path`) goes +through ``object.__setattr__`` inside ``__post_init__`` -- this is the +standard, documented escape hatch for "derive a field after validation" on +a frozen dataclass. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import datetime +import math +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +# Design: anchor key_dir/state_dir to this file's directory (not the +# process cwd) so that `python provision.py` behaves identically no matter +# where the user's shell happens to be sitting when they invoke it. +_THIS_DIR = Path(__file__).resolve().parent + +_ALLOWED_PURCHASING = frozenset({"spot", "ondemand", "spot-then-ondemand"}) +_ALLOWED_TOPOLOGY = frozenset({"fc", "torus", "both"}) + +# Default sweep of NCCL collectives profiled by the (sibling-work-package) +# profiling scripts. Kept here, not in profiling code, so a single +# `--collectives` override on the CLI is the one place a user needs to +# change to alter the sweep. +_DEFAULT_COLLECTIVES: Tuple[str, ...] = ( + "all_reduce", + "all_gather", + "reduce_scatter", + "alltoall", + "broadcast", + "sendrecv", +) + +# The study profiles one 8x H100 node; a torus overlay must therefore have +# axis dimensions whose product is exactly 8 (one "logical GPU slot" per +# axis position), regardless of how many axes are used. +_TORUS_GPU_COUNT = 8 + +_RUN_ID_TIME_FORMAT = "%Y%m%d-%H%M%S" + + +def _default_run_id() -> str: + """Generate a fresh, sortable run identifier. + + Returns + ------- + str + ``"correl-" + UTC timestamp`` formatted as + ``%Y%m%d-%H%M%S`` (e.g. ``"correl-20260716-161503"``). Lexicographic + sort order matches chronological order, which is convenient when + listing ``.state/*.json`` files or AMI/SG names in a shell. + + Notes + ----- + Uses UTC (not local time) so run ids are unambiguous and comparable + regardless of which machine or timezone invokes the script. + """ + return "correl-" + datetime.datetime.now(datetime.timezone.utc).strftime( + _RUN_ID_TIME_FORMAT + ) + + +def _is_power_of_two(n: int) -> bool: + """Return whether a positive integer is an exact power of two. + + Parameters + ---------- + n : int + Value to test. + + Returns + ------- + bool + ``True`` if ``n > 0`` and ``n & (n - 1) == 0``, ``False`` otherwise + (including for ``n <= 0``). + + Examples + -------- + >>> _is_power_of_two(1024) + True + >>> _is_power_of_two(0) + False + >>> _is_power_of_two(3) + False + """ + return n > 0 and (n & (n - 1)) == 0 + + +def _parse_torus_dims(value: Any) -> Tuple[int, ...]: + """Normalize a torus-dimensions spec into a tuple of ints. + + Accepts the three shapes this value can arrive in depending on + construction path: an already-correct ``tuple[int, ...]`` (direct + ``Config(...)`` construction), a ``list``/``tuple`` of ints or numeric + strings (from a parsed YAML ``--config`` file), or a delimited string + like ``"2x2x2"`` (from the CLI, where argparse hands raw strings to + ``type=`` callables). + + Parameters + ---------- + value : Any + Torus dimensions in any of the accepted shapes described above. + + Returns + ------- + tuple[int, ...] + The parsed per-axis dimensions, in the order given. + + Raises + ------ + ValueError + If ``value`` is an empty string, or any component cannot be parsed + as an integer. + + Examples + -------- + >>> _parse_torus_dims("2x2x2") + (2, 2, 2) + >>> _parse_torus_dims([2, 4]) + (2, 4) + >>> _parse_torus_dims((8,)) + (8,) + """ + if isinstance(value, (list, tuple)): + try: + return tuple(int(v) for v in value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"torus_dims entries must all be integers, got {value!r}" + ) from exc + + text = str(value).strip().lower() + if not text: + raise ValueError("torus_dims must not be empty") + try: + return tuple(int(part) for part in text.split("x")) + except ValueError as exc: + raise ValueError( + f"torus_dims={value!r} is not a valid dims string; " + "expected a form like '2x2x2'" + ) from exc + + +def _parse_collectives(value: Any) -> Tuple[str, ...]: + """Normalize a collectives spec into a tuple of collective names. + + Parameters + ---------- + value : Any + Either a ``list``/``tuple`` of collective-name strings (from a + parsed YAML ``--config`` file or direct construction), or a + comma-separated string (from the CLI). + + Returns + ------- + tuple[str, ...] + The parsed collective names, in the order given, with surrounding + whitespace stripped and empty entries dropped (so a trailing comma + like ``"all_reduce,"`` does not produce a spurious ``""`` entry). + + Examples + -------- + >>> _parse_collectives("all_reduce, broadcast") + ('all_reduce', 'broadcast') + >>> _parse_collectives(["all_reduce", "broadcast"]) + ('all_reduce', 'broadcast') + """ + if isinstance(value, (list, tuple)): + return tuple(str(v) for v in value) + return tuple(part.strip() for part in str(value).split(",") if part.strip()) + + +@dataclasses.dataclass(frozen=True) +class Config: + """Immutable configuration for one correlation-study AWS provisioning run. + + All fields have defaults, so ``Config()`` alone yields a fully valid + configuration (one on-demand-or-spot ``p5.48xlarge`` in ``us-east-1``, + fully-connected topology, a freshly generated ``run_id``). Every + construction path (direct, :meth:`from_args`, :meth:`from_parsed`) runs + the same validation in :meth:`__post_init__`, so an invalid ``Config`` + can never be observed by downstream code. + + Parameters + ---------- + region : str, default "us-east-1" + AWS region to provision in. Also used for the SSM AMI lookup and + must have the requested ``instance_type`` available. + availability_zone : str or None, default None + Specific AZ within ``region`` to pin the instance to. ``None`` + lets AWS/the spot fleet choose. + instance_type : str, default "p5.48xlarge" + EC2 instance type. The 8x H100 NVSwitch topology this study + profiles is specific to ``p5.48xlarge``; other types are accepted + without validation but are not what the rest of this work package + was designed against. + purchasing : str, default "spot-then-ondemand" + One of ``"spot"``, ``"ondemand"``, ``"spot-then-ondemand"``. See + ``provision.launch_instance`` for the fallback semantics of the + combined mode. + topology : str, default "fc" + One of ``"fc"`` (fully connected, the NVSwitch-native topology), + ``"torus"`` (a logical torus overlay profiled on top of the same + physical node), or ``"both"``. + torus_dims : tuple[int, ...], default (2, 2, 2) + Per-axis dimensions of the logical torus. Precondition: the + product of all dimensions must equal 8 (one axis slot per GPU on + the node). Accepts a ``"2x2x2"``-style string on construction and + normalizes it to a tuple of ints. + collectives : tuple[str, ...], default (all_reduce, all_gather, + reduce_scatter, alltoall, broadcast, sendrecv) + NCCL collectives the (sibling) profiling scripts should sweep. + Accepts a comma-separated string on construction. + min_mib : int, default 1 + Smallest message size, in MiB, in the profiling sweep. + Precondition: must be a power of two and ``<= max_mib``. + max_mib : int, default 1024 + Largest message size, in MiB, in the profiling sweep. + Precondition: must be a power of two and ``>= min_mib``. + run_id : str or None, default None + Identifier used to name/tag every AWS resource created for this + run (instance, security group, key pair, state file). If left as + ``None``, a fresh id is generated in :meth:`__post_init__` as + ``"correl-" + UTC timestamp``. See the NOTE in + :meth:`__post_init__` for why generation happens there rather than + only in :meth:`from_args`. + tag_project : str, default "accelforge-correlation" + Value written to the ``Project`` tag on every AWS resource this + run creates; also what ``teardown.py`` filters + ``describe_instances`` on to discover a run's resources. + ssh_user : str, default "ubuntu" + Login user baked into the deep-learning AMI, used when printing + the ``ssh`` command at the end of provisioning. + ami_ssm_parameter : str, default + "/aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" + SSM public parameter name that resolves to the latest matching + Deep Learning AMI id for ``region``. + root_volume_gb : int, default 200 + Size, in GiB, of the root ``gp3`` EBS volume attached at + ``/dev/sda1``. + dead_man_minutes : int, default 120 + Minutes of wall-clock time after which the on-instance dead-man + timer (armed by setup scripts owned by a sibling work package) + force-shuts-down the instance. This is a safety net independent of + ``teardown.py`` actually being run; see the README's guardrails + section. + key_dir : pathlib.Path, default "/keys" + Directory where generated SSH private keys are written. + state_dir : pathlib.Path, default "/.state" + Directory where per-run provisioning state JSON is written. + + Raises + ------ + ValueError + Raised by :meth:`__post_init__` if ``purchasing`` or ``topology`` + is not one of the allowed values, if ``torus_dims`` does not + multiply out to 8 or contains a non-positive entry, if + ``min_mib > max_mib``, or if either ``min_mib`` or ``max_mib`` is + not a power of two. + + Examples + -------- + >>> cfg = Config(region="us-west-2", purchasing="ondemand") + >>> cfg.region, cfg.purchasing + ('us-west-2', 'ondemand') + >>> cfg.run_id is not None + True + """ + + region: str = "us-east-1" + availability_zone: Optional[str] = None + instance_type: str = "p5.48xlarge" + purchasing: str = "spot-then-ondemand" + topology: str = "fc" + torus_dims: Tuple[int, ...] = (2, 2, 2) + collectives: Tuple[str, ...] = _DEFAULT_COLLECTIVES + min_mib: int = 1 + max_mib: int = 1024 + run_id: Optional[str] = None + tag_project: str = "accelforge-correlation" + ssh_user: str = "ubuntu" + ami_ssm_parameter: str = ( + "/aws/service/deeplearning/ami/x86_64/" + "base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" + ) + root_volume_gb: int = 200 + dead_man_minutes: int = 120 + key_dir: Path = _THIS_DIR / "keys" + state_dir: Path = _THIS_DIR / ".state" + + def __post_init__(self) -> None: + """Normalize field representations and validate all preconditions. + + Raises + ------ + ValueError + See the class docstring's ``Raises`` section; this method is + where every one of those checks is actually enforced. + + Notes + ----- + Runs for *every* construction path (``Config(...)`` directly, + :meth:`from_args`, :meth:`from_parsed`), because dataclasses always + call ``__post_init__`` after ``__init__``. This is a deliberate + choice over validating only inside :meth:`from_args`: it means a + test (or a future caller) that builds ``Config(purchasing="bogus")`` + directly fails loudly at construction time instead of silently + producing an invalid config that only misbehaves once it reaches + AWS calls. + """ + # Frozen dataclasses disallow `self.field = ...`; object.__setattr__ + # is the standard, documented way to set fields from __post_init__. + object.__setattr__(self, "torus_dims", _parse_torus_dims(self.torus_dims)) + object.__setattr__(self, "collectives", _parse_collectives(self.collectives)) + object.__setattr__(self, "key_dir", Path(self.key_dir)) + object.__setattr__(self, "state_dir", Path(self.state_dir)) + + if self.run_id is None: + # NOTE: the work-package spec describes run_id's default as + # "generated ... at parse time", which most literally refers to + # Config.from_args. We instead generate it here, in + # __post_init__, so every construction path gets a valid run_id + # -- see the docstring Notes above for why. This is the more + # conservative reading: it can never produce a Config with + # run_id=None reaching AWS tag values, which the "at parse + # time" phrasing on its own does not guarantee for direct + # `Config(...)` construction. + object.__setattr__(self, "run_id", _default_run_id()) + + if self.purchasing not in _ALLOWED_PURCHASING: + raise ValueError( + f"purchasing={self.purchasing!r} is not one of " + f"{sorted(_ALLOWED_PURCHASING)}" + ) + if self.topology not in _ALLOWED_TOPOLOGY: + raise ValueError( + f"topology={self.topology!r} is not one of {sorted(_ALLOWED_TOPOLOGY)}" + ) + + # A negative-dimension axis (e.g. (-2, -2, 2)) could still multiply + # out to 8, silently passing a bare product check; reject it + # explicitly since it is never physically meaningful for a torus. + if any(d <= 0 for d in self.torus_dims): + raise ValueError( + f"torus_dims={self.torus_dims!r} must all be positive integers" + ) + product = math.prod(self.torus_dims) + if product != _TORUS_GPU_COUNT: + raise ValueError( + f"torus_dims={self.torus_dims!r} has product {product}, " + f"expected {_TORUS_GPU_COUNT} (one p5.48xlarge node = " + f"{_TORUS_GPU_COUNT} GPUs)" + ) + + if self.min_mib > self.max_mib: + raise ValueError( + f"min_mib={self.min_mib} must be <= max_mib={self.max_mib}" + ) + if not _is_power_of_two(self.min_mib): + raise ValueError(f"min_mib={self.min_mib} must be a power of two") + if not _is_power_of_two(self.max_mib): + raise ValueError(f"max_mib={self.max_mib} must be a power of two") + + @classmethod + def add_args(cls, parser: argparse.ArgumentParser) -> None: + """Register every ``Config`` field as a kebab-case CLI flag. + + Intended to be called by a script's own parser setup (see + ``provision.py``/``teardown.py``) *before* that script adds its + own extra flags (e.g. ``--dry-run``), so the two flag sets share + one ``argparse.ArgumentParser`` and one ``--help`` output. + + Parameters + ---------- + parser : argparse.ArgumentParser + Parser to add arguments to, mutated in place. + + Notes + ----- + Design: every flag added here uses ``default=argparse.SUPPRESS`` + instead of the field's real default. This means an unset flag is + simply *absent* from the parsed namespace, which is exactly what + :meth:`from_parsed` needs to implement "CLI flags override + ``--config`` YAML values override dataclass defaults" -- if every + flag instead defaulted to its real value, :meth:`from_parsed` could + not distinguish "user explicitly passed the default value" from + "user didn't pass this flag at all", and CLI flags could never be + overridden by anything. + """ + # Only used to render human-readable defaults into --help text; + # never used for the actual default values (see Notes above). + defaults = cls() + + parser.add_argument( + "--config", + type=str, + default=None, + help=( + "Path to a YAML file of Config field overrides, applied " + "before CLI flags (CLI flags always win over --config)." + ), + ) + parser.add_argument( + "--region", + type=str, + default=argparse.SUPPRESS, + help=f"AWS region (default: {defaults.region!r}).", + ) + parser.add_argument( + "--availability-zone", + type=str, + default=argparse.SUPPRESS, + help="AWS availability zone, e.g. us-east-1a (default: let AWS choose).", + ) + parser.add_argument( + "--instance-type", + type=str, + default=argparse.SUPPRESS, + help=f"EC2 instance type (default: {defaults.instance_type!r}).", + ) + parser.add_argument( + "--purchasing", + type=str, + choices=sorted(_ALLOWED_PURCHASING), + default=argparse.SUPPRESS, + help=f"Purchasing strategy (default: {defaults.purchasing!r}).", + ) + parser.add_argument( + "--topology", + type=str, + choices=sorted(_ALLOWED_TOPOLOGY), + default=argparse.SUPPRESS, + help=f"NCCL topology to profile (default: {defaults.topology!r}).", + ) + parser.add_argument( + "--torus-dims", + type=_parse_torus_dims, + default=argparse.SUPPRESS, + help=( + "Logical torus dims as e.g. '2x2x2'; product must be 8 " + f"(default: {'x'.join(str(d) for d in defaults.torus_dims)!r})." + ), + ) + parser.add_argument( + "--collectives", + type=_parse_collectives, + default=argparse.SUPPRESS, + help=( + "Comma-separated NCCL collectives to sweep " + f"(default: {','.join(defaults.collectives)!r})." + ), + ) + parser.add_argument( + "--min-mib", + type=int, + default=argparse.SUPPRESS, + help=( + "Smallest message size in MiB, must be a power of two " + f"(default: {defaults.min_mib})." + ), + ) + parser.add_argument( + "--max-mib", + type=int, + default=argparse.SUPPRESS, + help=( + "Largest message size in MiB, must be a power of two " + f"(default: {defaults.max_mib})." + ), + ) + parser.add_argument( + "--run-id", + type=str, + default=argparse.SUPPRESS, + help=( + "Run identifier used to tag/name all resources " + "(default: generated as 'correl-')." + ), + ) + parser.add_argument( + "--tag-project", + type=str, + default=argparse.SUPPRESS, + help=f"Value for the 'Project' tag on all resources (default: {defaults.tag_project!r}).", + ) + parser.add_argument( + "--ssh-user", + type=str, + default=argparse.SUPPRESS, + help=f"SSH login user for the AMI (default: {defaults.ssh_user!r}).", + ) + parser.add_argument( + "--ami-ssm-parameter", + type=str, + default=argparse.SUPPRESS, + help="SSM parameter name to resolve the AMI id from.", + ) + parser.add_argument( + "--root-volume-gb", + type=int, + default=argparse.SUPPRESS, + help=f"Root EBS volume size in GiB (default: {defaults.root_volume_gb}).", + ) + parser.add_argument( + "--dead-man-minutes", + type=int, + default=argparse.SUPPRESS, + help=( + "Minutes before the on-instance dead-man timer force-shuts-down " + f"the instance (default: {defaults.dead_man_minutes})." + ), + ) + parser.add_argument( + "--key-dir", + type=Path, + default=argparse.SUPPRESS, + help=f"Directory to store the generated SSH private key (default: {defaults.key_dir}).", + ) + parser.add_argument( + "--state-dir", + type=Path, + default=argparse.SUPPRESS, + help=f"Directory to store per-run provisioning state JSON (default: {defaults.state_dir}).", + ) + + @classmethod + def from_parsed(cls, namespace: argparse.Namespace) -> "Config": + """Build a :class:`Config` from an already-parsed argparse namespace. + + Meant to be used together with :meth:`add_args`: a caller builds + one ``ArgumentParser``, calls ``Config.add_args(parser)``, adds its + own extra flags, calls ``parser.parse_args(argv)``, and passes the + resulting namespace here. Any namespace attributes that are not + ``Config`` field names (e.g. a caller's own ``--dry-run``) are + ignored, so the same namespace can safely be shared with + script-specific flags. + + Parameters + ---------- + namespace : argparse.Namespace + Parsed CLI arguments, as produced by + ``parser.parse_args(...)`` on a parser that included + :meth:`add_args`'s flags. If it has a ``config`` attribute + (from the ``--config`` flag) that is truthy, that path is + loaded as a YAML mapping of field overrides. + + Returns + ------- + Config + A validated ``Config`` built from, in increasing priority: + dataclass defaults, then ``--config`` YAML values, then + explicitly-passed CLI flags. + + Raises + ------ + ValueError + If ``--config`` points at a YAML document whose top level is + not a mapping, or if any field fails :meth:`__post_init__` + validation. + OSError + If ``--config`` points at a path that cannot be opened. + """ + field_names = {f.name for f in dataclasses.fields(cls)} + values: Dict[str, Any] = {} + + config_path = getattr(namespace, "config", None) + if config_path: + # Design: import PyYAML lazily, inside this branch, rather than + # at module top. Only the --config path needs it; a plain + # `provision.py --help` (or any run that never passes + # --config) must keep working even in an environment that + # only has boto3 installed and not PyYAML. + import yaml + + with open(config_path, "r") as fh: + yaml_values = yaml.safe_load(fh) + if yaml_values is None: + yaml_values = {} + if not isinstance(yaml_values, dict): + raise ValueError( + f"--config file {config_path!r} must contain a top-level " + f"YAML mapping, got {type(yaml_values).__name__}" + ) + # Silently drop unknown keys rather than raising: this lets a + # single shared YAML file carry keys meant for other tools + # (e.g. a future orchestrate.py section) without every + # consumer needing to know about every other consumer's keys. + values.update({k: v for k, v in yaml_values.items() if k in field_names}) + + # CLI flags win over --config values. Because add_args() gives + # every flag default=argparse.SUPPRESS, `namespace` only carries a + # key for a field the user actually typed on the command line, so + # this unconditional overwrite is exactly "CLI beats YAML beats + # dataclass default". + for key, value in vars(namespace).items(): + if key in field_names: + values[key] = value + + return cls(**values) + + @classmethod + def from_args(cls, argv: Optional[List[str]] = None) -> "Config": + """Parse ``argv`` with a fresh, ``Config``-only parser. + + Convenience wrapper around :meth:`add_args` + :meth:`from_parsed` + for callers that only need ``Config``'s own flags and do not have + any script-specific flags of their own to add. + + Parameters + ---------- + argv : list[str] or None, default None + Argument list to parse, as passed to + ``argparse.ArgumentParser.parse_args``. ``None`` means "read + from ``sys.argv[1:]``", argparse's own default behavior. + + Returns + ------- + Config + The parsed, validated configuration. + + Examples + -------- + >>> Config.from_args(["--region", "us-west-2", "--purchasing", "ondemand"]).region + 'us-west-2' + """ + parser = argparse.ArgumentParser( + description="accelforge NCCL correlation-study provisioning config." + ) + cls.add_args(parser) + namespace = parser.parse_args(argv) + return cls.from_parsed(namespace) diff --git a/notebooks/astrasim2_correlation/correlation/data/.gitkeep b/notebooks/astrasim2_correlation/correlation/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/correlation/orchestrate.py b/notebooks/astrasim2_correlation/correlation/orchestrate.py new file mode 100644 index 00000000..00d5dddd --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/orchestrate.py @@ -0,0 +1,935 @@ +"""End-to-end CLI for the NCCL correlation study's empirical leg. + +This is the "run everything" work package: it ties together the +provisioning infrastructure (``config.py``, ``provision.py``, +``teardown.py``) and the profiling infrastructure (``setup_node.sh``, +``run_profile.sh``, ``parse_nccl.py``, ``torus_bench/``) -- all owned by +sibling work packages and imported/invoked here, never modified -- into one +command an operator runs to go from "nothing provisioned" to "CSVs sitting +in ``data///csv/`` and the instance torn down". + +Pipeline +-------- +1. Parse CLI args into a :class:`config.Config` plus this script's own + ``--yes``/``--keep-alive``/``--dry-run``/``--ssh-cidr`` flags. +2. Print the run header, cost warning, a wall-time estimate, and the leg + plan (via :func:`legs_for`); ask for interactive confirmation unless + ``--yes`` (skipped entirely for ``--dry-run``, which spends no money). +3. Resolve the AMI, create the SSH key pair and security group, and call + ``provision.launch_instance`` -- exactly the sequence ``provision.py`` + itself runs, reusing its functions directly rather than reimplementing + any of them. If ``--dry-run``, stop here (the key pair and security + group above were still created for real; see the ``--dry-run`` flag's + help text). +4. Write the provisioning state file immediately, before waiting for SSH + (see the design comment at that call site for why). +5. Wait for the instance to become SSH-reachable, then update the state + file with the now-known public IP. +6. scp the profiling scripts and ``torus_bench/`` onto the instance. +7. ssh in to run ``setup_node.sh`` (arms the dead-man timer, builds + nccl-tests/torus_bench). +8. For each leg selected by ``--topology`` (see :func:`legs_for`), ssh in + to run ``run_profile.sh`` for the full collective sweep, then scp the + results back to ``data///``. +9. In a ``finally`` block around steps 5-8: tear the instance down (unless + ``--keep-alive``), so a crash or a failed profiling leg never leaves an + (expensive, 8x H100) instance running unattended. See the design + comment on :func:`_teardown_and_cleanup` for how a teardown failure + itself is handled without masking whatever exception was already + propagating. + +Design: no new AWS/SSH logic here +----------------------------------- +Every AWS API call in this module goes through a ``provision.py`` or +``teardown.py`` function that already exists, is already tested, and is +already documented as part of this work package's contract (see those +modules' docstrings). This module's own responsibility is narrower: +sequencing those calls correctly, building ``ssh``/``scp`` argv lists +(:func:`build_ssh_cmd`, :func:`build_scp_cmd`), and running them as +subprocesses. Keeping that boundary sharp is also what makes this module +testable without any real AWS/SSH/SCP access -- every seam it introduces +(the two ``build_*_cmd`` functions, plus the imported provisioning +functions) is a plain function that a test can monkeypatch or inspect the +return value of, per this work package's "no AWS, no network, no ssh in +tests" constraint. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from config import Config +from provision import ( + _COST_WARNING, + _prompt_yes_no, + _require_boto3, + caller_ip, + ensure_key_pair, + ensure_security_group, + launch_instance, + resolve_ami, + wait_for_instance, + write_state, +) +from teardown import teardown_run + +# Design: guarded import, matching provision.py/teardown.py's own +# convention (see provision.py's module docstring for the full rationale) +# -- so `python orchestrate.py --help` keeps working even in a Python +# environment that lacks boto3, since argparse's own --help handling exits +# before main() ever reaches _require_boto3(). Importing `boto3` here as a +# module-level name of orchestrate.py's own (rather than reaching into +# `provision.boto3`) keeps this module's boto3.client(...) calls readable +# without poking at another module's internals; _require_boto3() (reused +# from provision.py, not redefined) is still what actually validates +# availability before any client is constructed, since both imports +# resolve to the same cached sys.modules entry (or both to None) in any +# given interpreter. +try: + import boto3 +except ImportError: # pragma: no cover - exercised only when boto3 truly absent + boto3 = None + +# --------------------------------------------------------------------------- +# Module-level path constants +# --------------------------------------------------------------------------- + +# Anchor every sibling-file path to this file's own directory (not the +# process cwd), matching config.py's identical _THIS_DIR convention -- so +# `python orchestrate.py` behaves the same regardless of the caller's shell +# cwd. +_THIS_DIR = Path(__file__).resolve().parent + +_SETUP_NODE_SH = _THIS_DIR / "setup_node.sh" +_RUN_PROFILE_SH = _THIS_DIR / "run_profile.sh" +_PARSE_NCCL_PY = _THIS_DIR / "parse_nccl.py" +_TORUS_BENCH_DIR = _THIS_DIR / "torus_bench" + +# Design: expose the fetched-results root as its own module-level constant +# (rather than inlining `_THIS_DIR / "data"` at the one call site) purely +# so tests can monkeypatch `orchestrate._DATA_DIR` to a tmp_path and +# guarantee the end-to-end test never writes into the real repository's +# data/ directory -- every other AWS/ssh/scp side effect in a test is +# already monkeypatched away, and this is the one remaining plain +# filesystem write main() would otherwise perform unconditionally. +_DATA_DIR = _THIS_DIR / "data" + +# run_profile.sh's argument for the FC leg: the profiling scripts +# treat "8" as "all 8 GPUs, fully connected" -- there is no logical torus +# shape to describe for that leg (see run_profile.sh's `-g 8` on the FC +# path), unlike the torus leg where is a "DxDx..." shape string. +_FC_DIMS = "8" + +_BYTES_PER_MIB = 2**20 + + +# --------------------------------------------------------------------------- +# Pure helpers (no I/O, no AWS, no subprocess) -- kept separate from main() +# specifically so they are trivially unit-testable per this work package's +# "no AWS, no network, no ssh in tests" constraint. +# --------------------------------------------------------------------------- + + +def legs_for(topology: str) -> List[str]: + """Expand a ``Config.topology`` value into the ordered list of legs to run. + + Parameters + ---------- + topology : str + Typically ``cfg.topology``, one of ``"fc"``, ``"torus"``, or + ``"both"`` (``Config.__post_init__`` already validates this, so + this function does not re-validate it -- see Notes). + + Returns + ------- + list[str] + ``["fc", "torus"]`` if ``topology == "both"``; otherwise the + single-element list ``[topology]``. + + Notes + ----- + Deliberately permissive for any value other than ``"both"``: it simply + echoes that value back as a one-element list rather than checking it + against the allowed set. Re-validating here would duplicate + ``Config.__post_init__``'s already-authoritative check for no benefit, + since every caller in this module only ever passes an already-validated + ``cfg.topology``. + + Examples + -------- + >>> legs_for("both") + ['fc', 'torus'] + >>> legs_for("fc") + ['fc'] + >>> legs_for("torus") + ['torus'] + """ + if topology == "both": + return ["fc", "torus"] + return [topology] + + +def _dims_for_leg(leg: str, torus_dims: Tuple[int, ...]) -> str: + """Compute run_profile.sh's ```` argument for one leg. + + Parameters + ---------- + leg : str + ``"fc"`` or ``"torus"``. + torus_dims : tuple[int, ...] + Per-axis torus dimensions, e.g. ``(2, 2, 2)`` (only consulted when + ``leg == "torus"``). + + Returns + ------- + str + :data:`_FC_DIMS` (``"8"``) for the FC leg; otherwise + ``torus_dims`` joined with ``"x"``, e.g. ``"2x2x2"``. + + Examples + -------- + >>> _dims_for_leg("fc", (2, 2, 2)) + '8' + >>> _dims_for_leg("torus", (2, 2, 2)) + '2x2x2' + """ + if leg == "fc": + return _FC_DIMS + return "x".join(str(d) for d in torus_dims) + + +def _estimate_wall_time_message(legs: List[str]) -> str: + """Build the human-readable wall-time estimate printed before confirmation. + + Parameters + ---------- + legs : list[str] + The leg plan, as returned by :func:`legs_for`. + + Returns + ------- + str + A one-line estimate: ~30-60 minutes when both legs are selected + (the work-package spec's own figure for a full FC+torus sweep), + or roughly half that for a single leg. + + Notes + ----- + This is a coarse, documented *estimate* for setting operator + expectations before a real-money confirmation prompt, not a measured + or SLA'd figure -- actual time depends on instance boot time, spot + availability, and the exact collective/message-size sweep configured. + """ + if len(legs) >= 2: + return ( + "Estimated wall time: ~30-60 minutes for both legs " + "(excludes instance boot/provisioning time)." + ) + return ( + "Estimated wall time: ~15-30 minutes for a single leg " + "(excludes instance boot/provisioning time)." + ) + + +def build_ssh_cmd( + key_path: Path, + user: str, + ip: str, + remote_cmd: str, + known_hosts_path: Path, +) -> List[str]: + """Build an ``ssh`` argv list to run one command on the provisioned instance. + + Parameters + ---------- + key_path : pathlib.Path + Path to the local PEM private key (as returned by + ``provision.ensure_key_pair``). + user : str + SSH login user, typically ``cfg.ssh_user``. + ip : str + Target host's public IP address. + remote_cmd : str + The full remote command line to execute, e.g. + ``"DEADMAN_MINUTES=120 bash ~/setup_node.sh"``. Passed to ``ssh`` + as a single trailing argv element; ``ssh`` hands it to the remote + login shell for interpretation, so ordinary shell syntax (env var + prefixes, multiple space-separated arguments) works as expected + without any extra quoting from this function. + known_hosts_path : pathlib.Path + Path to a per-run known-hosts file (typically + ``cfg.state_dir / "known_hosts"``). Kept as an explicit parameter + (rather than a hardcoded/global path) so this function stays pure + and independently testable -- see the module docstring's "Design: + no new AWS/SSH logic here" section. + + .. note:: + NOTE ON SPEC DEVIATION: the work-package spec lists this + function's signature as ``build_ssh_cmd(key_path, user, ip, + remote_cmd)`` -- four parameters -- but also requires the + ``-o UserKnownHostsFile=/known_hosts`` option, which + cannot be constructed without knowing ``state_dir`` from + *somewhere*. Reaching into a module-global ``Config`` from + inside this function would break the "pure helper, easy to + unit-test" property the spec explicitly asks for. Adding + ``known_hosts_path`` as an explicit fifth parameter is the + minimal change that preserves purity/testability; every + call site in this module passes ``cfg.state_dir / "known_hosts"`` + for it, matching the spec's intent. + + Returns + ------- + list[str] + argv list of the form ``["ssh", "-i", , "-o", + "StrictHostKeyChecking=accept-new", "-o", "UserKnownHostsFile=<...>", + "-o", "ConnectTimeout=30", "@", ]``, ready to + pass to ``subprocess.run``. + + Examples + -------- + >>> build_ssh_cmd(Path("/k/id.pem"), "ubuntu", "1.2.3.4", "echo hi", Path("/s/known_hosts")) + ... # doctest: +NORMALIZE_WHITESPACE + ['ssh', '-i', '/k/id.pem', '-o', 'StrictHostKeyChecking=accept-new', + '-o', 'UserKnownHostsFile=/s/known_hosts', '-o', 'ConnectTimeout=30', + 'ubuntu@1.2.3.4', 'echo hi'] + """ + return [ + "ssh", + "-i", + str(key_path), + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"UserKnownHostsFile={known_hosts_path}", + "-o", + "ConnectTimeout=30", + f"{user}@{ip}", + remote_cmd, + ] + + +def build_scp_cmd( + key_path: Path, + sources: List[str], + dest: str, + known_hosts_path: Path, + recursive: bool = False, +) -> List[str]: + """Build an ``scp`` argv list to copy one or more paths to/from the instance. + + Parameters + ---------- + key_path : pathlib.Path + Path to the local PEM private key. + sources : list[str] + Source path(s) to copy, in whatever form ``scp`` accepts: plain + local paths for a push, or a single ``"user@ip:remote/path"`` + string for a fetch. Must be non-empty. + dest : str + Destination, again in whatever form ``scp`` accepts (a local + directory for a fetch, or ``"user@ip:remote/path"`` for a push). + known_hosts_path : pathlib.Path + Path to a per-run known-hosts file. See :func:`build_ssh_cmd`'s + docstring for why this is an explicit parameter rather than an + implicit global (the same rationale applies here). + recursive : bool, default False + If ``True``, prepend ``-r`` (needed for copying a directory, e.g. + ``torus_bench/`` or a remote ``results_/`` directory). + + Returns + ------- + list[str] + argv list of the form ``["scp", ["-r"], "-i", , "-o", + "StrictHostKeyChecking=accept-new", "-o", + "UserKnownHostsFile=<...>", "-o", "ConnectTimeout=30", *sources, + dest]``, ready to pass to ``subprocess.run``. The ``-r`` flag (when + present) is placed immediately after ``"scp"``, before every other + option, so its position is fixed and independently assertable in + tests regardless of how many sources are given. + + Raises + ------ + ValueError + If ``sources`` is empty -- an ``scp`` invocation with no source + path is never meaningful and would otherwise fail cryptically at + the OS level instead of at this argv-building step. + + Examples + -------- + >>> build_scp_cmd(Path("/k/id.pem"), ["a.sh", "b.sh"], "ubuntu@1.2.3.4:~/", Path("/s/known_hosts")) + ... # doctest: +NORMALIZE_WHITESPACE + ['scp', '-i', '/k/id.pem', '-o', 'StrictHostKeyChecking=accept-new', + '-o', 'UserKnownHostsFile=/s/known_hosts', '-o', 'ConnectTimeout=30', + 'a.sh', 'b.sh', 'ubuntu@1.2.3.4:~/'] + >>> build_scp_cmd(Path("/k/id.pem"), ["dir"], "ubuntu@1.2.3.4:~/dir", Path("/s/known_hosts"), recursive=True)[:2] + ['scp', '-r'] + """ + if not sources: + raise ValueError("build_scp_cmd requires at least one source path") + + cmd: List[str] = ["scp"] + if recursive: + cmd.append("-r") + cmd += [ + "-i", + str(key_path), + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"UserKnownHostsFile={known_hosts_path}", + "-o", + "ConnectTimeout=30", + ] + cmd += list(sources) + cmd.append(dest) + return cmd + + +# --------------------------------------------------------------------------- +# Subprocess execution +# --------------------------------------------------------------------------- + + +def _run_streaming(cmd: List[str]) -> None: + """Run an ``ssh``/``scp`` argv list, streaming its output live. + + Parameters + ---------- + cmd : list[str] + argv list, typically from :func:`build_ssh_cmd` or + :func:`build_scp_cmd`. + + Raises + ------ + subprocess.CalledProcessError + If the subprocess exits with a non-zero status (``check=True``). + + Notes + ----- + Deliberately does not pass ``capture_output``/``stdout``/``stderr`` -- + the child process's output streams straight through to this process's + own stdout/stderr, so an operator watching a multi-minute profiling + sweep sees live progress rather than a silent hang followed by a wall + of buffered text at the end. Prints the command line first (to stdout) + so the corresponding output block is identifiable in a long combined + log. + """ + print("+ " + " ".join(cmd)) + subprocess.run(cmd, check=True) + + +# --------------------------------------------------------------------------- +# Pipeline stages -- each a thin, single-responsibility wrapper around a +# handful of _run_streaming calls, factored out of main() so that function +# stays a readable top-to-bottom sequence rather than one long body. +# --------------------------------------------------------------------------- + + +def _push_files(cfg: Config, key_path: Path, public_ip: str, known_hosts_path: Path) -> None: + """scp the profiling scripts and torus_bench/ onto the instance. + + Parameters + ---------- + cfg : Config + Run configuration; only ``cfg.ssh_user`` is consulted directly + (the rest flows through ``key_path``/``public_ip``/ + ``known_hosts_path``). + key_path : pathlib.Path + Local PEM private key path. + public_ip : str + Instance's public IP, as returned by ``provision.wait_for_instance``. + known_hosts_path : pathlib.Path + Per-run known-hosts file path. + + Notes + ----- + Two separate ``scp`` invocations, matching the two different transfer + shapes: (1) three individual files pushed non-recursively to the + remote home directory, and (2) the ``torus_bench/`` directory pushed + recursively to ``~/torus_bench`` so ``setup_node.sh`` can build it. + Side effect: two subprocess invocations over the network to the + instance. + """ + remote_home = f"{cfg.ssh_user}@{public_ip}:~/" + print("=== Pushing profiling scripts (setup_node.sh, run_profile.sh, parse_nccl.py) ===") + _run_streaming( + build_scp_cmd( + key_path, + [str(_SETUP_NODE_SH), str(_RUN_PROFILE_SH), str(_PARSE_NCCL_PY)], + remote_home, + known_hosts_path, + ) + ) + + remote_torus_dir = f"{cfg.ssh_user}@{public_ip}:~/torus_bench" + print("=== Pushing torus_bench/ ===") + _run_streaming( + build_scp_cmd( + key_path, + [str(_TORUS_BENCH_DIR)], + remote_torus_dir, + known_hosts_path, + recursive=True, + ) + ) + + +def _run_setup(cfg: Config, key_path: Path, public_ip: str, known_hosts_path: Path) -> None: + """ssh in and run ``setup_node.sh`` (arms dead-man timer, builds binaries). + + Parameters + ---------- + cfg : Config + Run configuration; ``cfg.dead_man_minutes`` is forwarded to + ``setup_node.sh`` as the ``DEADMAN_MINUTES`` environment variable. + key_path : pathlib.Path + Local PEM private key path. + public_ip : str + Instance's public IP. + known_hosts_path : pathlib.Path + Per-run known-hosts file path. + + Notes + ----- + Must run after :func:`_push_files` (``setup_node.sh`` and + ``torus_bench/`` must already be on the instance) and before any + profiling leg (``setup_node.sh`` is what builds the nccl-tests and + torus_bench binaries those legs depend on, and arms the on-instance + dead-man safety timer). Side effect: one subprocess invocation over + the network to the instance. + """ + remote_cmd = f"DEADMAN_MINUTES={cfg.dead_man_minutes} bash ~/setup_node.sh" + print("=== Running setup_node.sh ===") + _run_streaming(build_ssh_cmd(key_path, cfg.ssh_user, public_ip, remote_cmd, known_hosts_path)) + + +def _run_leg( + cfg: Config, + leg: str, + key_path: Path, + public_ip: str, + known_hosts_path: Path, + run_data_dir: Path, +) -> None: + """Run one profiling leg on the instance and fetch its results locally. + + Parameters + ---------- + cfg : Config + Run configuration (``torus_dims``, ``min_mib``/``max_mib``, + ``collectives``, ``ssh_user``). + leg : str + ``"fc"`` or ``"torus"``. + key_path : pathlib.Path + Local PEM private key path. + public_ip : str + Instance's public IP. + known_hosts_path : pathlib.Path + Per-run known-hosts file path. + run_data_dir : Path + Local directory this run's results should land under (typically + ``_DATA_DIR / cfg.run_id``); this leg's results land specifically + at ``run_data_dir / leg``. + + Notes + ----- + Side effects: one ``ssh`` subprocess invocation that runs the full + collective sweep for this leg on the instance (this is the + long-running step -- see :func:`_estimate_wall_time_message`), a + ``mkdir`` of ``run_data_dir`` (the PARENT of this leg's fetch + destination -- see the inline comment above that call for why the leaf + directory itself is deliberately left uncreated), and one ``scp -r`` + subprocess invocation that fetches the results back, creating the new + local directory tree at ``run_data_dir / leg`` itself. + + Prints the list of fetched ``*.csv`` files at the end, so an operator + watching the run can immediately confirm data landed without a + separate ``ls``. + + Raises + ------ + RuntimeError + If ``run_data_dir / leg`` already exists before the fetch -- see + the inline comment above the check for why this is treated as an + error rather than silently proceeding. + """ + dims = _dims_for_leg(leg, cfg.torus_dims) + min_bytes = cfg.min_mib * _BYTES_PER_MIB + max_bytes = cfg.max_mib * _BYTES_PER_MIB + remote_results_dir = f"~/results_{leg}" + remote_cmd = ( + f"bash ~/run_profile.sh {remote_results_dir} {leg} {min_bytes} {max_bytes} {dims} " + f"{' '.join(cfg.collectives)}" + ) + + print(f"=== Profiling leg: {leg} (dims={dims}, {cfg.min_mib}-{cfg.max_mib} MiB) ===") + _run_streaming(build_ssh_cmd(key_path, cfg.ssh_user, public_ip, remote_cmd, known_hosts_path)) + + # Design: fetch into a leaf directory (run_data_dir/leg) that must NOT + # already exist when scp runs. `scp -r user@host:~/results_fc + # ` renames the copied directory to when + # doesn't yet exist, landing its contents (csv/, raw/, + # metadata.txt) directly inside it -- exactly the data/// + # layout this work package's spec requires. Pre-creating that leaf + # directory first would instead make scp nest an extra results_fc/ + # level inside it, since scp's "copy into vs. rename to" behavior + # depends on whether the destination path already exists. + local_leg_dir = run_data_dir / leg + if local_leg_dir.exists(): + # A pre-existing leaf directory means a previous run already fetched + # results for this exact run_id+leg combination (or something else + # created the path). Silently proceeding would make scp nest an + # extra results_/ level inside the existing directory instead + # of landing csv/raw/metadata.txt directly in it (see the comment + # above), corrupting the data/// layout without any + # error -- raise instead so a re-run with a colliding run_id+leg + # fails loudly here rather than silently mis-nesting fetched data. + raise RuntimeError( + f"{local_leg_dir} already exists; a previous fetch for run_id=" + f"{cfg.run_id!r} leg={leg!r} already landed results there. " + "Refusing to scp into it again (that would nest an extra " + f"results_{leg}/ level inside the existing directory instead of " + "csv/raw/metadata.txt landing directly in it). Remove or rename " + "the existing directory, or re-run with a different --run-id." + ) + # Only the PARENT (run_data_dir) is created here -- local_leg_dir itself + # must be left absent (see the comment above) for scp's rename-vs-nest + # behavior to produce the right layout. Without this mkdir, the very + # first leg fetched under a fresh run_id would fail outright: scp cannot + # write to / if itself doesn't exist + # yet (parents=True/exist_ok=True mirrors provision.write_state's own + # "create the directory, then write into it" idiom). + run_data_dir.mkdir(parents=True, exist_ok=True) + fetch_source = f"{cfg.ssh_user}@{public_ip}:{remote_results_dir}" + print(f"=== Fetching {leg} leg results ===") + _run_streaming( + build_scp_cmd( + key_path, + [fetch_source], + str(local_leg_dir), + known_hosts_path, + recursive=True, + ) + ) + + csv_files = sorted((local_leg_dir / "csv").glob("*.csv")) + print(f"Fetched {leg} leg results to {local_leg_dir}:") + for csv_path in csv_files: + print(f" {csv_path}") + + +def _print_keep_alive_notice(cfg: Config, key_path: Path, state: Dict[str, Any]) -> None: + """Print the SSH command and a loud cost reminder for a ``--keep-alive`` run. + + Parameters + ---------- + cfg : Config + Run configuration (``ssh_user``, ``run_id``, ``dead_man_minutes``). + key_path : pathlib.Path + Local PEM private key path, included in the printed SSH command. + state : dict + This run's state dict; ``state["public_ip"]`` is used if present. + + Notes + ----- + Called from :func:`main`'s ``finally`` block in place of + :func:`_teardown_and_cleanup` when ``--keep-alive`` was passed. Does + not delete the state file (unlike the teardown path), since + ``teardown.py`` needs it later to find and tear down this + still-running instance. + """ + ip_display = state.get("public_ip") or "" + print("=" * 70) + print("--keep-alive set: instance left RUNNING. YOU ARE STILL BEING BILLED.") + print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{ip_display}") + print( + f"Dead-man shutdown deadline: ~{cfg.dead_man_minutes} minutes after " + "setup_node.sh armed it (this is a backstop independent of teardown.py)." + ) + print(f"Tear down manually with: python teardown.py --run-id {cfg.run_id}") + print("=" * 70) + + +def _teardown_and_cleanup( + ec2_client, state: Dict[str, Any], state_path: Path, cfg: Config +) -> None: + """Tear down this run's AWS resources and remove its local state file. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + state : dict + This run's state dict, as passed to ``teardown.teardown_run``. + state_path : pathlib.Path + Path to this run's ``.state/.json`` file, removed only if + teardown succeeds. + cfg : Config + Run configuration, used only to print ``cfg.run_id`` into the + follow-up messages below. + + Notes + ----- + Called from :func:`main`'s ``finally`` block, which may itself be + running while an exception from the profiling steps (e.g. a failed + ``run_profile.sh`` invocation) is already propagating out of the + surrounding ``try``. This is exactly the scenario the work-package + spec calls out: a ``finally`` block that itself raises would, under + Python's ordinary exception semantics, cause *that new* exception to + be what the caller sees instead of the original one -- effectively + masking the original failure behind a teardown failure, even though + both are independently worth surfacing. This function therefore + catches any exception ``teardown_run`` raises, prints it (and does not + delete the state file, since a failed teardown may have left resources + behind that ``teardown.py --run-id `` will still need it to + find), and simply returns -- letting whatever exception was already + active in the caller's ``try`` block continue propagating undisturbed. + """ + print(f"Tearing down run {cfg.run_id}...") + try: + teardown_run(ec2_client, state, delete_key=False) + except Exception as teardown_exc: # noqa: BLE001 + # Design/WHY deliberately broad and deliberately NOT re-raised: + # see this function's docstring Notes above. Printing to stderr + # (rather than raising) is what prevents this teardown failure + # from masking an in-flight exception from the try block this + # finally belongs to. + print(f"ERROR: teardown failed: {teardown_exc}", file=sys.stderr) + print( + "Manual cleanup required -- the instance may still be running. " + f"Try: python teardown.py --run-id {cfg.run_id}", + file=sys.stderr, + ) + return + + if state_path.exists(): + state_path.unlink() + print(f"Removed state file {state_path}.") + print("Recommended audit: python teardown.py --verify") + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: Optional[List[str]] = None) -> int: + """Parse CLI args and run the full provision -> profile -> teardown pipeline. + + Parameters + ---------- + argv : list[str] or None, default None + Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` + reads from ``sys.argv[1:]``. + + Returns + ------- + int + ``0`` on success (including a completed ``--dry-run``), ``1`` if + the user declined the interactive confirmation prompt. + + Raises + ------ + Exception + Any exception raised by provisioning, SSH/SCP subprocess failures + (``subprocess.CalledProcessError``), or waiting for the instance + propagates out of this function uncaught -- per the work-package + spec, a profiling/provisioning failure is a real failure and + should surface as one (a nonzero process exit code via + ``raise SystemExit(main())`` in ``__main__``), not be silently + downgraded to a return code. The ``finally`` block described below + still runs before that propagation completes. + + Notes + ----- + Side effects (skipped when ``--dry-run`` is passed, past the point + where the SSH key pair and security group are created -- see the + ``--dry-run`` flag's help text): creates an SSH key pair and security + group, launches an instance, writes/updates a state JSON file, scp's + profiling scripts and results to/from the instance, ssh's in to run + setup and profiling commands, and (unless ``--keep-alive``) tears the + instance down again. + + Design/WHY the state file is written immediately after + ``launch_instance`` returns, before ``wait_for_instance`` is even + called: ``wait_for_instance`` can block for several minutes (instance + boot, then polling for SSH) and can itself raise (``TimeoutError``, + ``WaiterError``). If this controller process crashes or is killed + during that wait, an instance is still running and being billed with + no local record of it unless the state file was already written + beforehand. Writing state right after launch -- with ``public_ip`` as + an explicit placeholder, filled in by a second ``write_state`` call + once ``wait_for_instance`` returns -- means ``teardown.py`` (which + also cross-checks live AWS tags, not just local state, per its own + docstring) can find and tear down this run from local state alone even + in that crash scenario, without waiting on SSH reachability first. + """ + parser = argparse.ArgumentParser( + prog="orchestrate.py", + description=( + "End-to-end NCCL correlation-study orchestration: provision one " + "p5.48xlarge instance, push profiling scripts, run the FC " + "and/or torus profiling sweep, fetch results into data//, " + "and tear the instance down." + ), + ) + Config.add_args(parser) + parser.add_argument( + "--dry-run", + action="store_true", + help=( + "Perform DryRun=True authorization checks only; launch no " + "instance and run no profiling. NOTE: a real SSH key pair and " + "security group ARE still created in AWS even with --dry-run, " + "mirroring provision.py's documented --dry-run semantics -- " + "only launch_instance's actual launch and everything after it " + "(waiting for SSH, pushing files, profiling, teardown) is " + "skipped." + ), + ) + parser.add_argument( + "--keep-alive", + action="store_true", + help=( + "Skip automatic teardown after profiling completes (or fails); " + "leave the instance running and print its SSH command instead " + "of tearing it down. The on-instance dead-man timer " + "(--dead-man-minutes) still applies regardless of this flag -- " + "it only disables orchestrate.py's own teardown call, not that " + "backstop." + ), + ) + parser.add_argument( + "--ssh-cidr", + type=str, + default=None, + help=( + "CIDR block allowed SSH access, e.g. 1.2.3.4/32. Overrides " + "auto-detected caller IP (see provision.caller_ip())." + ), + ) + parser.add_argument( + "--yes", + action="store_true", + help="Skip the interactive cost-confirmation prompt (not required for --dry-run).", + ) + args = parser.parse_args(argv) + cfg = Config.from_parsed(args) + + # Fail fast on a missing boto3 before printing anything else, matching + # provision.py's own main() -- no point asking the operator to confirm + # a cost warning for a run that cannot possibly proceed. + _require_boto3() + + legs = legs_for(cfg.topology) + print(f"=== accelforge correlation-study orchestration: run_id={cfg.run_id} ===") + print(_COST_WARNING) + print(_estimate_wall_time_message(legs)) + print(f"Legs to profile: {', '.join(legs)} (topology={cfg.topology!r})") + print( + f"Purchasing mode: {cfg.purchasing}. Instance type: {cfg.instance_type}. " + f"Region: {cfg.region}." + ) + + if not args.dry_run and not args.yes: + if not _prompt_yes_no("Proceed with provisioning and profiling? [yes/N]: "): + print("Aborted by user.") + return 1 + + ec2_client = boto3.client("ec2", region_name=cfg.region) + ssm_client = boto3.client("ssm", region_name=cfg.region) + + ssh_cidr = args.ssh_cidr + if not ssh_cidr: + ip = caller_ip() + ssh_cidr = f"{ip}/32" + print(f"SSH will be allowed from: {ssh_cidr}") + + ami_id = resolve_ami(ssm_client, cfg.ami_ssm_parameter) + print(f"Resolved AMI: {ami_id}") + + key_name = f"{cfg.tag_project}-{cfg.run_id}" + key_path = ensure_key_pair(ec2_client, key_name, cfg.key_dir) + print(f"Key pair ready: {key_name} -> {key_path}") + + sg_name = f"{cfg.tag_project}-{cfg.run_id}-sg" + sg_id = ensure_security_group(ec2_client, sg_name, ssh_cidr, cfg.tag_project, cfg.run_id) + print(f"Security group ready: {sg_id}") + + # Mirrors provision.py's main(): the dry_run flag flows all the way + # into launch_instance (which makes a real, DryRun=True API call) so a + # dry run exercises the exact same request-building code path a real + # launch would, rather than short-circuiting before this call. + launch_result = launch_instance( + ec2_client, cfg, ami_id, sg_id, key_name, dry_run=args.dry_run + ) + + if args.dry_run: + print( + f"Dry run complete (purchasing checked: {launch_result['purchasing_used']}); " + "no instance was launched, no SSH was attempted, and no profiling ran. " + "The key pair and security group above WERE created for real -- " + f"clean them up with: python teardown.py --run-id {cfg.run_id} --delete-key" + ) + return 0 + + instance_id = launch_result["instance_id"] + purchasing_used = launch_result["purchasing_used"] + print(f"Launched instance {instance_id} ({purchasing_used}).") + + # See this function's "Design/WHY" docstring Notes above for the full + # rationale: write state now, with public_ip left as an explicit + # placeholder, rather than waiting until wait_for_instance (which can + # block for minutes and can itself fail) returns. + state: Dict[str, Any] = { + "run_id": cfg.run_id, + "region": cfg.region, + "instance_id": instance_id, + "sg_id": sg_id, + "key_name": key_name, + "key_path": str(key_path), + "public_ip": None, + "purchasing_used": purchasing_used, + "ami_id": ami_id, + } + state_path = write_state(cfg.state_dir, cfg.run_id, state) + print(f"State written to: {state_path} (public_ip pending SSH reachability).") + + known_hosts_path = cfg.state_dir / "known_hosts" + run_data_dir = _DATA_DIR / cfg.run_id + + try: + public_ip = wait_for_instance(ec2_client, instance_id) + state["public_ip"] = public_ip + write_state(cfg.state_dir, cfg.run_id, state) + print(f"Instance is running and SSH-reachable at {public_ip}") + print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{public_ip}") + + _push_files(cfg, key_path, public_ip, known_hosts_path) + _run_setup(cfg, key_path, public_ip, known_hosts_path) + + for leg in legs: + _run_leg(cfg, leg, key_path, public_ip, known_hosts_path, run_data_dir) + finally: + # Teardown-in-finally: this block runs whether the try body above + # succeeded, raised (e.g. a failed run_profile.sh -> a propagating + # subprocess.CalledProcessError), or was interrupted -- so an + # (expensive, 8x H100) instance is never left running just because + # one profiling leg failed partway through. See + # _teardown_and_cleanup's docstring for how a *second* failure + # (teardown itself failing) is handled without masking whichever + # exception was already propagating out of the try body. + if args.keep_alive: + _print_keep_alive_notice(cfg, key_path, state) + else: + _teardown_and_cleanup(ec2_client, state, state_path, cfg) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/parse_nccl.py b/notebooks/astrasim2_correlation/correlation/parse_nccl.py new file mode 100644 index 00000000..8a4c6d41 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/parse_nccl.py @@ -0,0 +1,483 @@ +"""Parse raw NCCL collective-communication profiling logs into tidy CSVs. + +This module turns the stdout of two profiling tools into a single, unified +CSV schema that the correlation notebook (``correlation.ipynb``) consumes: + +1. **nccl-tests** (upstream NVIDIA binaries: ``all_reduce_perf``, + ``all_gather_perf``, ``reduce_scatter_perf``, ``alltoall_perf``, + ``broadcast_perf``, ``sendrecv_perf``) run against the fully-connected + (FC) NVSwitch fabric. +2. ``torus_bench`` (a custom binary built by a sibling work package) run + against a torus-topology emulation on the same physical fabric. + +Both tools are parsed independently (:func:`parse_nccl_tests` and +:func:`parse_torus_bench`) and their results are reshaped into the shared +schema documented at :data:`UNIFIED_CSV_FIELDNAMES` before being written to +disk with :func:`rows_to_csv`. The module is also a CLI entry point (see +:func:`main`) so it can be invoked directly from ``run_profile.sh`` on the +profiling instance without any extra Python dependencies. + +Notes +----- +Stdlib-only by design: this script runs on a freshly provisioned EC2 +instance where installing a virtualenv is unwanted overhead. Only ``csv``, +``argparse``, and ``pathlib`` (plus ``sys`` for the CLI entry point) are +used, so it works under any plain ``python3`` >= 3.8. +""" + +from __future__ import annotations + +import argparse +import csv +import sys +from pathlib import Path +from typing import Union + +# Design: the unified schema is a module-level constant (rather than being +# implicit in whatever keys happen to be in the first row dict) so that +# rows_to_csv() always emits a stable, predictable column order regardless +# of which source produced the rows, and so the notebook can rely on the +# header never silently reordering itself as this module evolves. +UNIFIED_CSV_FIELDNAMES: list[str] = [ + "source", + "topology", + "dims", + "collective", + "size_bytes", + "count", + "dtype", + "time_us", + "algbw_GBps", + "busbw_GBps", + "wrong", +] + +# Minimum number of whitespace-separated tokens a nccl-tests data row must +# have before it is considered parseable. A row always carries at least +# size, count, type, redop, root (5 leading columns) plus the 8 trailing +# out-of-place/in-place metric columns = 13 tokens; 10 is used as a looser +# lower bound per the spec so that unexpected/future nccl-tests column +# layouts with slightly fewer leading columns are still accepted as long as +# the trailing-8 structure holds. +_MIN_NCCL_TESTS_TOKENS = 10 + +# Number of trailing tokens on a nccl-tests data row that carry the +# out-of-place/in-place timing results. This is the anchor of the parsing +# strategy; see parse_nccl_tests() docstring for the rationale. +_TRAILING_METRIC_TOKENS = 8 + +# Sentinel line prefix emitted by torus_bench for each data point. Chosen +# by the sibling work package specifically so it is trivial to grep/parse +# out of interleaved '#'-prefixed human-readable log noise. +_TORUS_SENTINEL_PREFIX = "TORUSBENCH," + + +def parse_nccl_tests(text: str) -> list[dict]: + """Parse the stdout of an nccl-tests collective benchmark binary. + + nccl-tests binaries (``all_reduce_perf``, ``all_gather_perf``, + ``reduce_scatter_perf``, ``alltoall_perf``, ``broadcast_perf``, + ``sendrecv_perf``) share a common output shape: a block of ``#``-prefixed + header/comment lines, followed by one data row per message size, followed + by ``#``-prefixed summary lines. The *leading* columns of a data row vary + per collective (e.g. ``redop``/``root`` are meaningless for + ``alltoall_perf`` and print as ``none``/``-1``), but the *trailing* eight + columns are always, in order: out-of-place ``time``, ``algbw``, ``busbw``, + ``#wrong``, then in-place ``time``, ``algbw``, ``busbw``, ``#wrong``. + + Parameters + ---------- + text : str + Raw stdout captured from an nccl-tests binary invocation. May + contain blank lines and ``#``-prefixed comment/header/summary lines + interleaved with data rows. + + Returns + ------- + list of dict + One dict per parsed data row, in file order, with keys: + ``size_bytes`` (int), ``count`` (int), ``dtype`` (str), + ``time_us`` (float), ``algbw_GBps`` (float), ``busbw_GBps`` (float), + ``wrong`` (str; ``"0"``, another digit string, or ``"N/A"`` when + validation was disabled for the run). Only the *out-of-place* + metrics are kept, matching the spec's trailing-token convention; + the in-place metrics are intentionally discarded since the + correlation study only needs one consistent number per size. + + Notes + ----- + Design: rather than hand-writing a distinct column layout per collective + (which would need to track every nccl-tests release), this uses a single + robust rule anchored on the *trailing* 8 tokens, which nccl-tests has + kept stable across collectives and versions even as leading columns + (redop, root) have been added/repurposed. A line is treated as a data + row only if tokens[0] and tokens[1] both parse as int -- this + distinguishes real data rows (which always start with two integers: + size in bytes, element count) from stray non-'#' lines (blank-ish + whitespace, malformed output, or future header formats) without needing + to hard-code the '#' comment convention as the *only* skip signal. + + This function does not raise on malformed input; unparseable lines are + silently skipped so that a partially-corrupt log (e.g. truncated by a + crashed run) still yields whatever valid rows it contains. + + Examples + -------- + >>> text = ( + ... "# nThread 1 nGpus 8\\n" + ... " 1048576 262144 float sum -1 " + ... "98.52 10.64 18.62 0 97.11 10.80 18.90 0\\n" + ... ) + >>> rows = parse_nccl_tests(text) + >>> rows[0]["size_bytes"], rows[0]["time_us"], rows[0]["wrong"] + (1048576, 98.52, '0') + """ + rows: list[dict] = [] + for line in text.splitlines(): + stripped = line.strip() + # Skip blank lines and '#'-prefixed header/comment/summary lines. + if not stripped or stripped.startswith("#"): + continue + + tokens = stripped.split() + if len(tokens) < _MIN_NCCL_TESTS_TOKENS: + continue + + try: + size_bytes = int(tokens[0]) + count = int(tokens[1]) + except ValueError: + # Not a data row (e.g. stray non-'#' text); skip rather than + # raise so one bad line doesn't abort parsing of an otherwise + # good log. + continue + + dtype = tokens[2] + try: + time_us = float(tokens[-_TRAILING_METRIC_TOKENS]) + algbw_gbps = float(tokens[-_TRAILING_METRIC_TOKENS + 1]) + busbw_gbps = float(tokens[-_TRAILING_METRIC_TOKENS + 2]) + except ValueError: + # The trailing columns didn't parse as floats -- not a real + # data row (defensive; shouldn't happen given the int checks + # above already filtered most non-data lines). + continue + wrong = tokens[-_TRAILING_METRIC_TOKENS + 3] + + rows.append( + { + "size_bytes": size_bytes, + "count": count, + "dtype": dtype, + "time_us": time_us, + "algbw_GBps": algbw_gbps, + "busbw_GBps": busbw_gbps, + "wrong": wrong, + } + ) + return rows + + +def parse_torus_bench(text: str) -> list[dict]: + """Parse the stdout of the custom ``torus_bench`` binary. + + ``torus_bench`` prints ``#``-prefixed human-readable header lines plus + machine-parseable sentinel lines of the exact form:: + + TORUSBENCH,,,,, + + Parameters + ---------- + text : str + Raw stdout captured from a ``torus_bench`` invocation. + + Returns + ------- + list of dict + One dict per ``TORUSBENCH,`` sentinel line, in file order, with + keys: ``collective`` (str), ``dims`` (str, e.g. ``"2x2x2"``), + ``size_bytes`` (int), ``time_us`` (float), ``wrong`` (str; one of + ``"0"`` (check passed), ``"1"`` (check failed), or ``"N/A"`` + (validation was not run for this data point)). + + Notes + ----- + Design: the sentinel line is comma-delimited (unlike nccl-tests' + whitespace-delimited columns) specifically so the sibling work package + could emit it without worrying about column-alignment padding; this + parser simply looks for the fixed ``TORUSBENCH,`` prefix and splits on + commas, ignoring every other line (including the human-readable ``#`` + header). This makes the parser forward-compatible with additional + ``#``-prefixed diagnostic lines torus_bench might add later. + + The ``check`` field's three-way encoding (``1``/``0``/``-``) is + remapped onto the same ``wrong`` vocabulary nccl-tests uses (a per-row + "wrongness" indicator string) so downstream consumers (rows_to_csv, + the notebook) can treat the ``wrong`` column uniformly across sources: + ``check == "1"`` (validation ran and passed) maps to ``wrong = "0"`` + (zero wrong elements); ``check == "-"`` (validation was skipped for + this run) maps to ``wrong = "N/A"``, mirroring nccl-tests' own "N/A" + convention for validation-disabled runs; anything else (i.e. + ``check == "0"``, validation ran and failed) maps to ``wrong = "1"``. + + Malformed sentinel lines (wrong field count, non-numeric size/time) are + silently skipped rather than raising, for the same reasons as + :func:`parse_nccl_tests`. + """ + rows: list[dict] = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped.startswith(_TORUS_SENTINEL_PREFIX): + continue + + fields = stripped.split(",") + # TORUSBENCH,,,,, = 6 fields. + if len(fields) != 6: + continue + + _, collective, dims, size_bytes_str, time_us_str, check = fields + try: + size_bytes = int(size_bytes_str) + time_us = float(time_us_str) + except ValueError: + continue + + if check == "1": + wrong = "0" + elif check == "-": + wrong = "N/A" + else: + wrong = "1" + + rows.append( + { + "collective": collective, + "dims": dims, + "size_bytes": size_bytes, + "time_us": time_us, + "wrong": wrong, + } + ) + return rows + + +def rows_to_csv(rows: list[dict], out_path: Union[str, Path]) -> None: + """Write unified-schema rows to a CSV file. + + Parameters + ---------- + rows : list of dict + Rows already reshaped into the unified schema (see + :data:`UNIFIED_CSV_FIELDNAMES` for the exact column set and order). + Each dict must contain every key in ``UNIFIED_CSV_FIELDNAMES``; + missing keys are written as empty cells by :class:`csv.DictWriter` + default behavior is NOT relied upon here -- callers are expected to + supply complete rows (see :func:`main` for how CLI callers build + them). Extra keys beyond the unified schema are rejected by + :class:`csv.DictWriter` (``extrasaction="raise"``, the default) so + schema drift is caught early rather than silently dropped. + out_path : str or pathlib.Path + Destination file path. Parent directories are NOT created by this + function; callers must ensure the directory exists. + + Returns + ------- + None + + Raises + ------ + ValueError + If a row dict contains a key not present in + :data:`UNIFIED_CSV_FIELDNAMES` (raised by the underlying + :class:`csv.DictWriter`). + OSError + If ``out_path`` cannot be opened for writing (e.g. parent directory + does not exist, permission denied). + + Notes + ----- + Opens the file with ``newline=""`` as recommended by the :mod:`csv` + module docs, so that the csv module's own line-ending handling is used + verbatim rather than being double-translated by Python's text-mode + newline translation. + """ + out_path = Path(out_path) + with out_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=UNIFIED_CSV_FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + +def _build_arg_parser() -> argparse.ArgumentParser: + """Construct the CLI argument parser for this module. + + Returns + ------- + argparse.ArgumentParser + Parser accepting the raw log path plus labeling/validation flags + described in the module CLI usage (see :func:`main`). + """ + parser = argparse.ArgumentParser( + prog="parse_nccl.py", + description=( + "Parse a raw nccl-tests or torus_bench profiling log into the " + "unified CSV schema consumed by the correlation notebook." + ), + ) + parser.add_argument( + "raw_log", + type=Path, + help="Path to the raw stdout log captured from the profiling binary.", + ) + parser.add_argument( + "--source", + required=True, + choices=["nccl-tests", "torus_bench"], + help="Which tool produced raw_log.", + ) + parser.add_argument( + "--collective", + required=True, + help=( + "Collective name. For --source nccl-tests this labels every " + "output row directly (the tool's own stdout does not name the " + "collective). For --source torus_bench this is instead " + "cross-checked against the collective embedded in each " + "TORUSBENCH sentinel line; a mismatch is an error." + ), + ) + parser.add_argument( + "--topology", + required=True, + choices=["fc", "torus"], + help="Fabric topology label to stamp onto every output row.", + ) + parser.add_argument( + "--dims", + default=None, + help=( + "Dimension string (e.g. '2x2x2'). For --source nccl-tests this " + "overrides the default dims label of '8' (the fixed GPU count " + "of a single NVSwitch-connected node). For --source " + "torus_bench, if given, it is cross-checked against the dims " + "embedded in each TORUSBENCH sentinel line; a mismatch is an " + "error. If omitted for torus_bench, the sentinel's own dims " + "value is used unchecked." + ), + ) + parser.add_argument( + "--out", + required=True, + type=Path, + help="Destination CSV path.", + ) + return parser + + +# Default dims label for nccl-tests rows when --dims is not supplied on the +# CLI. FC-leg runs are always against a single 8x-H100 NVSwitch node, so "8" +# (the GPU count) is the natural default; --dims exists mainly to let the +# CLI stay uniform with the torus_bench invocation and to support future +# multi-node FC runs without changing this module. +_DEFAULT_NCCL_TESTS_DIMS = "8" + + +def main(argv: Union[list, None] = None) -> None: + """CLI entry point: parse a raw log and write the unified CSV. + + Parameters + ---------- + argv : list of str, optional + Argument vector to parse in place of ``sys.argv[1:]``. Primarily + useful for testing; production invocations (from ``run_profile.sh``) + pass ``None`` and rely on ``sys.argv``. + + Returns + ------- + None + + Raises + ------ + SystemExit + Raised by :mod:`argparse` on invalid/missing arguments (exit code + 2), or explicitly via ``parser.error()`` when a ``--collective``/ + ``--dims`` value supplied on the CLI disagrees with the value + embedded in a torus_bench sentinel line (exit code 2). Also raised + implicitly if ``raw_log`` cannot be read (propagates as an + unhandled :class:`OSError`, not caught here -- a missing/unreadable + input log is a hard setup error the caller (``run_profile.sh``) + should see immediately rather than have masked). + """ + parser = _build_arg_parser() + args = parser.parse_args(argv) + + text = args.raw_log.read_text(encoding="utf-8") + + if args.source == "torus_bench": + parsed = parse_torus_bench(text) + # Design: validate CLI-supplied --collective/--dims against what + # the sentinel lines actually say, rather than trusting the CLI + # blindly. This catches operator error in run_profile.sh (e.g. a + # copy-paste mistake wiring the wrong collective's log into the + # wrong parse invocation) at parse time instead of silently + # mislabeling data that later gets combined into the notebook. + for parsed_row in parsed: + if parsed_row["collective"] != args.collective: + parser.error( + f"--collective {args.collective!r} does not match " + f"collective {parsed_row['collective']!r} found in " + f"{args.raw_log}" + ) + if args.dims is not None and parsed_row["dims"] != args.dims: + parser.error( + f"--dims {args.dims!r} does not match dims " + f"{parsed_row['dims']!r} found in {args.raw_log}" + ) + rows = [ + { + "source": "torus_bench", + "topology": args.topology, + "dims": parsed_row["dims"], + "collective": parsed_row["collective"], + "size_bytes": parsed_row["size_bytes"], + # Design: count/dtype/algbw/busbw are left empty for torus + # rows per spec -- bandwidth conventions for the torus + # topology (e.g. what counts as "algorithm bandwidth" when + # hops differ per link) are derived in the notebook from + # size_bytes/time_us/dims, not computed here, to keep this + # parser topology-agnostic. + "count": "", + "dtype": "", + "time_us": parsed_row["time_us"], + "algbw_GBps": "", + "busbw_GBps": "", + "wrong": parsed_row["wrong"], + } + for parsed_row in parsed + ] + else: + parsed = parse_nccl_tests(text) + dims = args.dims if args.dims is not None else _DEFAULT_NCCL_TESTS_DIMS + rows = [ + { + "source": "nccl-tests", + "topology": args.topology, + "dims": dims, + "collective": args.collective, + "size_bytes": parsed_row["size_bytes"], + "count": parsed_row["count"], + "dtype": parsed_row["dtype"], + "time_us": parsed_row["time_us"], + "algbw_GBps": parsed_row["algbw_GBps"], + "busbw_GBps": parsed_row["busbw_GBps"], + "wrong": parsed_row["wrong"], + } + for parsed_row in parsed + ] + + args.out.parent.mkdir(parents=True, exist_ok=True) + rows_to_csv(rows, args.out) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/notebooks/astrasim2_correlation/correlation/provision.py b/notebooks/astrasim2_correlation/correlation/provision.py new file mode 100644 index 00000000..c6ca9fa4 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/provision.py @@ -0,0 +1,895 @@ +"""Provision one p5.48xlarge (8x H100, NVSwitch) EC2 instance for NCCL profiling. + +This script is the "up" half of the correlation study's empirical leg: it +launches exactly one spot-first, on-demand-fallback instance, waits for it +to be SSH-reachable, and records everything needed to find/tear it down +again in a small JSON state file. See ``teardown.py`` for the "down" half +and ``README.md`` for the full runbook. + +Every function below is written to be independently importable and +testable: ``orchestrate.py`` (a sibling work package, written separately) +imports these functions directly rather than shelling out to this file, so +their signatures are part of this module's public contract and must not +change without updating that caller too. + +Design: boto3 is not a repo dependency +---------------------------------------- +accelforge's ``pyproject.toml`` does not (and per this work package's scope +must not) depend on ``boto3`` -- only this AWS-provisioning corner of one +notebook's correlation study needs it. The import below is therefore +guarded: importing this module never fails just because boto3 is missing, +so ``python provision.py --help`` keeps working in any environment. Actual +AWS calls fail fast with a clear "pip install boto3" message via +:func:`_require_boto3`, called once at the top of :func:`main` before any +client is constructed. +""" + +from __future__ import annotations + +import argparse +import json +import os +import socket +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Optional + +from config import Config + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: # pragma: no cover - exercised only when boto3 truly absent + boto3 = None + # Design: fall back to plain Exception as a placeholder so that + # `except ClientError:` clauses elsewhere in this module remain valid + # Python (no NameError at import time) even when boto3 is missing. + # Those clauses are only ever reached after _require_boto3() has + # already raised, so this placeholder is never actually matched in + # practice -- it exists purely to keep module import side-effect-free. + class ClientError(Exception): # type: ignore[no-redef] + pass + + +def _require_boto3() -> None: + """Raise a clear, actionable error if boto3 is not installed. + + Raises + ------ + SystemExit + Always, if ``boto3`` failed to import. The message tells the user + exactly how to fix it rather than surfacing a bare + ``ModuleNotFoundError`` traceback. + """ + if boto3 is None: + raise SystemExit( + "boto3 is required for AWS provisioning but is not installed in " + "this Python environment.\n" + "Install it with: pip install boto3" + ) + + +# ClientError codes under which "spot-then-ondemand" purchasing retries on +# demand instead of failing the whole run. All represent spot-market +# scarcity/limits rather than a request-shape problem, so retrying the same +# request as on-demand is expected to succeed. p5.48xlarge (8x H100) is a +# scarce, high-demand instance type, so hitting these in practice is not +# unusual and should not be treated as fatal when the operator has opted +# into a fallback. +_SPOT_FALLBACK_ERROR_CODES = frozenset( + { + "InsufficientInstanceCapacity", + "SpotMaxPriceTooLow", + "MaxSpotInstanceCountExceeded", + "Unsupported", + "InstanceLimitExceeded", + } +) + +_SSH_POLL_INTERVAL_S = 5.0 +_SSH_POLL_TIMEOUT_S = 5 * 60.0 + +_COST_WARNING = ( + "COST WARNING: p5.48xlarge on-demand pricing is roughly $30-55/hr " + "depending on region and current AWS pricing. VERIFY CURRENT PRICING " + "before proceeding: https://aws.amazon.com/ec2/pricing/on-demand/" +) + + +def resolve_ami(ssm_client, parameter: str) -> str: + """Resolve an AMI id from a public SSM parameter. + + Parameters + ---------- + ssm_client : botocore.client.BaseClient + A boto3 ``ssm`` client (or a stub thereof). + parameter : str + Fully-qualified SSM parameter name, e.g. + ``"/aws/service/deeplearning/ami/x86_64/.../latest/ami-id"``. + + Returns + ------- + str + The AMI id stored at ``parameter``. + + Raises + ------ + botocore.exceptions.ClientError + If ``parameter`` does not exist or the caller lacks + ``ssm:GetParameter`` permission; propagated unmodified so callers + see AWS's own error code and message. + """ + response = ssm_client.get_parameter(Name=parameter) + return response["Parameter"]["Value"] + + +def caller_ip() -> str: + """Discover the caller's public IPv4 address via checkip.amazonaws.com. + + Used by :func:`main` to scope the provisioned security group's SSH + ingress rule to just this machine, when the operator has not supplied + an explicit ``--ssh-cidr``. + + Returns + ------- + str + The caller's public IP as a dotted-quad string. + + Raises + ------ + RuntimeError + If the IP could not be discovered for any reason (network error, + timeout, or an empty response body). See Notes for why this fails + loudly rather than falling back to any sentinel value. + + Notes + ----- + Design: fails *fast* (raises) rather than falling back to a sentinel. + An earlier version of this function failed *open* to the sentinel + ``"0.0.0.0"`` on discovery failure, reasoning that a transient DNS blip + or checkip.amazonaws.com outage shouldn't hard-fail an + otherwise-working run. That reasoning had a bug: every caller turns + this return value into a CIDR via ``f"{ip}/32"``, so the sentinel + actually produced ``"0.0.0.0/32"`` -- a CIDR matching no address at + all -- which locks *everyone*, including the operator, out over SSH, + the opposite of what the old warning text claimed ("falling back to + 0.0.0.0/0", i.e. open to the world). Worse, that silent misconfiguration + was only discoverable *after* a real key pair, security group, and + instance had already been created and billing had already started. + Raising here instead is strictly better on both axes this function + cares about -- SAFE (no accidental everyone-blocked security group) and + SECURE (no accidental world-open one either) -- and it fires before any + AWS resource exists or any money is spent: both call sites + (``provision.main`` and ``orchestrate.main``) invoke this function only + when ``--ssh-cidr`` was not supplied, and always before + ``ensure_key_pair``/``ensure_security_group``/``launch_instance``. The + error message tells the operator exactly how to proceed: re-run with + ``--ssh-cidr /32``. + """ + try: + with urllib.request.urlopen("https://checkip.amazonaws.com", timeout=10) as resp: + ip = resp.read().decode("utf-8").strip() + if not ip: + raise ValueError("empty response body from checkip.amazonaws.com") + return ip + except (urllib.error.URLError, ValueError, OSError) as exc: + raise RuntimeError( + "Could not determine your public IP via checkip.amazonaws.com " + f"({exc!r}). No AWS resources have been created yet, so there is " + "nothing to clean up -- re-run with --ssh-cidr /32 to " + "supply your CIDR explicitly instead of relying on auto-detection." + ) from exc + + +def ensure_key_pair(ec2_client, key_name: str, key_dir: Path) -> Path: + """Create (or reuse) an EC2 key pair and its local PEM file. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + key_name : str + Name to give the key pair in AWS. + key_dir : pathlib.Path + Local directory to write ``.pem`` into. Created if it + does not already exist. + + Returns + ------- + pathlib.Path + Path to the local PEM file (either freshly written, or the + existing one being reused). + + Raises + ------ + RuntimeError + If AWS reports the key pair already exists (``ClientError`` code + ``InvalidKeyPair.Duplicate``) but no local PEM file is present. + AWS never returns private key material for a pre-existing key + pair, so there is no way to recover the PEM in this situation -- + the caller must delete the AWS-side key pair or choose a + different ``run_id``. + botocore.exceptions.ClientError + For any other ``create_key_pair`` failure, propagated unmodified. + + Notes + ----- + Side effect: writes a file to disk at ``/.pem`` with + ``0o600`` permissions (owner read/write only, matching what ``ssh`` + requires of private key files). + """ + key_dir.mkdir(parents=True, exist_ok=True) + key_path = key_dir / f"{key_name}.pem" + + try: + response = ec2_client.create_key_pair( + KeyName=key_name, KeyType="rsa", KeyFormat="pem" + ) + except ClientError as exc: + error_code = exc.response.get("Error", {}).get("Code", "") + if error_code == "InvalidKeyPair.Duplicate": + if key_path.exists(): + print( + f"Key pair {key_name!r} already exists in AWS and a local " + f"PEM was found at {key_path}; reusing it." + ) + return key_path + raise RuntimeError( + f"Key pair {key_name!r} already exists in AWS, but no local " + f"PEM file was found at {key_path}. AWS never returns private " + "key material for a pre-existing key pair, so it cannot be " + "recovered. Either delete the AWS-side key pair " + f"(aws ec2 delete-key-pair --key-name {key_name}) and re-run, " + "or pass a different --run-id so a fresh key pair name is used." + ) from exc + raise + + key_material = response["KeyMaterial"] + # Design: use os.open with the 0o600 mode baked into file creation + # (rather than write-then-chmod) so the PEM is never briefly readable + # at default (often world-readable) permissions between those two + # steps. + fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as fh: + fh.write(key_material) + return key_path + + +def ensure_security_group( + ec2_client, group_name: str, ssh_cidr: str, tag_project: str, run_id: str +) -> str: + """Create a security group in the default VPC allowing SSH from one CIDR. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + group_name : str + Name to give the new security group. + ssh_cidr : str + CIDR block (e.g. ``"1.2.3.4/32"`` or ``"0.0.0.0/0"``) to allow + inbound TCP/22 from. + tag_project : str + Value for the ``Project`` tag on the new group. + run_id : str + Value for the ``RunId`` tag on the new group, and included in its + description. + + Returns + ------- + str + The new security group's id. + + Raises + ------ + RuntimeError + If the region has no default VPC (``describe_vpcs`` returns no + results for ``isDefault=true``). NCCL profiling on a single node + has no cross-VPC requirements, so this script deliberately does + not attempt to create or select a non-default VPC -- that is out + of scope for a short-lived profiling instance. + botocore.exceptions.ClientError + For any ``create_security_group``/``authorize_security_group_ingress``/ + ``create_tags`` failure, propagated unmodified. + """ + vpcs = ec2_client.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}]) + vpc_list = vpcs.get("Vpcs", []) + if not vpc_list: + raise RuntimeError( + "No default VPC found in this region. Create one " + "(aws ec2 create-default-vpc) or provision a VPC manually, then " + "re-run." + ) + vpc_id = vpc_list[0]["VpcId"] + + create_resp = ec2_client.create_security_group( + GroupName=group_name, + Description=f"accelforge correlation study SG for run {run_id}", + VpcId=vpc_id, + ) + sg_id = create_resp["GroupId"] + + ec2_client.authorize_security_group_ingress( + GroupId=sg_id, + IpPermissions=[ + { + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "IpRanges": [ + { + "CidrIp": ssh_cidr, + "Description": "SSH access for accelforge correlation study", + } + ], + } + ], + ) + + ec2_client.create_tags( + Resources=[sg_id], + Tags=[ + {"Key": "Project", "Value": tag_project}, + {"Key": "RunId", "Value": run_id}, + {"Key": "Name", "Value": group_name}, + ], + ) + return sg_id + + +def _build_run_instances_kwargs( + cfg: Config, ami_id: str, sg_id: str, key_name: str, dry_run: bool, use_spot: bool +) -> dict: + """Build the ``run_instances`` kwargs shared by the spot and on-demand paths. + + Parameters + ---------- + cfg : Config + Run configuration. + ami_id : str + AMI id resolved by :func:`resolve_ami`. + sg_id : str + Security group id from :func:`ensure_security_group`. + key_name : str + Key pair name from :func:`ensure_key_pair`. + dry_run : bool + Whether to set ``DryRun=True`` on the request. + use_spot : bool + Whether to request a spot instance (adds ``InstanceMarketOptions``) + or an on-demand one. + + Returns + ------- + dict + Keyword arguments ready to pass to ``ec2_client.run_instances(**kwargs)``. + + Notes + ----- + Factored out of :func:`launch_instance` so the spot attempt and the + on-demand fallback attempt build their request the same way apart from + the one ``InstanceMarketOptions`` difference -- avoids the two paths + silently drifting apart (e.g. one attempt forgetting a tag) as this + function evolves. + """ + kwargs = { + "ImageId": ami_id, + "InstanceType": cfg.instance_type, + "KeyName": key_name, + "SecurityGroupIds": [sg_id], + "MinCount": 1, + "MaxCount": 1, + # "terminate" (not "stop") so an in-instance `shutdown` -- e.g. the + # dead-man timer armed by setup scripts -- fully releases the + # instance rather than leaving it (and its EBS billing) stopped + # but still provisioned. + "InstanceInitiatedShutdownBehavior": "terminate", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "VolumeSize": cfg.root_volume_gb, + "VolumeType": "gp3", + "DeleteOnTermination": True, + }, + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + {"Key": "Project", "Value": cfg.tag_project}, + {"Key": "RunId", "Value": cfg.run_id}, + {"Key": "Name", "Value": f"{cfg.tag_project}-{cfg.run_id}"}, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + {"Key": "Project", "Value": cfg.tag_project}, + {"Key": "RunId", "Value": cfg.run_id}, + {"Key": "Name", "Value": f"{cfg.tag_project}-{cfg.run_id}"}, + ], + }, + ], + "DryRun": dry_run, + } + if cfg.availability_zone: + kwargs["Placement"] = {"AvailabilityZone": cfg.availability_zone} + if use_spot: + kwargs["InstanceMarketOptions"] = { + "MarketType": "spot", + "SpotOptions": { + "SpotInstanceType": "one-time", + "InstanceInterruptionBehavior": "terminate", + }, + } + return kwargs + + +def launch_instance( + ec2_client, + cfg: Config, + ami_id: str, + sg_id: str, + key_name: str, + dry_run: bool = False, +) -> dict: + """Launch exactly one instance, honoring ``cfg.purchasing``. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + cfg : Config + Run configuration; ``cfg.purchasing`` selects the strategy below. + ami_id : str + AMI id resolved by :func:`resolve_ami`. + sg_id : str + Security group id from :func:`ensure_security_group`. + key_name : str + Key pair name from :func:`ensure_key_pair`. + dry_run : bool, default False + If ``True``, sets ``DryRun=True`` on every ``run_instances`` call. + AWS answers a dry run with an error either way: ``DryRunOperation`` + means the call would have succeeded, ``UnauthorizedOperation`` + means the caller lacks permission. This function treats those two + codes accordingly rather than as generic failures. + + Returns + ------- + dict + ``{"instance_id": str or None, "purchasing_used": "spot" or "ondemand"}``. + ``instance_id`` is ``None`` when ``dry_run=True`` and the + authorization check succeeded, since no instance was actually + created in that case. + + Raises + ------ + RuntimeError + If a dry run reports ``UnauthorizedOperation`` (the configured + credentials cannot launch this instance type/configuration). + botocore.exceptions.ClientError + - If ``cfg.purchasing == "spot"`` and the spot request fails for + any reason (no fallback is attempted in this mode). + - If ``cfg.purchasing == "spot-then-ondemand"`` and the spot + request fails with a code *not* in + :data:`_SPOT_FALLBACK_ERROR_CODES` (that set is deliberately + narrow -- e.g. a malformed request should fail loudly rather + than silently retrying as on-demand and masking the bug). + - If the (possibly-fallback) on-demand request itself fails. + + Notes + ----- + Purchasing strategies: + + - ``"ondemand"``: on-demand only, no spot attempt. + - ``"spot"``: spot only; any failure propagates without a fallback. + - ``"spot-then-ondemand"`` (the default): attempts spot first. If that + attempt fails with one of :data:`_SPOT_FALLBACK_ERROR_CODES` -- + capacity/limit/market conditions rather than a malformed request -- + it prints the failure and retries once as on-demand. Any other + ``ClientError`` code (e.g. a parameter validation error) propagates + immediately without a fallback attempt, since retrying on-demand + would not fix a malformed request and would only obscure the real + error. + """ + + def _run(use_spot: bool) -> dict: + kwargs = _build_run_instances_kwargs(cfg, ami_id, sg_id, key_name, dry_run, use_spot) + try: + response = ec2_client.run_instances(**kwargs) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "DryRunOperation": + # AWS's DryRun contract: this specific error code means + # "you WOULD have been authorized to make this call" -- it + # is deliberately raised as an error even on the success + # path, so seeing it here IS the successful outcome of a + # dry run, not a failure. + print("dry-run OK: authorized") + return { + "instance_id": None, + "purchasing_used": "spot" if use_spot else "ondemand", + } + if code == "UnauthorizedOperation": + raise RuntimeError( + "AWS denied the run_instances permission check " + "(UnauthorizedOperation). The configured credentials lack " + f"ec2:RunInstances (or a related) permission for " + f"{cfg.instance_type}." + ) from exc + raise + instance_id = response["Instances"][0]["InstanceId"] + return { + "instance_id": instance_id, + "purchasing_used": "spot" if use_spot else "ondemand", + } + + if cfg.purchasing == "ondemand": + return _run(use_spot=False) + + if cfg.purchasing == "spot": + return _run(use_spot=True) + + # cfg.purchasing == "spot-then-ondemand" (validated by Config.__post_init__ + # to be one of exactly these three values). + try: + return _run(use_spot=True) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in _SPOT_FALLBACK_ERROR_CODES: + print(f"Spot request failed ({code}); falling back to on-demand.") + return _run(use_spot=False) + raise + + +def wait_for_instance(ec2_client, instance_id: str) -> str: + """Block until an instance is running and accepting TCP connections on port 22. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + instance_id : str + Id of the instance to wait for. + + Returns + ------- + str + The instance's public IPv4 address. + + Raises + ------ + RuntimeError + If ``describe_instances`` returns no matching instance, or the + instance has no public IP address (e.g. it landed in a subnet that + does not auto-assign one). + TimeoutError + If port 22 does not become reachable within + :data:`_SSH_POLL_TIMEOUT_S` seconds of the instance reaching the + ``running`` state. + botocore.exceptions.WaiterError + If the ``instance_running`` waiter itself times out or the + instance transitions to a terminal failure state. + + Notes + ----- + Two-stage wait, because "EC2 says running" and "sshd is accepting + connections" are different events with a real gap between them (boot, + cloud-init, driver/NCCL setup on the deep learning AMI): first the + ``instance_running`` waiter (AWS-side state), then a plain TCP connect + poll against port 22 (this script's own liveness check), every + :data:`_SSH_POLL_INTERVAL_S` seconds for up to + :data:`_SSH_POLL_TIMEOUT_S`. + """ + waiter = ec2_client.get_waiter("instance_running") + waiter.wait(InstanceIds=[instance_id]) + + describe = ec2_client.describe_instances(InstanceIds=[instance_id]) + reservations = describe.get("Reservations", []) + if not reservations or not reservations[0].get("Instances"): + raise RuntimeError( + f"describe_instances returned no data for instance {instance_id!r}" + ) + instance = reservations[0]["Instances"][0] + public_ip = instance.get("PublicIpAddress") + if not public_ip: + raise RuntimeError( + f"Instance {instance_id} is running but has no public IP address. " + "Check that its subnet auto-assigns public IPs." + ) + + _wait_for_ssh_port(public_ip) + return public_ip + + +def _wait_for_ssh_port( + host: str, + port: int = 22, + interval_s: float = _SSH_POLL_INTERVAL_S, + timeout_s: float = _SSH_POLL_TIMEOUT_S, +) -> None: + """Poll a TCP port until it accepts a connection or a timeout elapses. + + Parameters + ---------- + host : str + Hostname or IP address to connect to. + port : int, default 22 + TCP port to poll. + interval_s : float, default 5.0 + Seconds to sleep between connection attempts. + timeout_s : float, default 300.0 + Total seconds to poll before giving up. + + Raises + ------ + TimeoutError + If no connection succeeds within ``timeout_s`` seconds. + + Notes + ----- + Uses ``socket.create_connection`` (a plain TCP connect/close) rather + than an actual SSH handshake -- sufficient to confirm sshd is up + without adding a paramiko/fabric dependency for a single boolean + liveness check. + """ + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + with socket.create_connection((host, port), timeout=interval_s): + return + except OSError: + pass + time.sleep(interval_s) + raise TimeoutError( + f"Timed out after {timeout_s}s waiting for {host}:{port} to accept " + "TCP connections (SSH not yet reachable)." + ) + + +def write_state(state_dir, run_id: str, state: dict) -> Path: + """Write a run's provisioning state to ``/.json``. + + Parameters + ---------- + state_dir : str or pathlib.Path + Directory to write the state file into. Created if it does not + already exist. + run_id : str + Run identifier; also the state file's basename (without ``.json``). + state : dict + JSON-serializable state to write. Expected (by ``teardown.py``) to + contain ``run_id``, ``region``, ``instance_id``, ``sg_id``, + ``key_name``, ``key_path``, ``public_ip``, ``purchasing_used``, and + ``ami_id``, but this function itself does not validate the shape + of ``state`` -- it is a thin, schema-agnostic writer so callers + (including future ones) are free to add fields. + + Returns + ------- + pathlib.Path + Path to the written state file. + + Notes + ----- + Side effect: writes ``/.json``, creating + ``state_dir`` if needed. Uses ``json.dump(..., default=str)`` so a + stray non-JSON-native value (e.g. if a caller forgets to stringify a + ``pathlib.Path``) is coerced to its string form instead of raising a + ``TypeError`` deep in a provisioning run. + """ + state_dir = Path(state_dir) + state_dir.mkdir(parents=True, exist_ok=True) + state_path = state_dir / f"{run_id}.json" + with open(state_path, "w") as fh: + json.dump(state, fh, indent=2, default=str) + return state_path + + +def _prompt_yes_no(prompt: str) -> bool: + """Ask an interactive yes/no question, returning ``True`` only for "yes". + + Parameters + ---------- + prompt : str + Text to show before the input cursor. + + Returns + ------- + bool + ``True`` if the user typed exactly ``"yes"`` (case-insensitive, + surrounding whitespace ignored); ``False`` for anything else, + including EOF/blank input. Requiring the full word "yes" (not just + "y") is a deliberate speed bump before an action that costs real + money. + """ + try: + answer = input(prompt) + except EOFError: + return False + return answer.strip().lower() == "yes" + + +def main(argv: Optional[list] = None) -> int: + """Parse CLI args and provision one instance end to end. + + Parameters + ---------- + argv : list[str] or None, default None + Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` + reads from ``sys.argv[1:]``. + + Returns + ------- + int + Process exit code: ``0`` on success (or a completed dry run), + ``1`` if the user declined the cost confirmation prompt. + + Notes + ----- + Side effects (skipped entirely when ``--dry-run`` is passed, except + for the ``DryRun=True`` API calls themselves): creates an SSH key pair + and local PEM, creates a security group, launches an instance, waits + for it to be SSH-reachable, and writes a state JSON file -- TWICE: once + immediately after launch (``public_ip`` as an explicit placeholder) and + again once ``wait_for_instance`` returns a real IP, so a crash or + interruption during the (potentially multi-minute) SSH wait still + leaves a local record of a running, billing instance. See the inline + "Design/WHY" comment at that first ``write_state`` call for the full + rationale. Prints a cost warning and requires interactive ``"yes"`` + confirmation before any of that happens, unless ``--yes`` is passed. + + This function deliberately never tears anything down itself, even on a + ``wait_for_instance`` failure -- ``provision.py``'s contract is + provision-and-leave-running (see ``README.md``'s "advanced / manual + control" section); recovery in that failure case is a printed + ``teardown.py`` command for the operator to run, not an automatic call. + """ + parser = argparse.ArgumentParser( + prog="provision.py", + description=( + "Provision one p5.48xlarge (8x H100, NVSwitch) EC2 instance, " + "spot-first with on-demand fallback, for NCCL profiling." + ), + ) + Config.add_args(parser) + parser.add_argument( + "--dry-run", + action="store_true", + help="Perform DryRun=True authorization checks only; launch nothing.", + ) + parser.add_argument( + "--ssh-cidr", + type=str, + default=None, + help=( + "CIDR block allowed SSH access, e.g. 1.2.3.4/32. Overrides " + "auto-detected caller IP (see caller_ip())." + ), + ) + parser.add_argument( + "--yes", + action="store_true", + help="Skip the interactive cost-confirmation prompt.", + ) + args = parser.parse_args(argv) + cfg = Config.from_parsed(args) + + # Fail fast on a missing boto3 before printing anything else -- no + # point asking the operator to confirm a cost warning for a run that + # cannot possibly proceed. + _require_boto3() + + print(f"=== accelforge correlation-study provisioning: run_id={cfg.run_id} ===") + print(_COST_WARNING) + print( + f"Purchasing mode: {cfg.purchasing}. Instance type: {cfg.instance_type}. " + f"Region: {cfg.region}." + ) + + if not args.dry_run and not args.yes: + if not _prompt_yes_no("Proceed with provisioning? [yes/N]: "): + print("Aborted by user.") + return 1 + + ec2_client = boto3.client("ec2", region_name=cfg.region) + ssm_client = boto3.client("ssm", region_name=cfg.region) + + ssh_cidr = args.ssh_cidr + if not ssh_cidr: + ip = caller_ip() + ssh_cidr = f"{ip}/32" + print(f"SSH will be allowed from: {ssh_cidr}") + + ami_id = resolve_ami(ssm_client, cfg.ami_ssm_parameter) + print(f"Resolved AMI: {ami_id}") + + key_name = f"{cfg.tag_project}-{cfg.run_id}" + key_path = ensure_key_pair(ec2_client, key_name, cfg.key_dir) + print(f"Key pair ready: {key_name} -> {key_path}") + + sg_name = f"{cfg.tag_project}-{cfg.run_id}-sg" + sg_id = ensure_security_group(ec2_client, sg_name, ssh_cidr, cfg.tag_project, cfg.run_id) + print(f"Security group ready: {sg_id}") + + launch_result = launch_instance( + ec2_client, cfg, ami_id, sg_id, key_name, dry_run=args.dry_run + ) + if args.dry_run: + print( + f"Dry run complete (purchasing checked: {launch_result['purchasing_used']}); " + "no instance was launched." + ) + return 0 + + instance_id = launch_result["instance_id"] + purchasing_used = launch_result["purchasing_used"] + print(f"Launched instance {instance_id} ({purchasing_used}).") + + # Design/WHY (closes an orphan-instance window): write the state file -- + # with public_ip as an explicit placeholder -- IMMEDIATELY after + # launch_instance returns, before wait_for_instance is even called. + # wait_for_instance can block for several minutes (instance boot, then + # polling for SSH) and can itself raise (TimeoutError, WaiterError, or + # simply be interrupted by Ctrl-C/a killed process). Before this fix, a + # crash in that window left an instance running and billing with NO + # local record of it at all, since write_state was only ever called + # once, after wait_for_instance returned successfully. Writing state + # now -- and rewriting it once the real public_ip is known below -- + # means teardown.py can always find and tear down this run from local + # state alone, even if this process never gets past wait_for_instance. + state = { + "run_id": cfg.run_id, + "region": cfg.region, + "instance_id": instance_id, + "sg_id": sg_id, + "key_name": key_name, + "key_path": str(key_path), + "public_ip": None, + "purchasing_used": purchasing_used, + "ami_id": ami_id, + } + state_path = write_state(cfg.state_dir, cfg.run_id, state) + print(f"State written to: {state_path} (public_ip pending SSH reachability).") + + print("Waiting for it to become reachable...") + try: + public_ip = wait_for_instance(ec2_client, instance_id) + except Exception: + # The instance is still running (and billing) regardless of why + # wait_for_instance failed -- print a loud, impossible-to-miss block + # naming exactly how to find and recover it, then re-raise + # unmodified so this failure still surfaces as a nonzero exit code + # (never silently swallowed). + print("=" * 70, file=sys.stderr) + print("ERROR: wait_for_instance failed (see traceback below).", file=sys.stderr) + print(f"Instance {instance_id} IS STILL RUNNING AND BILLING.", file=sys.stderr) + print(f"State file: {state_path}", file=sys.stderr) + print( + f"Recover with: python teardown.py --run-id {cfg.run_id} --region {cfg.region}", + file=sys.stderr, + ) + print("=" * 70, file=sys.stderr) + raise + + print(f"Instance is running and SSH-reachable at {public_ip}") + + state["public_ip"] = public_ip + state_path = write_state(cfg.state_dir, cfg.run_id, state) + + print(f"State written to: {state_path}") + print(f"Public IP: {public_ip}") + print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{public_ip}") + print( + "Remember: tear this down when done " + f"(python teardown.py --run-id {cfg.run_id})." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/run_profile.sh b/notebooks/astrasim2_correlation/correlation/run_profile.sh new file mode 100644 index 00000000..372f580e --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/run_profile.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# run_profile.sh -- run one profiling leg (FC or torus) for one or more +# NCCL collectives on a provisioned 8x-H100 instance, and parse each raw +# log into the unified CSV schema consumed by the correlation notebook. +# +# This script runs ON the EC2 instance (not locally); it has no GPUs to +# talk to when checked out anywhere else, so it is verified here by +# `bash -n` (syntax check) only -- see the WP2 report for details. +# +# Usage: +# ./run_profile.sh \ +# [...] +# +# Env overrides: +# NCCL_TESTS_DIR Path to a built nccl-tests checkout. Default: $HOME/nccl-tests +# TORUS_BENCH_BIN Path to the built torus_bench binary. Default: $HOME/torus_bench/torus_bench +# PARSE_NCCL Path to parse_nccl.py. Default: the copy next to this script. +# WARMUP nccl-tests/torus_bench warmup iteration count. Default: 5 +# ITERS nccl-tests/torus_bench measured iteration count. Default: 20 +# +# Design: fail fast and loud (set -euo pipefail) rather than silently +# continuing past a failed collective run or a failed parse -- a partial, +# uncaught failure here would otherwise show up much later as a confusing +# gap in the correlation notebook's data rather than as a build/run error +# on the instance where it's cheap to diagnose. +set -euo pipefail + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- +if [[ $# -lt 6 ]]; then + echo "Usage: $0 [...]" >&2 + exit 1 +fi + +results_dir="$1"; shift +topology="$1"; shift +min_bytes="$1"; shift +max_bytes="$1"; shift +dims="$1"; shift +# Remaining positional args are the list of collectives to profile in this leg. +collectives=("$@") + +if [[ "$topology" != "fc" && "$topology" != "torus" ]]; then + echo "ERROR: must be 'fc' or 'torus', got '$topology'" >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Environment / defaults +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +NCCL_TESTS_DIR="${NCCL_TESTS_DIR:-$HOME/nccl-tests}" +TORUS_BENCH_BIN="${TORUS_BENCH_BIN:-$HOME/torus_bench/torus_bench}" +# Design: PARSE_NCCL defaults to the copy sitting next to this script +# (rather than requiring it on PATH or hard-coding an absolute install +# path) so the pair of files can be scp'd to the instance as a unit and +# just work. +PARSE_NCCL="${PARSE_NCCL:-$SCRIPT_DIR/parse_nccl.py}" +WARMUP="${WARMUP:-5}" +ITERS="${ITERS:-20}" + +RAW_DIR="$results_dir/raw" +CSV_DIR="$results_dir/csv" +mkdir -p "$RAW_DIR" "$CSV_DIR" + +# --------------------------------------------------------------------------- +# metadata.txt: written exactly once per invocation (not once per +# collective), since it captures machine/software state that doesn't +# change across the collectives loop below. +# --------------------------------------------------------------------------- +metadata_file="$results_dir/metadata.txt" +{ + echo "=== date ===" + date -u + echo + echo "=== uname -a ===" + uname -a + echo + echo "=== nvidia-smi --query-gpu=name,driver_version --format=csv ===" + nvidia-smi --query-gpu=name,driver_version --format=csv + echo + echo "=== nvidia-smi topo -m ===" + nvidia-smi topo -m + echo + echo "=== nccl-tests git rev ===" + # Best-effort: nccl-tests may not be a git checkout (e.g. if a tarball + # was scp'd instead), so a failed `git rev-parse` here must not abort + # the whole script under `set -e`. + git -C "$NCCL_TESTS_DIR" rev-parse HEAD 2>/dev/null || echo "unknown (not a git checkout or NCCL_TESTS_DIR missing)" +} > "$metadata_file" +echo "Wrote $metadata_file" + +# --------------------------------------------------------------------------- +# FC leg: stock nccl-tests binaries. +# --------------------------------------------------------------------------- +run_fc_leg() { + local collective="$1" + local bin="$NCCL_TESTS_DIR/build/${collective}_perf" + local raw_log="$RAW_DIR/fc_${collective}.log" + local out_csv="$CSV_DIR/fc_${collective}.csv" + + if [[ ! -x "$bin" ]]; then + echo "ERROR: nccl-tests binary not found: $bin" >&2 + echo " Build nccl-tests first, e.g. via setup_node.sh, or:" >&2 + echo " make -j -C \"$NCCL_TESTS_DIR\" MPI=0 CUDA_HOME=\"\${CUDA_HOME:-/usr/local/cuda}\"" >&2 + exit 1 + fi + + echo "=== FC leg: $collective ===" + "$bin" -b "$min_bytes" -e "$max_bytes" -f 2 -g 8 -w "$WARMUP" -n "$ITERS" -c 1 | tee "$raw_log" + python3 "$PARSE_NCCL" "$raw_log" \ + --source nccl-tests \ + --collective "$collective" \ + --topology fc \ + --dims "$dims" \ + --out "$out_csv" + + # Fail loudly on a header-only CSV: parse_nccl.py can exit 0 while emitting zero data rows + # (e.g. every swept size hit a divisibility skip for this dims/size combination), which + # would otherwise look identical to a real, successful leg -- a silent data gap discovered + # only much later in the correlation notebook, rather than here, where the raw log needed + # to diagnose it is still on disk and cheap to inspect. + csv_lines="$(wc -l < "$out_csv")" + if [[ "$csv_lines" -le 1 ]]; then + echo "ERROR: $out_csv has no data rows (header-only or empty) -- see $raw_log" >&2 + exit 1 + fi + echo "Wrote $out_csv" +} + +# --------------------------------------------------------------------------- +# Torus leg: custom torus_bench binary (built by the sibling work package). +# --------------------------------------------------------------------------- +run_torus_leg() { + local collective="$1" + local raw_log="$RAW_DIR/torus_${collective}.log" + local out_csv="$CSV_DIR/torus_${collective}.csv" + + if [[ ! -x "$TORUS_BENCH_BIN" ]]; then + echo "ERROR: torus_bench binary not found: $TORUS_BENCH_BIN" >&2 + echo " Build torus_bench first (see setup_node.sh), e.g.:" >&2 + echo " make -C \"\$TORUS_BENCH_DIR\" torus_bench" >&2 + exit 1 + fi + + echo "=== Torus leg: $collective ===" + "$TORUS_BENCH_BIN" --collective "$collective" --dims "$dims" \ + -b "$min_bytes" -e "$max_bytes" -f 2 -w "$WARMUP" -n "$ITERS" --check \ + | tee "$raw_log" + python3 "$PARSE_NCCL" "$raw_log" \ + --source torus_bench \ + --collective "$collective" \ + --topology torus \ + --dims "$dims" \ + --out "$out_csv" + + # See run_fc_leg's identical check above for the full rationale: a header-only CSV here + # (e.g. every swept size hit torus_bench's own divisibility SKIP for this dims/size + # combination) must fail the script loudly rather than silently passing as "done". + csv_lines="$(wc -l < "$out_csv")" + if [[ "$csv_lines" -le 1 ]]; then + echo "ERROR: $out_csv has no data rows (header-only or empty) -- see $raw_log" >&2 + exit 1 + fi + echo "Wrote $out_csv" +} + +# --------------------------------------------------------------------------- +# Main loop: one leg (selected by $topology), all requested collectives. +# --------------------------------------------------------------------------- +for collective in "${collectives[@]}"; do + if [[ "$topology" == "fc" ]]; then + run_fc_leg "$collective" + else + run_torus_leg "$collective" + fi +done + +echo "Done. Results in $results_dir" diff --git a/notebooks/astrasim2_correlation/correlation/setup_node.sh b/notebooks/astrasim2_correlation/correlation/setup_node.sh new file mode 100644 index 00000000..ec407e99 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/setup_node.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# +# setup_node.sh -- idempotent one-time (but safe-to-rerun) setup for an +# EC2 p5.48xlarge (8x H100, NVSwitch) profiling instance: arms a dead-man +# shutdown, verifies the GPU fabric is visible, builds nccl-tests and +# torus_bench if not already built, and prints a versions summary. +# +# This script runs ON the EC2 instance; it is verified here by `bash -n` +# (syntax check) only -- see the WP2 report for details. +# +# Env: +# DEADMAN_MINUTES Minutes until the dead-man shutdown fires. Default: 120 +# NCCL_TESTS_DIR Where to clone/build nccl-tests. Default: $HOME/nccl-tests +# TORUS_BENCH_DIR Where torus_bench is scp'd/built. Default: $HOME/torus_bench +# CUDA_HOME CUDA toolkit root used to build nccl-tests. Default: /usr/local/cuda +set -euo pipefail + +DEADMAN_MINUTES="${DEADMAN_MINUTES:-120}" +NCCL_TESTS_DIR="${NCCL_TESTS_DIR:-$HOME/nccl-tests}" +TORUS_BENCH_DIR="${TORUS_BENCH_DIR:-$HOME/torus_bench}" +CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" + +# --------------------------------------------------------------------------- +# Step 1: arm the dead-man switch FIRST, before anything else can fail or +# hang. +# +# Design/WHY this must be first: the instance is launched with +# shutdown-behavior=terminate, so a `shutdown -P` here is what actually +# terminates (not just stops) the instance and stops billing. If setup +# were to fail, hang (e.g. a stuck `make`, a stalled git clone over a flaky +# network), or if the orchestrating controller process on the caller's side +# dies/loses connectivity, this is the only backstop that guarantees the +# (expensive, 8x H100) instance doesn't run forever. Arming it before any +# other step -- including the GPU sanity check below, which could itself +# hang on a broken driver -- ensures the cost cap applies unconditionally +# from the very start of setup, not only after setup "succeeds". +# --------------------------------------------------------------------------- +echo "Arming dead-man shutdown: instance will terminate in ${DEADMAN_MINUTES} minutes unless this script (or a later run of it) is used to push it back further." +# Cancel any already-pending shutdown before arming a new one: issuing a second `shutdown` +# while one is already pending errors on some systemd versions, and cancel-then-rearm also +# makes a re-run of this script push the deadline BACK (rather than erroring or stacking), +# which is the desired semantics for a legitimate re-setup (e.g. extending a long-running +# sweep with a fresh DEADMAN_MINUTES). `|| true` because there being no pending shutdown to +# cancel (the common case, e.g. this script's first run) is not an error. +sudo shutdown -c 2>/dev/null || true +sudo shutdown -P "+${DEADMAN_MINUTES}" + +# --------------------------------------------------------------------------- +# Step 2: verify nvidia-smi works and all 8 GPUs are visible. Fail loudly +# (rather than proceeding to build against a broken/partial driver) since +# every downstream profiling run depends on this. +# --------------------------------------------------------------------------- +if ! command -v nvidia-smi >/dev/null 2>&1; then + echo "ERROR: nvidia-smi not found on PATH. Is the NVIDIA driver installed?" >&2 + exit 1 +fi + +if ! nvidia-smi >/dev/null; then + echo "ERROR: nvidia-smi is present but failed to run. Driver/GPU problem?" >&2 + exit 1 +fi + +gpu_count="$(nvidia-smi -L | wc -l)" +if [[ "$gpu_count" -ne 8 ]]; then + echo "ERROR: expected 8 GPUs (p5.48xlarge), found $gpu_count. Aborting." >&2 + exit 1 +fi +echo "OK: nvidia-smi reports $gpu_count GPUs." + +# --------------------------------------------------------------------------- +# Step 3: build nccl-tests if it isn't already built. +# +# Idempotent: only clones if NCCL_TESTS_DIR doesn't exist yet, and only +# (re)builds if the all_reduce_perf binary is missing -- a rerun of this +# script after a successful first run is a fast no-op here. +# --------------------------------------------------------------------------- +if [[ -x "$NCCL_TESTS_DIR/build/all_reduce_perf" ]]; then + echo "OK: nccl-tests already built at $NCCL_TESTS_DIR." +else + if [[ ! -d "$NCCL_TESTS_DIR" ]]; then + echo "Cloning nccl-tests into $NCCL_TESTS_DIR ..." + git clone https://github.com/NVIDIA/nccl-tests "$NCCL_TESTS_DIR" + fi + echo "Building nccl-tests (MPI=0, CUDA_HOME=$CUDA_HOME) ..." + make -j -C "$NCCL_TESTS_DIR" MPI=0 CUDA_HOME="$CUDA_HOME" +fi + +# --------------------------------------------------------------------------- +# Step 4: build torus_bench if it isn't already built. +# +# TORUS_BENCH_DIR is scp'd onto the instance by the orchestrator (a sibling +# work package owns the torus_bench source); if it hasn't landed yet, warn +# and continue rather than failing -- the FC leg (nccl-tests) can still run +# without it, and setup_node.sh may legitimately run before the orchestrator +# has finished copying torus_bench over. +# --------------------------------------------------------------------------- +torus_bench_bin="$TORUS_BENCH_DIR/torus_bench" +if [[ -x "$torus_bench_bin" ]]; then + echo "OK: torus_bench already built at $torus_bench_bin." +elif [[ -d "$TORUS_BENCH_DIR" ]]; then + echo "Building torus_bench ..." + make -C "$TORUS_BENCH_DIR" torus_bench +else + echo "WARNING: $TORUS_BENCH_DIR not found (expected to be scp'd there by the orchestrator)." >&2 + echo " Skipping torus_bench build; the FC leg can still run without it." >&2 +fi + +# --------------------------------------------------------------------------- +# Step 5: print a versions summary for the run's metadata/provenance. +# Every lookup here is best-effort (guarded so a missing tool doesn't abort +# the script under `set -e`), since this is diagnostic output, not a hard +# requirement. +# --------------------------------------------------------------------------- +echo "=== Versions ===" + +if command -v nvcc >/dev/null 2>&1; then + nvcc --version | tail -n 1 +else + echo "nvcc: not found on PATH" +fi + +nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | sed 's/^/driver_version: /' + +# NCCL version discovery: prefer asking a Python-visible torch build (most +# accurate for the environment that will actually run collectives), and +# fall back to scanning the linker cache for libnccl if torch isn't +# importable. Both are best-effort. +if python3 -c "import torch; print(torch.cuda.nccl.version())" 2>/dev/null; then + : +elif ldconfig -p | grep -qi libnccl; then + echo "libnccl found via ldconfig:" + ldconfig -p | grep -i libnccl +else + echo "NCCL version: could not be determined (no torch, no libnccl in ldconfig cache)" +fi + +echo "setup_node.sh complete." diff --git a/notebooks/astrasim2_correlation/correlation/teardown.py b/notebooks/astrasim2_correlation/correlation/teardown.py new file mode 100644 index 00000000..e533125a --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/teardown.py @@ -0,0 +1,850 @@ +"""Tear down AWS resources created by ``provision.py``. + +This script is the "down" half of the correlation study's empirical leg +(see ``provision.py`` for the "up" half and ``README.md`` for the full +runbook). It is designed to be safe to run more than once, and to work +even when its own local state is missing or stale, because it discovers +targets two ways and reconciles them: + +1. Local state files under ``.state/*.json``, written by + ``provision.py``'s ``write_state`` -- the fast, detailed path, since a + state file already has the security group id and key pair name without + any extra API calls. +2. A live ``describe_instances`` search filtered on the ``Project`` tag + (and ``RunId`` tag, when ``--run-id`` is given) -- the authoritative + path, since it reflects what AWS actually has running right now even if + a state file was deleted, never written (a crash mid-provision), or the + run was started from a different machine/checkout. + +Every function below is written to be independently importable and +testable, matching ``provision.py``'s convention; see that module's +docstring for why boto3 is imported guarded rather than as a hard +dependency. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +from config import Config + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: # pragma: no cover - exercised only when boto3 truly absent + boto3 = None + + # See provision.py's identical placeholder for why this exists: keeps + # `except ClientError:` clauses valid Python even without boto3 + # installed, without ever actually being reachable (main() always + # calls _require_boto3() first). + class ClientError(Exception): # type: ignore[no-redef] + pass + + +def _require_boto3() -> None: + """Raise a clear, actionable error if boto3 is not installed. + + Raises + ------ + SystemExit + Always, if ``boto3`` failed to import. See ``provision._require_boto3`` + for the identical rationale; kept as a separate copy here (rather + than importing it from ``provision``) so this module has no + import-time dependency on ``provision.py`` at all. + """ + if boto3 is None: + raise SystemExit( + "boto3 is required for AWS teardown but is not installed in this " + "Python environment.\n" + "Install it with: pip install boto3" + ) + + +# Instance states worth discovering/tearing down. Deliberately excludes +# "shutting-down" and "terminated": those instances are already on their +# way out or gone and do not need (and, for "terminated", cannot receive) +# a terminate_instances call. +_ACTIVE_STATES = ("pending", "running", "stopping", "stopped") + +# EC2 does not release the ENI-to-security-group association the instant +# terminate_instances returns (or the instance_terminated waiter is +# satisfied); delete_security_group can fail with DependencyViolation for a +# short window afterward while that teardown finishes propagating. Retrying +# with a fixed backoff is expected to succeed within a few attempts rather +# than being a genuine, permanent conflict. +_SG_DELETE_MAX_RETRIES = 5 +_SG_DELETE_RETRY_SLEEP_S = 5.0 + + +def find_tagged_instances( + ec2_client, tag_project: str, run_id: Optional[str] = None +) -> List[dict]: + """Find EC2 instances tagged for this study, optionally scoped to one run. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + tag_project : str + Value the ``Project`` tag must match. + run_id : str or None, default None + If given, additionally require the ``RunId`` tag to match this + value. If ``None``, instances from every run under ``tag_project`` + are returned. + + Returns + ------- + list[dict] + Raw ``Instance`` dicts (the ``Reservations[].Instances[]`` shape + returned by ``describe_instances``), for instances currently in + one of :data:`_ACTIVE_STATES`. Empty list if none match. + + Notes + ----- + Paginates via ``NextToken`` manually (rather than + ``ec2_client.get_paginator(...)``) so this function works identically + against a plain client and a ``botocore.stub.Stubber``-wrapped one used + in tests, without needing the Stubber to understand paginator internals. + """ + filters = [ + {"Name": "tag:Project", "Values": [tag_project]}, + {"Name": "instance-state-name", "Values": list(_ACTIVE_STATES)}, + ] + if run_id: + filters.append({"Name": "tag:RunId", "Values": [run_id]}) + + instances: List[dict] = [] + kwargs: Dict[str, Any] = {"Filters": filters} + while True: + response = ec2_client.describe_instances(**kwargs) + for reservation in response.get("Reservations", []): + instances.extend(reservation.get("Instances", [])) + next_token = response.get("NextToken") + if not next_token: + break + kwargs["NextToken"] = next_token + return instances + + +def _delete_security_group_with_retry( + ec2_client, + sg_id: str, + max_retries: int = _SG_DELETE_MAX_RETRIES, + retry_sleep_s: float = _SG_DELETE_RETRY_SLEEP_S, +) -> None: + """Delete a security group, retrying on ``DependencyViolation``. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + sg_id : str + Security group id to delete. + max_retries : int, default 5 + Maximum number of ``delete_security_group`` attempts. + retry_sleep_s : float, default 5.0 + Seconds to sleep between retries. + + Raises + ------ + RuntimeError + If every attempt fails with ``DependencyViolation`` (the ENI + association never cleared in time). + botocore.exceptions.ClientError + For any ``ClientError`` code other than ``DependencyViolation`` or + ``InvalidGroup.NotFound``, propagated immediately without retry. + + Notes + ----- + See the module-level comment on :data:`_SG_DELETE_MAX_RETRIES` for why + ``DependencyViolation`` specifically is retried rather than treated as + fatal on the first failure. + """ + last_exc: Optional[ClientError] = None + for attempt in range(1, max_retries + 1): + try: + ec2_client.delete_security_group(GroupId=sg_id) + print(f"Deleted security group {sg_id}.") + return + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code == "InvalidGroup.NotFound": + print(f"Security group {sg_id} already gone.") + return + if code != "DependencyViolation": + raise + last_exc = exc + if attempt < max_retries: + print( + f"delete_security_group({sg_id}) hit DependencyViolation " + f"(attempt {attempt}/{max_retries}); retrying in " + f"{retry_sleep_s}s..." + ) + time.sleep(retry_sleep_s) + raise RuntimeError( + f"Failed to delete security group {sg_id} after {max_retries} attempts " + "due to a persistent DependencyViolation." + ) from last_exc + + +def teardown_run(ec2_client, state: dict, delete_key: bool) -> None: + """Terminate one run's instance and clean up its security group and key. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + state : dict + Per-run state, either loaded from a ``.state/*.json`` file (see + ``provision.write_state``) or synthesized from a live + ``describe_instances`` result (see ``_derive_state_from_instance``). + Recognized keys: ``instance_id``, ``sg_id``, ``key_name``, + ``key_path``. All are optional -- a missing key simply skips that + cleanup step, so a partially-populated state (e.g. derived from + AWS alone, with no known ``key_path``) still tears down whatever it + can. + delete_key : bool + If ``True``, also delete the AWS-side key pair (and the local PEM, + if ``state["key_path"]`` is known and exists on disk). + + Raises + ------ + RuntimeError + If security group deletion exhausts its retries (see + :func:`_delete_security_group_with_retry`). + botocore.exceptions.ClientError + For any AWS failure other than the specific "already gone" codes + this function is written to tolerate (``InvalidInstanceID.NotFound`` + for the instance, ``InvalidGroup.NotFound`` for the security + group), propagated unmodified. + botocore.exceptions.WaiterError + If the ``instance_terminated`` waiter times out or the instance + reaches an unexpected terminal state. + + Notes + ----- + Order matters: instance termination is started and waited on *before* + security group deletion is attempted, because the security group + cannot be deleted while an instance's network interface still + references it (see :data:`_SG_DELETE_MAX_RETRIES`'s docstring). + """ + instance_id = state.get("instance_id") + if instance_id: + try: + ec2_client.terminate_instances(InstanceIds=[instance_id]) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code != "InvalidInstanceID.NotFound": + raise + print( + f"Instance {instance_id} already gone " + "(InvalidInstanceID.NotFound); continuing teardown." + ) + else: + print( + f"Termination requested for {instance_id}; waiting for it to " + "fully terminate..." + ) + waiter = ec2_client.get_waiter("instance_terminated") + waiter.wait(InstanceIds=[instance_id]) + print(f"Instance {instance_id} terminated.") + + sg_id = state.get("sg_id") + if sg_id: + _delete_security_group_with_retry(ec2_client, sg_id) + + if delete_key: + key_name = state.get("key_name") + if key_name: + try: + ec2_client.delete_key_pair(KeyName=key_name) + print(f"Deleted AWS key pair {key_name}.") + except ClientError as exc: + # Best-effort: a stale/already-deleted key pair should not + # block the rest of teardown, but it is still reported, not + # silently dropped. + print(f"WARNING: failed to delete AWS key pair {key_name}: {exc}") + + key_path = state.get("key_path") + if key_path: + local_path = Path(key_path) + if local_path.exists(): + local_path.unlink() + print(f"Deleted local PEM {local_path}.") + elif key_name: + print( + f"No local PEM path known for key pair {key_name!r} (this run's " + "state was derived from AWS alone, not a local state file); " + "only the AWS-side key pair was deleted. If a PEM for it exists " + "on this or another machine, remove it manually." + ) + + +def _derive_state_from_instance(instance: dict) -> dict: + """Reconstruct a minimal teardown ``state`` dict from a live instance. + + Used when a tagged instance is discovered via :func:`find_tagged_instances` + but has no matching local ``.state/*.json`` file (deleted, never + written, or written on a different machine). EC2 instance descriptions + already carry everything ``teardown_run`` needs except the local PEM + path, which cannot be recovered this way. + + Parameters + ---------- + instance : dict + One ``Instance`` dict as returned by ``describe_instances``. + + Returns + ------- + dict + ``{"run_id", "instance_id", "sg_id", "key_name", "key_path"}``, + with ``key_path`` always ``None`` (see above) and ``run_id`` taken + from the instance's ``RunId`` tag, falling back to the instance id + itself if that tag is somehow missing. + """ + tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])} + security_groups = instance.get("SecurityGroups", []) + return { + "run_id": tags.get("RunId", instance["InstanceId"]), + "instance_id": instance["InstanceId"], + "sg_id": security_groups[0]["GroupId"] if security_groups else None, + "key_name": instance.get("KeyName"), + "key_path": None, + } + + +def _state_dir_files(state_dir: Path, run_id: Optional[str]) -> List[Path]: + """List local state files relevant to this teardown invocation. + + Parameters + ---------- + state_dir : pathlib.Path + Directory containing ``.json`` state files. + run_id : str or None + If given, look only for ``/.json``. If + ``None``, return every ``*.json`` file in ``state_dir``. + + Returns + ------- + list[pathlib.Path] + Matching, existing file paths, sorted for deterministic output. + Empty list if ``state_dir`` does not exist or nothing matches. + """ + if not state_dir.exists(): + return [] + if run_id: + candidate = state_dir / f"{run_id}.json" + return [candidate] if candidate.exists() else [] + return sorted(state_dir.glob("*.json")) + + +def _load_state_file(path: Path) -> dict: + """Load one state JSON file. + + Parameters + ---------- + path : pathlib.Path + Path to a ``.state/.json`` file. + + Returns + ------- + dict + The parsed JSON content. + + Raises + ------ + OSError + If ``path`` cannot be opened. + json.JSONDecodeError + If ``path`` does not contain valid JSON. + """ + with open(path, "r") as fh: + return json.load(fh) + + +def _prompt_yes_no(prompt: str) -> bool: + """Ask an interactive yes/no question, returning ``True`` only for "yes". + + Parameters + ---------- + prompt : str + Text to show before the input cursor. + + Returns + ------- + bool + ``True`` only if the user typed exactly ``"yes"`` + (case-insensitive); ``False`` otherwise, including on EOF. + """ + try: + answer = input(prompt) + except EOFError: + return False + return answer.strip().lower() == "yes" + + +def _verify(ec2_client, tag_project: str, run_id: Optional[str]) -> int: + """Audit for any still-running tagged instances, without tearing anything down. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof). + tag_project : str + Value the ``Project`` tag must match. + run_id : str or None + If given, scope the audit to just this run. + + Returns + ------- + int + ``0`` (and prints ``"no running instances"``) if nothing tagged + remains in :data:`_ACTIVE_STATES`; ``1`` (and prints a table) if + anything does. Intended for use as a CI/cron safety check after a + teardown, so a stuck resource is caught rather than silently + left running and accruing cost. + """ + instances = find_tagged_instances(ec2_client, tag_project, run_id) + if not instances: + print("no running instances") + return 0 + + print("Tagged instances still present:") + print(f"{'InstanceId':<21} {'State':<12} {'RunId'}") + for instance in instances: + tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])} + state_name = instance.get("State", {}).get("Name", "unknown") + print(f"{instance['InstanceId']:<21} {state_name:<12} {tags.get('RunId', '?')}") + return 1 + + +# --------------------------------------------------------------------------- +# Region resolution (Fix 4a) +# --------------------------------------------------------------------------- +# +# Design/WHY: before this fix, teardown.py always defaulted --region to +# "us-east-1" and used exactly that one region for every discovery/teardown +# call, regardless of what region a run's own state file recorded. A run +# provisioned in any other region (e.g. because capacity/quota forced a +# different --region at provision time) was therefore invisible to +# `teardown.py --all` and to a bare `teardown.py --run-id ` unless the +# operator remembered to pass --region explicitly every time -- silently +# leaving that run's instance running and billing. The fix: --region now +# defaults to None (so this module can tell "the user explicitly asked for +# us-east-1" apart from "the user said nothing"), and region resolution +# follows this priority, per run: +# 1. an explicit --region flag always wins (it is an explicit override of +# whatever a state file might say, e.g. for recovering a run whose +# state file was hand-edited or lost a region field); +# 2. otherwise, a matching local state file's own recorded "region" field +# is authoritative (this is what makes --all correctly span multiple +# regions in one invocation); +# 3. otherwise (no --region, no matching/region-bearing state file -- +# e.g. an instance discovered via live AWS tags alone, with no local +# state at all), fall back to Config's own default region +# ("us-east-1"), matching this module's pre-fix behavior for that one +# case. +# resolve_regions() is a pure function (no AWS calls, no I/O) precisely so +# this priority logic is unit-testable on its own, per the work-package +# spec's explicit ask. + + +def resolve_regions(args, states: Dict[str, dict]) -> Dict[str, List[str]]: + """Determine which AWS region(s) to operate in, and which run_ids live in each. + + See the "Region resolution" design comment immediately above this + function for the full priority rationale (explicit ``--region`` flag, + then a state file's own recorded region, then Config's default). + + Parameters + ---------- + args : argparse.Namespace or any object with ``.region`` and ``.run_id`` + Only ``args.region`` (str or None) and ``args.run_id`` (str or + None) are consulted; duck-typed so a test can pass a minimal stand-in + without building a full parsed CLI namespace. + states : dict[str, dict] + Every locally known state dict, keyed by run_id, as loaded from + ``.state/*.json`` files -- NOT pre-filtered to ``args.run_id``; this + function does that scoping itself. + + Returns + ------- + dict[str, list[str]] + Mapping of region -> list of run_ids (drawn from `states`) resolved + to that region. + + - If ``args.run_id`` is set: exactly one key (the resolved region + for that one run), mapping to ``[args.run_id]``. + - Otherwise (``--all`` or a bare ``--verify``): one key per distinct + region recorded across every entry in `states` (grouped), PLUS + the default/fallback region (``args.region`` if given, else + Config's default) as its own key even if no state file happens to + record it -- so a caller iterating this mapping's keys always + still searches that region too, for tag-only discovery of + instances with no matching local state file at all (e.g. a crash + before ``provision.write_state`` ever ran). + + Notes + ----- + Pure function: makes no AWS calls and performs no I/O, so it is + directly unit-testable without a Stubber or any boto3 client. + + Examples + -------- + >>> import argparse + >>> args = argparse.Namespace(region=None, run_id=None) + >>> resolve_regions(args, {"run-a": {"region": "us-west-2"}}) + {'us-east-1': [], 'us-west-2': ['run-a']} + """ + default_region = args.region or Config().region + + if args.run_id: + state = states.get(args.run_id) + region = args.region or (state.get("region") if state else None) or default_region + return {region: [args.run_id]} + + regions: Dict[str, List[str]] = {default_region: []} + for run_id, state in states.items(): + region = args.region or state.get("region") or default_region + regions.setdefault(region, []).append(run_id) + return regions + + +def _client_for_region(clients: Dict[str, Any], region: str): + """Return a cached boto3 ``ec2`` client for `region`, creating it on first use. + + Parameters + ---------- + clients : dict[str, botocore.client.BaseClient] + Mutable cache, keyed by region name; mutated in place on a cache + miss. Callers own the dict's lifetime (typically one per + :func:`main` invocation). + region : str + AWS region name to build (or reuse) a client for. + + Returns + ------- + botocore.client.BaseClient + A boto3 ``ec2`` client bound to `region`, reused across calls that + pass the same `region` and the same `clients` dict. + + Notes + ----- + Design: ``--all`` may need to talk to several regions in one + invocation (one per distinct region recorded across this study's state + files -- see :func:`resolve_regions`). Building a client lazily, keyed + by region, keeps the number of client objects (and any connection-pool + overhead) proportional to the number of *distinct regions* actually + involved in one run, rather than the number of runs or the number of + times a region happens to be revisited. + """ + if region not in clients: + clients[region] = boto3.client("ec2", region_name=region) + return clients[region] + + +def _teardown_one_region( + ec2_client, + tag_project: str, + region: str, + run_id_filter: Optional[str], + file_states_by_run: Dict[str, dict], + file_paths_by_run: Dict[str, Path], + delete_key: bool, + yes: bool, +) -> int: + """Discover, stale-clean, confirm, and tear down every matching resource in one region. + + Factored out of :func:`main` so :func:`main` itself only needs to + resolve regions (:func:`resolve_regions`) and loop over them; this is + also what makes the stale-state cleanup path (Fix 4b) directly + unit-testable with a single ``botocore.stub.Stubber``-wrapped client, + without needing to drive the CLI/argparse layer or monkeypatch + ``boto3.client`` at all. + + Parameters + ---------- + ec2_client : botocore.client.BaseClient + A boto3 ``ec2`` client (or a stub thereof) already bound to + `region`. + tag_project : str + Value the ``Project`` tag must match (``Config().tag_project``). + region : str + The AWS region `ec2_client` is bound to; used only in printed + messages (this function performs no region validation of its own). + run_id_filter : str or None + If given, :func:`find_tagged_instances` is scoped to just this one + run (mirrors the original single-``--run-id`` behavior); ``None`` + discovers every tagged run in `region`. + file_states_by_run : dict[str, dict] + Locally known state dicts for the run_ids :func:`resolve_regions` + assigned to this region, keyed by run_id. + file_paths_by_run : dict[str, pathlib.Path] + The corresponding ``.state/.json`` paths, same keys as + `file_states_by_run`. + delete_key : bool + Forwarded to every :func:`teardown_run` call in this region. + yes : bool + If ``False``, prompts for interactive confirmation before tearing + down any LIVE target found in this region. Stale-state cleanup + (see Notes) is never gated on this prompt: it only reclaims + resources this function has already determined are orphaned (no + matching live instance), not anything newly discovered as + still-in-use. + + Returns + ------- + int + ``0`` if every teardown/cleanup in this region succeeded (including + "nothing to do here"); ``1`` if any individual teardown/cleanup + raised, or if the user declined confirmation for this region's live + targets. + + Notes + ----- + Order of operations, and why (Fix 4b): (1) live AWS discovery via + :func:`find_tagged_instances`, reconciled against `file_states_by_run` + exactly as the pre-fix module-level code did; (2) any state file in + `file_states_by_run` with NO matching live instance is "stale" -- + rather than just unlinking its state file (the pre-fix behavior), it is + now routed through :func:`teardown_run` FIRST. ``teardown_run``'s + terminate step already tolerates ``InvalidInstanceID.NotFound``, so a + truly-dead instance is a safe no-op there -- but its security group and + (optionally) key pair do NOT disappear just because the instance is + gone, e.g. via the on-instance dead-man timer terminating it out from + under a local state file that was never cleaned up. Routing through + ``teardown_run`` first closes that SG/key-pair leak; the state file is + removed only if that call did NOT raise, so a failed stale cleanup + leaves the state file in place for a future retry instead of silently + losing track of the leak. (3) only THEN are any remaining LIVE targets + confirmed and torn down, matching the pre-fix confirmation UX. + """ + file_states_by_instance: Dict[str, dict] = { + s["instance_id"]: s for s in file_states_by_run.values() if s.get("instance_id") + } + aws_instances = find_tagged_instances(ec2_client, tag_project, run_id_filter) + + targets: List[dict] = [] + for instance in aws_instances: + instance_id = instance["InstanceId"] + if instance_id in file_states_by_instance: + targets.append(file_states_by_instance[instance_id]) + else: + targets.append(_derive_state_from_instance(instance)) + + exit_code = 0 + + # --- Stale-state cleanup: see the "Order of operations" note above. --- + live_run_ids = {t.get("run_id") for t in targets if t.get("run_id")} + for run_id_key in set(file_states_by_run) - live_run_ids: + print( + f"State file for run {run_id_key!r} (region {region}) has no matching live AWS " + "instance; routing it through teardown to reclaim any leftover security group/key " + "pair before removing the state file." + ) + try: + teardown_run(ec2_client, file_states_by_run[run_id_key], delete_key=delete_key) + except Exception as exc: # noqa: BLE001 + # Design: deliberately broad, matching the identical rationale + # on the live-target teardown loop below -- one stale run's + # cleanup failure must not abort the rest of a --all batch. + print(f"ERROR cleaning up stale run {run_id_key!r}: {exc}", file=sys.stderr) + exit_code = 1 + continue + state_path = file_paths_by_run.get(run_id_key) + if state_path and state_path.exists(): + state_path.unlink() + print(f"Removed state file {state_path}.") + + if not targets: + if not file_states_by_run: + print(f"No matching resources found in {region}.") + return exit_code + + print(f"The following will be torn down in {region}:") + for t in targets: + print( + f" run_id={t.get('run_id', '?')} instance_id={t.get('instance_id', '?')} " + f"sg_id={t.get('sg_id', '?')}" + ) + + if not yes and not _prompt_yes_no("Proceed with teardown? [yes/N]: "): + print("Aborted by user.") + return 1 + + for t in targets: + try: + teardown_run(ec2_client, t, delete_key=delete_key) + except Exception as exc: # noqa: BLE001 + # Design: deliberately broad. One run's teardown failure should + # not abort the rest of a --all batch; report it and keep + # going, then reflect the failure in the process exit code + # rather than swallowing it silently. + print(f"ERROR tearing down run {t.get('run_id', '?')}: {exc}", file=sys.stderr) + exit_code = 1 + continue + + run_id_key = t.get("run_id") + state_path = file_paths_by_run.get(run_id_key) + if state_path and state_path.exists(): + state_path.unlink() + print(f"Removed state file {state_path}.") + + return exit_code + + +def main(argv: Optional[list] = None) -> int: + """Parse CLI args and tear down matching resources. + + Parameters + ---------- + argv : list[str] or None, default None + Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` + reads from ``sys.argv[1:]``. + + Returns + ------- + int + Process exit code. For ``--verify``: ``0`` if every audited region + is clean, ``1`` if tagged instances remain in any of them. + Otherwise: ``0`` on a fully successful teardown across every region + involved (or nothing to do anywhere), ``1`` if the user declined + confirmation in any region or if any individual run's + teardown/stale-cleanup raised. + + Notes + ----- + ``--tag-project`` is deliberately not a flag here (this CLI's flag set + is fixed by the work-package spec to + ``[--region] [--run-id | --all] [--verify] [--delete-key] [--yes]``): + the ``Project`` tag value used for discovery is always + ``Config().tag_project`` (i.e. ``Config``'s dataclass default, + ``"accelforge-correlation"``). A run provisioned with a *custom* + ``--tag-project`` cannot be found by this CLI and must instead be torn + down by calling :func:`find_tagged_instances`/:func:`teardown_run` + directly with that project value -- both are plain importable + functions for exactly this reason. + + ``--region`` now defaults to ``None`` (Fix 4a), NOT ``"us-east-1"``: see + the "Region resolution" design comment above :func:`resolve_regions` + for the full rationale and priority order. This function may therefore + talk to *more than one* region in a single invocation (e.g. ``--all`` + spanning every region this study's local state knows about, or + ``--verify`` auditing all of them) -- :func:`_client_for_region` keeps + one cached boto3 client per distinct region actually needed. + """ + parser = argparse.ArgumentParser( + prog="teardown.py", + description="Tear down accelforge correlation-study AWS resources.", + ) + parser.add_argument( + "--region", + type=str, + default=None, + help=( + "AWS region. If omitted, each targeted run's own state-file-recorded " + "region is used when known, else Config's default region (us-east-1); " + "an explicit --region always overrides both. See resolve_regions()." + ), + ) + target = parser.add_mutually_exclusive_group() + target.add_argument("--run-id", type=str, default=None, help="Tear down only this run.") + target.add_argument( + "--all", + action="store_true", + help=( + "Tear down every accelforge-correlation-tagged run this study's local " + "state knows about, across every region those runs were provisioned in." + ), + ) + parser.add_argument( + "--verify", + action="store_true", + help="Audit only: report any still-running tagged instances and exit non-zero if any remain.", + ) + parser.add_argument( + "--delete-key", + action="store_true", + help="Also delete the AWS key pair and local PEM.", + ) + parser.add_argument("--yes", action="store_true", help="Skip interactive confirmation.") + args = parser.parse_args(argv) + + if not args.verify and not args.run_id and not args.all: + parser.error("one of --run-id, --all, or --verify is required") + + _require_boto3() + + # Design: instantiate a plain Config() rather than duplicating its + # tag_project/state_dir default logic here. See the "Notes" above on + # why --tag-project is not a teardown.py flag. + defaults = Config() + tag_project = defaults.tag_project + state_dir = defaults.state_dir + + # Load EVERY locally known state file up front, regardless of scope -- + # resolve_regions() needs the full set to group by region (Fix 4a); + # scope filtering (--run-id vs --all/--verify) happens inside + # resolve_regions() and the per-region loop below, not here. + all_states_by_run: Dict[str, dict] = {} + all_paths_by_run: Dict[str, Path] = {} + for path in _state_dir_files(state_dir, None): + try: + loaded = _load_state_file(path) + except (OSError, json.JSONDecodeError) as exc: + print(f"WARNING: could not read state file {path}: {exc}; skipping.") + continue + run_id_key = loaded.get("run_id", path.stem) + all_states_by_run[run_id_key] = loaded + all_paths_by_run[run_id_key] = path + + region_map = resolve_regions(args, all_states_by_run) + clients: Dict[str, Any] = {} + + if args.verify: + # Fix 4c: audit every region resolve_regions() knows about (not + # only the --region flag/fallback), so a run provisioned in a + # different region than the operator happens to be thinking about + # is not silently skipped by an audit meant to catch exactly that. + exit_code = 0 + for region in sorted(region_map): + print(f"--- Verifying region {region} ---") + client = _client_for_region(clients, region) + if _verify(client, tag_project, args.run_id) != 0: + exit_code = 1 + return exit_code + + exit_code = 0 + for region in sorted(region_map): + ec2_client = _client_for_region(clients, region) + run_ids_here = region_map[region] + file_states_by_run = { + rid: all_states_by_run[rid] for rid in run_ids_here if rid in all_states_by_run + } + file_paths_by_run = { + rid: all_paths_by_run[rid] for rid in run_ids_here if rid in all_paths_by_run + } + region_exit_code = _teardown_one_region( + ec2_client, + tag_project, + region, + args.run_id, + file_states_by_run, + file_paths_by_run, + args.delete_key, + args.yes, + ) + if region_exit_code != 0: + exit_code = 1 + + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/tests/__init__.py b/notebooks/astrasim2_correlation/correlation/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py b/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py new file mode 100644 index 00000000..79c73050 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py @@ -0,0 +1,580 @@ +"""Tests for orchestrate.py. + +Like ``test_provision_teardown.py``, these tests never touch real AWS, +never touch the network, and never spawn a real ``ssh``/``scp`` process. +Where ``test_provision_teardown.py`` achieves that with +``botocore.stub.Stubber`` (because it exercises ``provision.py``'s and +``teardown.py``'s own boto3-request-building code directly), this module +takes a different, coarser-grained approach: ``orchestrate.py`` mostly +*sequences* those already-tested functions rather than building its own +boto3 requests, so the provisioning functions themselves +(``resolve_ami``, ``ensure_key_pair``, ``ensure_security_group``, +``launch_instance``, ``wait_for_instance``), ``teardown.teardown_run``, +``orchestrate.caller_ip``, ``orchestrate.boto3``, and +``orchestrate.subprocess.run`` are all monkeypatched with lightweight +fakes that record what they were called with. This exercises +``orchestrate.py``'s own sequencing/argv-building logic -- the part this +work package is actually responsible for -- without needing Stubber +responses for calls this module never makes itself. + +Import strategy +----------------- +Mirrors ``test_provision_teardown.py`` exactly: ``correlation/`` has no +``__init__.py`` (deliberately not a package), so it is inserted onto +``sys.path`` explicitly before importing ``config``/``orchestrate``, +rather than relying on pytest's own rootdir-insertion behavior. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +# boto3 is not (and must not become) an accelforge package dependency; skip +# this whole module rather than error if it is not installed in the +# environment running the tests. orchestrate.py's own import of boto3 is +# guarded the same way provision.py's/teardown.py's is, but the tests below +# exercise real code paths that assume boto3 (and its stub-friendly +# botocore internals) are present, matching test_provision_teardown.py's +# identical importorskip. +boto3 = pytest.importorskip("boto3") + +_CORRELATION_DIR = Path(__file__).resolve().parent.parent +if str(_CORRELATION_DIR) not in sys.path: + sys.path.insert(0, str(_CORRELATION_DIR)) + +import config # noqa: E402 +import orchestrate # noqa: E402 + +Config = config.Config + + +# --------------------------------------------------------------------------- +# legs_for +# --------------------------------------------------------------------------- + + +def test_legs_for_both_returns_fc_then_torus(): + """"both" expands to the FC leg followed by the torus leg, in that order.""" + assert orchestrate.legs_for("both") == ["fc", "torus"] + + +def test_legs_for_fc_returns_single_leg(): + assert orchestrate.legs_for("fc") == ["fc"] + + +def test_legs_for_torus_returns_single_leg(): + assert orchestrate.legs_for("torus") == ["torus"] + + +# --------------------------------------------------------------------------- +# build_ssh_cmd / build_scp_cmd +# --------------------------------------------------------------------------- + + +def test_build_ssh_cmd_exact_argv(tmp_path): + """build_ssh_cmd's argv matches the exact option set/order the spec requires.""" + key_path = tmp_path / "keys" / "run.pem" + known_hosts_path = tmp_path / ".state" / "known_hosts" + + cmd = orchestrate.build_ssh_cmd(key_path, "ubuntu", "203.0.113.9", "echo hi", known_hosts_path) + + assert cmd == [ + "ssh", + "-i", + str(key_path), + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"UserKnownHostsFile={known_hosts_path}", + "-o", + "ConnectTimeout=30", + "ubuntu@203.0.113.9", + "echo hi", + ] + + +def test_build_scp_cmd_non_recursive_push_argv(tmp_path): + """A multi-source, non-recursive push builds argv with sources then dest, no -r.""" + key_path = tmp_path / "keys" / "run.pem" + known_hosts_path = tmp_path / ".state" / "known_hosts" + sources = ["setup_node.sh", "run_profile.sh", "parse_nccl.py"] + dest = "ubuntu@203.0.113.9:~/" + + cmd = orchestrate.build_scp_cmd(key_path, sources, dest, known_hosts_path) + + assert cmd == [ + "scp", + "-i", + str(key_path), + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"UserKnownHostsFile={known_hosts_path}", + "-o", + "ConnectTimeout=30", + "setup_node.sh", + "run_profile.sh", + "parse_nccl.py", + "ubuntu@203.0.113.9:~/", + ] + assert "-r" not in cmd + + +def test_build_scp_cmd_recursive_flag_is_first_positional_after_scp(tmp_path): + """recursive=True inserts -r immediately after the program name, before -i.""" + key_path = tmp_path / "keys" / "run.pem" + known_hosts_path = tmp_path / ".state" / "known_hosts" + + cmd = orchestrate.build_scp_cmd( + key_path, + ["ubuntu@203.0.113.9:~/results_fc"], + "/local/data/run/fc", + known_hosts_path, + recursive=True, + ) + + assert cmd[0] == "scp" + assert cmd[1] == "-r" + assert cmd[2] == "-i" + assert cmd[-2] == "ubuntu@203.0.113.9:~/results_fc" + assert cmd[-1] == "/local/data/run/fc" + + +def test_build_scp_cmd_rejects_empty_sources(tmp_path): + with pytest.raises(ValueError): + orchestrate.build_scp_cmd(tmp_path / "k.pem", [], "dest", tmp_path / "known_hosts") + + +# --------------------------------------------------------------------------- +# End-to-end orchestration (main()), everything monkeypatched +# --------------------------------------------------------------------------- + + +class _FakeBoto3: + """Stand-in for the `boto3` module, only supplying `.client(...)`. + + Design: monkeypatched onto `orchestrate.boto3` specifically (not the + real, globally-shared `boto3` module) so this fake never leaks into + any other module's view of boto3. Returns a plain sentinel string + rather than a real client, since every function orchestrate.py passes + that "client" to (resolve_ami, ensure_key_pair, ...) is itself + monkeypatched below and never actually calls a botocore method on it. + """ + + @staticmethod + def client(service_name: str, region_name: str = None): + return f"fake-{service_name}-client[{region_name}]" + + +@pytest.fixture +def orchestrate_fakes(tmp_path, monkeypatch): + """Monkeypatch every AWS/network/subprocess seam orchestrate.py has. + + Returns + ------- + dict + ``{"calls": list of (name, args) tuples recording every fake + invocation in order, "run_cmds": list of argv lists recorded by + the fake subprocess.run, "teardown_calls": list of state dicts + teardown_run was called with}``. + + Notes + ----- + Also points ``cfg``'s ``--key-dir``/``--state-dir`` and + ``orchestrate._DATA_DIR`` at ``tmp_path`` subdirectories (see + ``orchestrate._DATA_DIR``'s module docstring comment for why that + constant exists) so a full ``orchestrate.main(...)`` run in these + tests writes real state/CSV files only under pytest's ephemeral + ``tmp_path``, never into the real repository tree. + """ + calls: List[tuple] = [] + run_cmds: List[List[str]] = [] + teardown_calls: List[Dict[str, Any]] = [] + + monkeypatch.setattr(orchestrate, "boto3", _FakeBoto3) + monkeypatch.setattr(orchestrate, "caller_ip", lambda: (_ for _ in ()).throw( + AssertionError("caller_ip() should never be called when --ssh-cidr is passed") + )) + + def fake_resolve_ami(ssm_client, parameter): + calls.append(("resolve_ami", parameter)) + return "ami-fake0123456789" + + def fake_ensure_key_pair(ec2_client, key_name, key_dir): + calls.append(("ensure_key_pair", key_name)) + key_dir.mkdir(parents=True, exist_ok=True) + return key_dir / f"{key_name}.pem" + + def fake_ensure_security_group(ec2_client, group_name, ssh_cidr, tag_project, run_id): + calls.append(("ensure_security_group", group_name, ssh_cidr)) + return "sg-fake0123456789" + + def fake_launch_instance(ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False): + calls.append(("launch_instance", dry_run)) + if dry_run: + return {"instance_id": None, "purchasing_used": "ondemand"} + return {"instance_id": "i-fake0123456789", "purchasing_used": "spot"} + + def fake_wait_for_instance(ec2_client, instance_id): + # Ordering assertion baked into the fake itself (rather than only + # checked after main() returns): the state file must already + # exist, with this instance_id recorded, by the time + # wait_for_instance is called -- this is the exact "state written + # before the SSH wait" ordering the work-package spec requires. + # See orchestrate.main's docstring "Design/WHY" note. + state_path = Path(_last_state_dir[0]) / f"{_last_run_id[0]}.json" + assert state_path.exists(), "state file must be written before wait_for_instance is called" + written = json.loads(state_path.read_text()) + assert written["instance_id"] == instance_id + assert written["public_ip"] is None + calls.append(("wait_for_instance", instance_id)) + return "203.0.113.9" + + def fake_teardown_run(ec2_client, state, delete_key): + teardown_calls.append(dict(state)) + calls.append(("teardown_run", state.get("instance_id"), delete_key)) + + def fake_run(cmd, check=True, **kwargs): + run_cmds.append(list(cmd)) + return subprocess.CompletedProcess(cmd, 0) + + # _last_state_dir / _last_run_id let fake_wait_for_instance locate the + # state file without needing main()'s local `cfg` in scope; populated + # by the test itself right before calling orchestrate.main(...). + _last_state_dir: List[Path] = [None] + _last_run_id: List[str] = [None] + + monkeypatch.setattr(orchestrate, "resolve_ami", fake_resolve_ami) + monkeypatch.setattr(orchestrate, "ensure_key_pair", fake_ensure_key_pair) + monkeypatch.setattr(orchestrate, "ensure_security_group", fake_ensure_security_group) + monkeypatch.setattr(orchestrate, "launch_instance", fake_launch_instance) + monkeypatch.setattr(orchestrate, "wait_for_instance", fake_wait_for_instance) + monkeypatch.setattr(orchestrate, "teardown_run", fake_teardown_run) + monkeypatch.setattr(orchestrate.subprocess, "run", fake_run) + + data_dir = tmp_path / "data" + monkeypatch.setattr(orchestrate, "_DATA_DIR", data_dir) + + return { + "calls": calls, + "run_cmds": run_cmds, + "teardown_calls": teardown_calls, + "state_dir_holder": _last_state_dir, + "run_id_holder": _last_run_id, + "data_dir": data_dir, + } + + +def _base_argv(tmp_path, run_id: str, extra: List[str] = None) -> List[str]: + """Shared CLI args for the end-to-end tests below. + + Parameters + ---------- + tmp_path : pathlib.Path + pytest's per-test temp directory; key-dir/state-dir are pointed + here so no test ever touches the real correlation/keys or + correlation/.state directories. + run_id : str + Deterministic run id so tests can locate the state file/data + directory by name instead of discovering a generated one. + extra : list[str] or None + Additional argv to append (e.g. ``["--keep-alive"]``). + + Returns + ------- + list[str] + argv suitable for ``orchestrate.main(...)``. Always includes + ``--ssh-cidr`` explicitly so ``caller_ip()`` (a real network call) + is never reached, and ``--yes`` so no interactive prompt blocks + the test. + """ + argv = [ + "--yes", + "--topology", + "both", + "--run-id", + run_id, + "--key-dir", + str(tmp_path / "keys"), + "--state-dir", + str(tmp_path / "state"), + "--ssh-cidr", + "203.0.113.5/32", + "--collectives", + "all_reduce,alltoall", + "--min-mib", + "1", + "--max-mib", + "2", + "--torus-dims", + "2x2x2", + ] + if extra: + argv += extra + return argv + + +def test_main_end_to_end_happy_path(tmp_path, orchestrate_fakes, monkeypatch): + """Full main() run: state ordering, per-leg profiling, fetch, and teardown. + + Asserts, per the work-package spec's test #3: + - the state file is written after launch (before wait_for_instance is + called -- enforced inside the fake_wait_for_instance itself) and + contains instance_id, + - the setup ssh command is executed with DEADMAN_MINUTES set, + - exactly one run_profile.sh invocation per leg, with the correct + dims ("8" then "2x2x2") and byte bounds (1 MiB / 2 MiB here), + - one scp fetch per leg, + - teardown_run is called exactly once, at the end, + - the state file is removed afterward. + """ + run_id = "test-run-e2e" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + + exit_code = orchestrate.main(_base_argv(tmp_path, run_id)) + + assert exit_code == 0 + + # --- state file lifecycle ------------------------------------------------- + state_path = state_dir / f"{run_id}.json" + assert not state_path.exists(), "state file must be removed after a successful teardown" + + # --- setup command ---------------------------------------------------------- + run_cmds = orchestrate_fakes["run_cmds"] + setup_cmds = [c for c in run_cmds if "setup_node.sh" in c[-1]] + assert len(setup_cmds) == 1 + assert "DEADMAN_MINUTES=120" in setup_cmds[0][-1] + assert setup_cmds[0][0] == "ssh" + + # --- one run_profile.sh invocation per leg, correct dims/bytes ----------- + profile_cmds = [c for c in run_cmds if "run_profile.sh" in c[-1]] + assert len(profile_cmds) == 2 + fc_cmd, torus_cmd = profile_cmds[0][-1], profile_cmds[1][-1] + assert "results_fc fc 1048576 2097152 8 all_reduce alltoall" in fc_cmd + assert "results_torus torus 1048576 2097152 2x2x2 all_reduce alltoall" in torus_cmd + + # --- fetch per leg ------------------------------------------------------ + # Recursive scp fetch argv shape (per build_scp_cmd): [..., source, dest], + # so the source (a "user@ip:~/results_" string) is always the + # second-to-last element. + scp_fetch_cmds = [c for c in run_cmds if c[0] == "scp" and "-r" in c and "results_" in c[-2]] + fetch_sources = {c[-2] for c in scp_fetch_cmds} + assert any("results_fc" in s for s in fetch_sources) + assert any("results_torus" in s for s in fetch_sources) + + # --- push commands happened before setup/profiling ----------------------- + # A push argv is a flat list of local file-path elements followed by a + # remote dest string, so membership needs a substring scan across + # elements rather than an exact-element containment check (the pushed + # sources are full absolute paths, not the bare "setup_node.sh"). + def _any_elem_contains(cmd: List[str], needle: str) -> bool: + return any(needle in elem for elem in cmd) + + push_cmds = [c for c in run_cmds if c[0] == "scp" and _any_elem_contains(c, "setup_node.sh")] + assert len(push_cmds) == 1 + torus_push_cmds = [c for c in run_cmds if c[0] == "scp" and "-r" in c and "torus_bench" in c[-1]] + assert len(torus_push_cmds) == 1 + + # --- ordering: push -> setup -> (profile -> fetch) x legs ---------------- + def _first_index(predicate): + return next(i for i, c in enumerate(run_cmds) if predicate(c)) + + push_idx = _first_index(lambda c: c[0] == "scp" and _any_elem_contains(c, "setup_node.sh")) + setup_idx = _first_index(lambda c: c[0] == "ssh" and "setup_node.sh" in c[-1]) + fc_profile_idx = _first_index(lambda c: c[0] == "ssh" and "run_profile.sh" in c[-1] and " fc " in c[-1]) + assert push_idx < setup_idx < fc_profile_idx + + # --- teardown called exactly once, at the very end ----------------------- + assert len(orchestrate_fakes["teardown_calls"]) == 1 + assert orchestrate_fakes["teardown_calls"][0]["instance_id"] == "i-fake0123456789" + call_names = [c[0] for c in orchestrate_fakes["calls"]] + assert call_names[-1] == "teardown_run" + + +# --------------------------------------------------------------------------- +# Fix 1 (BLOCKER): _run_leg must create the fetch's PARENT directory (not the +# leaf) before scp runs, and must refuse to fetch into an already-existing +# leaf directory. +# --------------------------------------------------------------------------- + + +def test_main_creates_run_data_dir_before_fetch_scp(tmp_path, orchestrate_fakes, monkeypatch): + """run_data_dir (the fetch's parent) exists by the time each leg's fetch scp runs. + + Regression test for Fix 1: without ``run_data_dir.mkdir(...)`` before the + fetch, `scp -r user@host:~/results_fc /fc` would fail + outright the first time a run_id is used, since its parent directory + would not exist yet. This test captures os-path existence from *inside* + the fake ``subprocess.run`` at the exact moment a fetch command is + recorded, per the work-package spec's test #1. + """ + run_id = "test-run-mkdir" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + data_dir = orchestrate_fakes["data_dir"] + + run_data_dir_existed_at_fetch: List[bool] = [] + + def fake_run(cmd, check=True, **kwargs): + orchestrate_fakes["run_cmds"].append(list(cmd)) + # A fetch is a recursive scp whose source (second-to-last argv + # element, per build_scp_cmd's argv shape) names a remote + # results_ path -- distinguishes it from the (also recursive) + # torus_bench/ push, whose source is a local path instead. + if cmd[0] == "scp" and "-r" in cmd and "results_" in cmd[-2]: + run_data_dir_existed_at_fetch.append((data_dir / run_id).exists()) + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(orchestrate.subprocess, "run", fake_run) + + exit_code = orchestrate.main(_base_argv(tmp_path, run_id)) + + assert exit_code == 0 + # One fetch per leg (fc, torus); run_data_dir must already exist at both. + assert len(run_data_dir_existed_at_fetch) == 2 + assert all(run_data_dir_existed_at_fetch) + + +def test_run_leg_raises_if_leaf_dir_already_exists(tmp_path, monkeypatch): + """_run_leg refuses to re-fetch into an already-existing run_data_dir/leg. + + Direct unit test of :func:`orchestrate._run_leg` (rather than a full + ``main()`` run) per the work-package spec's test #2, exercising the + RuntimeError in isolation. ``_run_streaming`` is monkeypatched to a + recording no-op so no real ssh/scp subprocess is ever attempted; the + remote profiling ssh command runs (it happens before the leaf-dir check + in ``_run_leg``'s body), but the fetch scp must never be reached. + """ + run_cmds: List[List[str]] = [] + monkeypatch.setattr(orchestrate, "_run_streaming", lambda cmd: run_cmds.append(cmd)) + + cfg = Config(run_id="test-run-leaf-exists", torus_dims=(2, 2, 2), collectives="all_reduce") + run_data_dir = tmp_path / "data" / cfg.run_id + local_leg_dir = run_data_dir / "fc" + local_leg_dir.mkdir(parents=True) + + with pytest.raises(RuntimeError, match="already exists"): + orchestrate._run_leg( + cfg, + "fc", + tmp_path / "key.pem", + "203.0.113.9", + tmp_path / "known_hosts", + run_data_dir, + ) + + # Only the remote profiling ssh command (which precedes the leaf-dir + # check in _run_leg's body) ran; the fetch scp was never attempted. + assert len(run_cmds) == 1 + assert run_cmds[0][0] == "ssh" + + +def test_main_dry_run_stops_before_launch_and_ssh(tmp_path, orchestrate_fakes): + """--dry-run creates key+SG+a dry launch, but never waits/pushes/profiles/tears down.""" + run_id = "test-run-dry" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + + exit_code = orchestrate.main(_base_argv(tmp_path, run_id, extra=["--dry-run"])) + + assert exit_code == 0 + call_names = [c[0] for c in orchestrate_fakes["calls"]] + assert "ensure_key_pair" in call_names + assert "ensure_security_group" in call_names + assert "launch_instance" in call_names + assert "wait_for_instance" not in call_names + assert "teardown_run" not in call_names + assert orchestrate_fakes["run_cmds"] == [] + assert not (state_dir / f"{run_id}.json").exists() + + +def test_main_profiling_failure_still_tears_down_and_propagates(tmp_path, orchestrate_fakes, monkeypatch): + """A failed profiling subprocess still triggers teardown, and the error propagates.""" + run_id = "test-run-fail" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + + def failing_run(cmd, check=True, **kwargs): + orchestrate_fakes["run_cmds"].append(list(cmd)) + if "run_profile.sh" in cmd[-1]: + raise subprocess.CalledProcessError(returncode=1, cmd=cmd) + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(orchestrate.subprocess, "run", failing_run) + + with pytest.raises(subprocess.CalledProcessError): + orchestrate.main(_base_argv(tmp_path, run_id)) + + # Teardown must still have run despite the propagating exception. + assert len(orchestrate_fakes["teardown_calls"]) == 1 + assert orchestrate_fakes["teardown_calls"][0]["instance_id"] == "i-fake0123456789" + # And the state file was still cleaned up by that successful teardown. + assert not (state_dir / f"{run_id}.json").exists() + + +def test_main_teardown_failure_does_not_mask_original_exception(tmp_path, orchestrate_fakes, monkeypatch): + """If teardown_run ALSO raises, the original profiling exception still propagates.""" + run_id = "test-run-double-fail" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + + def failing_run(cmd, check=True, **kwargs): + orchestrate_fakes["run_cmds"].append(list(cmd)) + if "run_profile.sh" in cmd[-1]: + raise subprocess.CalledProcessError(returncode=1, cmd=cmd) + return subprocess.CompletedProcess(cmd, 0) + + def failing_teardown(ec2_client, state, delete_key): + orchestrate_fakes["teardown_calls"].append(dict(state)) + raise RuntimeError("simulated teardown failure (e.g. DependencyViolation exhausted)") + + monkeypatch.setattr(orchestrate.subprocess, "run", failing_run) + monkeypatch.setattr(orchestrate, "teardown_run", failing_teardown) + + # The ORIGINAL exception (CalledProcessError from profiling) must be + # what propagates, not the teardown's RuntimeError -- this is the + # "don't mask the original exception" behavior _teardown_and_cleanup + # documents. + with pytest.raises(subprocess.CalledProcessError): + orchestrate.main(_base_argv(tmp_path, run_id)) + + assert len(orchestrate_fakes["teardown_calls"]) == 1 + # Teardown failed, so the state file must NOT have been removed -- + # teardown.py --run-id needs it to find leftover resources later. + assert (state_dir / f"{run_id}.json").exists() + + +def test_main_keep_alive_skips_teardown_and_keeps_state_file(tmp_path, orchestrate_fakes, capsys): + """--keep-alive leaves the instance up: no teardown_run call, state file remains.""" + run_id = "test-run-keep-alive" + state_dir = tmp_path / "state" + orchestrate_fakes["state_dir_holder"][0] = state_dir + orchestrate_fakes["run_id_holder"][0] = run_id + + exit_code = orchestrate.main(_base_argv(tmp_path, run_id, extra=["--keep-alive"])) + + assert exit_code == 0 + assert orchestrate_fakes["teardown_calls"] == [] + call_names = [c[0] for c in orchestrate_fakes["calls"]] + assert "teardown_run" not in call_names + + state_path = state_dir / f"{run_id}.json" + assert state_path.exists() + written = json.loads(state_path.read_text()) + assert written["public_ip"] == "203.0.113.9" + + out = capsys.readouterr().out + assert "STILL BEING BILLED" in out + assert "ssh -i" in out diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py b/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py new file mode 100644 index 00000000..322e682b --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py @@ -0,0 +1,288 @@ +"""Tests for parse_nccl.py: the nccl-tests / torus_bench log parser. + +Fixtures under ``tests/fixtures/`` hold representative raw stdout captured +from the two profiling tools (see the ``correlation`` package's WP2 spec +for the exact log formats). These tests exercise the parsing functions +directly (:func:`parse_nccl.parse_nccl_tests`, +:func:`parse_nccl.parse_torus_bench`), the CSV writer +(:func:`parse_nccl.rows_to_csv`), and the CLI entry point end-to-end via +:mod:`subprocess`, using ``sys.executable`` so the tests run under whatever +interpreter is running pytest itself (matching how ``run_profile.sh`` +invokes this script with a plain ``python3``). +""" + +from __future__ import annotations + +import csv +import subprocess +import sys +from pathlib import Path + +import pytest + +from parse_nccl import ( + UNIFIED_CSV_FIELDNAMES, + parse_nccl_tests, + parse_torus_bench, + rows_to_csv, +) + +# Design: resolve fixture/script paths relative to this test file (not the +# CWD) so the suite passes regardless of where pytest is invoked from, per +# the spec's instruction to load fixtures with pathlib relative to the test +# file. +TESTS_DIR = Path(__file__).resolve().parent +FIXTURES_DIR = TESTS_DIR / "fixtures" +PARSE_NCCL_SCRIPT = TESTS_DIR.parent / "parse_nccl.py" + +FC_ALL_REDUCE_LOG = FIXTURES_DIR / "fc_all_reduce.log" +FC_ALLTOALL_LOG = FIXTURES_DIR / "fc_alltoall.log" +TORUS_ALL_REDUCE_LOG = FIXTURES_DIR / "torus_all_reduce.log" + + +def test_parse_nccl_tests_all_reduce(): + """fc_all_reduce.log parses to 2 rows with the expected first-row values. + + Exercises the common case: a well-formed all_reduce_perf log with a + standard 13-token data row (size, count, type, redop, root, then the + trailing-8 out-of-place/in-place metrics). + """ + text = FC_ALL_REDUCE_LOG.read_text(encoding="utf-8") + rows = parse_nccl_tests(text) + + assert len(rows) == 2 + first = rows[0] + assert first["size_bytes"] == 1048576 + assert first["count"] == 262144 + assert first["dtype"] == "float" + assert first["time_us"] == pytest.approx(98.52) + assert first["algbw_GBps"] == pytest.approx(10.64) + assert first["busbw_GBps"] == pytest.approx(18.62) + assert first["wrong"] == "0" + + +def test_parse_nccl_tests_alltoall_na_wrong(): + """fc_alltoall.log parses to 2 rows and exercises the 'N/A' #wrong path. + + alltoall_perf prints redop="none" and root="-1" instead of a real + reduction op/root -- this test confirms the trailing-8-token rule + parses those rows correctly regardless, and that an "N/A" out-of-place + #wrong value (validation disabled for that data point) is preserved + as the literal string "N/A" rather than raising or being coerced to a + number. + """ + text = FC_ALLTOALL_LOG.read_text(encoding="utf-8") + rows = parse_nccl_tests(text) + + assert len(rows) == 2 + assert rows[0]["wrong"] == "N/A" + assert rows[0]["size_bytes"] == 1048576 + assert rows[0]["time_us"] == pytest.approx(120.44) + # Second row is a normal (non-N/A) row, confirming N/A handling on row 0 + # didn't leak into subsequent parsing. + assert rows[1]["wrong"] == "0" + assert rows[1]["size_bytes"] == 2097152 + + +def test_parse_torus_bench_all_reduce(): + """torus_all_reduce.log parses to 3 rows with the expected first row. + + Exercises the comma-delimited TORUSBENCH sentinel format and the + check-field-to-wrong-string remapping (check "1" -> wrong "0"). + """ + text = TORUS_ALL_REDUCE_LOG.read_text(encoding="utf-8") + rows = parse_torus_bench(text) + + assert len(rows) == 3 + first = rows[0] + assert first["collective"] == "all_reduce" + assert first["dims"] == "2x2x2" + assert first["size_bytes"] == 1048576 + assert first["time_us"] == pytest.approx(142.11) + assert first["wrong"] == "0" + + +def test_rows_to_csv_round_trip(tmp_path): + """rows_to_csv() writes the exact unified header, in order, and empty + algbw/busbw cells for torus rows round-trip as empty strings. + + Builds one nccl-tests-shaped row and one torus_bench-shaped row (as + main() would produce them) and confirms the on-disk CSV, when read back + with csv.DictReader, has fieldnames matching UNIFIED_CSV_FIELDNAMES + exactly (order included) and that the torus row's bandwidth columns are + empty rather than "None" or some other stand-in. + """ + rows = [ + { + "source": "nccl-tests", + "topology": "fc", + "dims": "8", + "collective": "all_reduce", + "size_bytes": 1048576, + "count": 262144, + "dtype": "float", + "time_us": 98.52, + "algbw_GBps": 10.64, + "busbw_GBps": 18.62, + "wrong": "0", + }, + { + "source": "torus_bench", + "topology": "torus", + "dims": "2x2x2", + "collective": "all_reduce", + "size_bytes": 1048576, + "count": "", + "dtype": "", + "time_us": 142.11, + "algbw_GBps": "", + "busbw_GBps": "", + "wrong": "0", + }, + ] + out_path = tmp_path / "unified.csv" + + rows_to_csv(rows, out_path) + + with out_path.open(newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + assert reader.fieldnames == UNIFIED_CSV_FIELDNAMES + read_rows = list(reader) + + assert len(read_rows) == 2 + assert read_rows[1]["source"] == "torus_bench" + assert read_rows[1]["algbw_GBps"] == "" + assert read_rows[1]["busbw_GBps"] == "" + + +def _run_cli(*args: str) -> subprocess.CompletedProcess: + """Invoke parse_nccl.py's CLI as a subprocess. + + Parameters + ---------- + *args : str + Arguments to pass after the script path, e.g. the raw log path and + ``--source``/``--out``/etc. flags. + + Returns + ------- + subprocess.CompletedProcess + Result of the invocation, with stdout/stderr captured as text. + + Notes + ----- + Design: uses ``sys.executable`` (not a hard-coded ``python3``) so the + subprocess runs under the exact interpreter executing the test suite, + matching pytest's own environment rather than risking a PATH mismatch. + """ + return subprocess.run( + [sys.executable, str(PARSE_NCCL_SCRIPT), *args], + capture_output=True, + text=True, + ) + + +def test_cli_nccl_tests_end_to_end(tmp_path): + """CLI parses an nccl-tests log to a CSV with the expected row count.""" + out_path = tmp_path / "fc_all_reduce.csv" + result = _run_cli( + str(FC_ALL_REDUCE_LOG), + "--source", + "nccl-tests", + "--collective", + "all_reduce", + "--topology", + "fc", + "--out", + str(out_path), + ) + + assert result.returncode == 0, result.stderr + assert out_path.exists() + + with out_path.open(newline="", encoding="utf-8") as f: + read_rows = list(csv.DictReader(f)) + assert len(read_rows) == 2 + assert all(row["source"] == "nccl-tests" for row in read_rows) + assert all(row["dims"] == "8" for row in read_rows) + + +def test_cli_torus_bench_end_to_end(tmp_path): + """CLI parses a torus_bench log to a CSV with the expected row count.""" + out_path = tmp_path / "torus_all_reduce.csv" + result = _run_cli( + str(TORUS_ALL_REDUCE_LOG), + "--source", + "torus_bench", + "--collective", + "all_reduce", + "--topology", + "torus", + "--dims", + "2x2x2", + "--out", + str(out_path), + ) + + assert result.returncode == 0, result.stderr + assert out_path.exists() + + with out_path.open(newline="", encoding="utf-8") as f: + read_rows = list(csv.DictReader(f)) + assert len(read_rows) == 3 + assert all(row["source"] == "torus_bench" for row in read_rows) + assert all(row["algbw_GBps"] == "" for row in read_rows) + + +def test_cli_torus_bench_collective_mismatch_errors(tmp_path): + """CLI exits non-zero when --collective disagrees with the log content. + + Bonus coverage beyond the spec's 6 mandated cases: confirms the + validate-CLI-against-sentinel behavior documented in parse_nccl.main() + actually triggers a hard failure rather than silently mislabeling data, + since a silent mismatch here would corrupt the correlation notebook's + inputs without any visible signal. + """ + out_path = tmp_path / "should_not_be_created.csv" + result = _run_cli( + str(TORUS_ALL_REDUCE_LOG), + "--source", + "torus_bench", + "--collective", + "all_gather", # deliberately wrong; fixture is all_reduce + "--topology", + "torus", + "--out", + str(out_path), + ) + + assert result.returncode != 0 + assert not out_path.exists() + + +@pytest.mark.parametrize( + "log_text", + [ + "# just a comment\n\n# another comment\n", + "hello world\n", + "# nThread 1 nGpus 8\nhello world\n# trailing comment\n", + ], +) +def test_parse_nccl_tests_skips_malformed_lines(log_text): + """Comment-only, blank, and garbage lines are skipped without raising.""" + assert parse_nccl_tests(log_text) == [] + + +@pytest.mark.parametrize( + "log_text", + [ + "# torus_bench collective=all_reduce dims=2x2x2\n", + "hello world\n", + "# header\nhello world\nTORUSBENCH,incomplete,field\n", + ], +) +def test_parse_torus_bench_skips_malformed_lines(log_text): + """Comment-only, blank, garbage, and malformed sentinel lines are + skipped without raising (including a TORUSBENCH, line with too few + comma-separated fields).""" + assert parse_torus_bench(log_text) == [] diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py b/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py new file mode 100644 index 00000000..0e8caad2 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py @@ -0,0 +1,802 @@ +"""Tests for config.py, provision.py, and teardown.py. + +These tests never touch real AWS: every boto3 client call is intercepted +by ``botocore.stub.Stubber``, which asserts on the exact request +parameters and returns a canned response (or raises a canned +``ClientError``) instead of making a network call. This lets the tests +exercise real botocore request-building and error-handling code paths -- +including exact parameter shape assertions -- with zero network access and +zero AWS credentials, matching this work package's hard constraint that +nothing may call AWS. + +Import strategy +----------------- +``config.py``/``provision.py``/``teardown.py`` live directly under +``correlation/`` (one level above this ``tests/`` package), and +``correlation/`` is deliberately *not* a Python package (no +``__init__.py`` there -- see the work-package spec). ``provision.py`` and +``teardown.py`` both do a plain ``from config import Config``, which only +resolves if ``correlation/`` is on ``sys.path``. We therefore insert that +directory onto ``sys.path`` explicitly before importing any of the three +modules under test, rather than relying on pytest's own rootdir-insertion +behavior (which happens to also achieve this here, but only for the +specific "prepend" import mode and package-marker layout currently in +place -- an explicit insert is robust to either changing). +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import pytest + +# boto3 is not (and must not become) an accelforge package dependency; skip +# this whole module rather than error if it is not installed in the +# environment running the tests. +boto3 = pytest.importorskip("boto3") + +from botocore.exceptions import ClientError # noqa: E402 (after importorskip) +from botocore.stub import Stubber # noqa: E402 + +_CORRELATION_DIR = Path(__file__).resolve().parent.parent +if str(_CORRELATION_DIR) not in sys.path: + sys.path.insert(0, str(_CORRELATION_DIR)) + +import config # noqa: E402 +import provision # noqa: E402 +import teardown # noqa: E402 + +Config = config.Config + + +def _make_client(service_name: str): + """Build a boto3 client with dummy credentials, safe for Stubber use. + + Parameters + ---------- + service_name : str + E.g. ``"ec2"`` or ``"ssm"``. + + Returns + ------- + botocore.client.BaseClient + A real boto3 client object, never used to make a real network + call in these tests (every call site below is wrapped in a + ``Stubber`` context). + + Notes + ----- + Design: ``Stubber`` intercepts the HTTP send step, but botocore still + runs its normal request-signing step first, which raises + ``NoCredentialsError`` if no credentials are configured anywhere + (env vars, profile, instance metadata, ...). This test environment + intentionally has none, so every client is built with harmless dummy + static credentials purely to satisfy the signer -- these are never + sent anywhere, since Stubber never performs a real HTTP request. + """ + return boto3.client( + service_name, + region_name="us-east-1", + aws_access_key_id="testing", + aws_secret_access_key="testing", + ) + + +# --------------------------------------------------------------------------- +# resolve_ami +# --------------------------------------------------------------------------- + + +def test_resolve_ami_returns_stubbed_parameter_value(): + """resolve_ami extracts Parameter.Value from the SSM response.""" + ssm_client = _make_client("ssm") + stubber = Stubber(ssm_client) + parameter_name = ( + "/aws/service/deeplearning/ami/x86_64/" + "base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" + ) + stubber.add_response( + "get_parameter", + { + "Parameter": { + "Name": parameter_name, + "Value": "ami-0123456789abcdef0", + "Type": "String", + } + }, + {"Name": parameter_name}, + ) + + with stubber: + result = provision.resolve_ami(ssm_client, parameter_name) + + stubber.assert_no_pending_responses() + assert result == "ami-0123456789abcdef0" + + +# --------------------------------------------------------------------------- +# launch_instance +# --------------------------------------------------------------------------- + + +def test_launch_instance_spot_then_ondemand_falls_back_on_capacity_error(): + """A spot InsufficientInstanceCapacity error triggers an on-demand retry. + + Also asserts (per the work-package spec) that the first, failing + request was a spot request -- i.e. it carried InstanceMarketOptions -- + both directly (inspecting the kwargs dict) and indirectly (via + Stubber's expected_params, which would fail the test if + launch_instance's real spot request didn't match). + + Design (Fix 10): the FIRST call's expected_params is an independently + hardcoded literal dict, not derived from + ``provision._build_run_instances_kwargs`` -- deriving it from the same + helper the code under test calls would make this test circular (a bug + in that helper's request-shape would go undetected, since the test's + expectation and the code's actual request would drift together). The + literal below pins the exact request shape against the work-package + spec instead of against the code's own helper. The second (on-demand + fallback) call keeps using the helper-derived ``ondemand_kwargs`` for + convenience, since its shape isn't the focus of this particular test. + """ + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + cfg = Config(purchasing="spot-then-ondemand", run_id="test-run") + ami_id = "ami-0123456789abcdef0" + sg_id = "sg-0123456789abcdef0" + key_name = "accelforge-correlation-test-run" + + # Independently hardcoded, per the work-package spec's exact field list + # -- see the docstring above for why this must NOT be derived from + # provision._build_run_instances_kwargs. + spot_kwargs_literal = { + "ImageId": ami_id, + "InstanceType": "p5.48xlarge", + "KeyName": key_name, + "SecurityGroupIds": [sg_id], + "MinCount": 1, + "MaxCount": 1, + "InstanceInitiatedShutdownBehavior": "terminate", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "VolumeSize": 200, + "VolumeType": "gp3", + "DeleteOnTermination": True, + }, + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + {"Key": "Project", "Value": "accelforge-correlation"}, + {"Key": "RunId", "Value": "test-run"}, + {"Key": "Name", "Value": "accelforge-correlation-test-run"}, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + {"Key": "Project", "Value": "accelforge-correlation"}, + {"Key": "RunId", "Value": "test-run"}, + {"Key": "Name", "Value": "accelforge-correlation-test-run"}, + ], + }, + ], + "DryRun": False, + "InstanceMarketOptions": { + "MarketType": "spot", + "SpotOptions": { + "SpotInstanceType": "one-time", + "InstanceInterruptionBehavior": "terminate", + }, + }, + } + # Kept helper-derived for the fallback call, per the spec ("if convenient"). + ondemand_kwargs = provision._build_run_instances_kwargs( + cfg, ami_id, sg_id, key_name, dry_run=False, use_spot=False + ) + + # The spec's explicit ask: the FIRST request must have carried + # InstanceMarketOptions (spot), the fallback must not. + assert "InstanceMarketOptions" in spot_kwargs_literal + assert "InstanceMarketOptions" not in ondemand_kwargs + + stubber.add_client_error( + "run_instances", + service_error_code="InsufficientInstanceCapacity", + service_message="There is no Spot capacity available.", + expected_params=spot_kwargs_literal, + ) + stubber.add_response( + "run_instances", + {"Instances": [{"InstanceId": "i-0123456789abcdef0"}]}, + expected_params=ondemand_kwargs, + ) + + with stubber: + result = provision.launch_instance( + ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False + ) + + stubber.assert_no_pending_responses() + assert result == { + "instance_id": "i-0123456789abcdef0", + "purchasing_used": "ondemand", + } + + +def test_launch_instance_spot_only_does_not_fall_back(): + """purchasing="spot" propagates the ClientError instead of retrying on-demand.""" + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + cfg = Config(purchasing="spot", run_id="test-run") + ami_id = "ami-0123456789abcdef0" + sg_id = "sg-0123456789abcdef0" + key_name = "accelforge-correlation-test-run" + + spot_kwargs = provision._build_run_instances_kwargs( + cfg, ami_id, sg_id, key_name, dry_run=False, use_spot=True + ) + # Only one response is ever queued: if launch_instance incorrectly + # attempted a second (fallback) call, Stubber itself would raise for + # having no more queued responses, which is not a ClientError -- so + # pytest.raises(ClientError) below would fail loudly in that case too. + stubber.add_client_error( + "run_instances", + service_error_code="InsufficientInstanceCapacity", + service_message="There is no Spot capacity available.", + expected_params=spot_kwargs, + ) + + with stubber: + with pytest.raises(ClientError): + provision.launch_instance( + ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False + ) + + stubber.assert_no_pending_responses() + + +def test_launch_instance_dry_run_success_does_not_raise(capsys): + """A DryRunOperation error is treated as a successful authorization check.""" + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + cfg = Config(purchasing="ondemand", run_id="test-run") + ami_id = "ami-0123456789abcdef0" + sg_id = "sg-0123456789abcdef0" + key_name = "accelforge-correlation-test-run" + + ondemand_kwargs = provision._build_run_instances_kwargs( + cfg, ami_id, sg_id, key_name, dry_run=True, use_spot=False + ) + stubber.add_client_error( + "run_instances", + service_error_code="DryRunOperation", + service_message="Request would have succeeded, but DryRun flag is set.", + expected_params=ondemand_kwargs, + ) + + with stubber: + result = provision.launch_instance( + ec2_client, cfg, ami_id, sg_id, key_name, dry_run=True + ) + + stubber.assert_no_pending_responses() + assert result["purchasing_used"] == "ondemand" + # No instance was actually created during a dry run. + assert result["instance_id"] is None + assert "dry-run OK" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# ensure_security_group +# --------------------------------------------------------------------------- + + +def test_ensure_security_group_authorizes_requested_cidr(): + """ensure_security_group wires the given ssh_cidr into the ingress rule.""" + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + vpc_id = "vpc-0123456789abcdef0" + sg_id = "sg-0123456789abcdef0" + ssh_cidr = "203.0.113.5/32" + group_name = "accelforge-correlation-test-run-sg" + run_id = "test-run" + tag_project = "accelforge-correlation" + + stubber.add_response( + "describe_vpcs", + {"Vpcs": [{"VpcId": vpc_id, "IsDefault": True}]}, + {"Filters": [{"Name": "isDefault", "Values": ["true"]}]}, + ) + stubber.add_response( + "create_security_group", + {"GroupId": sg_id}, + { + "GroupName": group_name, + "Description": f"accelforge correlation study SG for run {run_id}", + "VpcId": vpc_id, + }, + ) + # The parameter that matters most here: expected_params pins the exact + # CidrIp ensure_security_group must send, so the test fails loudly if + # the wrong CIDR (or the wrong port) were ever authorized. + stubber.add_response( + "authorize_security_group_ingress", + {}, + { + "GroupId": sg_id, + "IpPermissions": [ + { + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "IpRanges": [ + { + "CidrIp": ssh_cidr, + "Description": "SSH access for accelforge correlation study", + } + ], + } + ], + }, + ) + stubber.add_response( + "create_tags", + {}, + { + "Resources": [sg_id], + "Tags": [ + {"Key": "Project", "Value": tag_project}, + {"Key": "RunId", "Value": run_id}, + {"Key": "Name", "Value": group_name}, + ], + }, + ) + + with stubber: + result = provision.ensure_security_group( + ec2_client, group_name, ssh_cidr, tag_project, run_id + ) + + stubber.assert_no_pending_responses() + assert result == sg_id + + +# --------------------------------------------------------------------------- +# teardown.find_tagged_instances +# --------------------------------------------------------------------------- + + +def test_find_tagged_instances_returns_ids_from_both_reservations(): + """find_tagged_instances flattens across multiple Reservations entries.""" + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + tag_project = "accelforge-correlation" + expected_filters = { + "Filters": [ + {"Name": "tag:Project", "Values": [tag_project]}, + { + "Name": "instance-state-name", + "Values": ["pending", "running", "stopping", "stopped"], + }, + ] + } + stubber.add_response( + "describe_instances", + { + "Reservations": [ + { + "Instances": [ + { + "InstanceId": "i-aaaa000000000001", + "State": {"Name": "running"}, + "Tags": [{"Key": "RunId", "Value": "run-a"}], + } + ] + }, + { + "Instances": [ + { + "InstanceId": "i-bbbb000000000002", + "State": {"Name": "pending"}, + "Tags": [{"Key": "RunId", "Value": "run-b"}], + } + ] + }, + ] + }, + expected_filters, + ) + + with stubber: + result = teardown.find_tagged_instances(ec2_client, tag_project) + + stubber.assert_no_pending_responses() + ids = {instance["InstanceId"] for instance in result} + assert ids == {"i-aaaa000000000001", "i-bbbb000000000002"} + + +# --------------------------------------------------------------------------- +# teardown security-group delete retry +# --------------------------------------------------------------------------- + + +def test_delete_security_group_retries_past_dependency_violation(monkeypatch): + """A DependencyViolation is retried (not fatal) and eventually succeeds.""" + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + sg_id = "sg-0123456789abcdef0" + + stubber.add_client_error( + "delete_security_group", + service_error_code="DependencyViolation", + service_message="resource sg-0123456789abcdef0 has a dependent object", + expected_params={"GroupId": sg_id}, + ) + stubber.add_response("delete_security_group", {}, {"GroupId": sg_id}) + + sleep_calls = [] + # Patch time.sleep as seen through teardown's own `import time`, so the + # retry loop does not actually block the test suite for + # _SG_DELETE_RETRY_SLEEP_S seconds. + monkeypatch.setattr(teardown.time, "sleep", lambda seconds: sleep_calls.append(seconds)) + + with stubber: + teardown._delete_security_group_with_retry(ec2_client, sg_id) + + stubber.assert_no_pending_responses() + assert len(sleep_calls) == 1 + + +# --------------------------------------------------------------------------- +# Fix 3 (MAJOR): caller_ip() fails fast instead of fail-open +# --------------------------------------------------------------------------- + + +def test_caller_ip_raises_on_discovery_failure(monkeypatch): + """caller_ip() raises RuntimeError (mentioning --ssh-cidr) on any discovery failure. + + Regression test for Fix 3: the old behavior returned the sentinel + "0.0.0.0" on failure, which every caller turned into the CIDR + "0.0.0.0/32" -- unreachable by anyone, including the operator -- only + after real AWS resources already existed and were already billing. + Failing fast here means the error surfaces before any of that happens. + """ + + def fake_urlopen(*args, **kwargs): + raise provision.urllib.error.URLError("simulated DNS failure") + + monkeypatch.setattr(provision.urllib.request, "urlopen", fake_urlopen) + + with pytest.raises(RuntimeError, match="--ssh-cidr"): + provision.caller_ip() + + +# --------------------------------------------------------------------------- +# Fix 2 (MAJOR): provision.main() closes the orphan-instance window +# --------------------------------------------------------------------------- + + +def test_provision_main_writes_state_before_wait_and_reports_recovery_on_failure( + tmp_path, monkeypatch, capsys +): + """provision.main() writes state (public_ip=None) before wait_for_instance runs, + and on a wait_for_instance failure prints a loud recovery block (instance + id, STILL RUNNING AND BILLING, state file path, exact recovery command) + and re-raises rather than silently losing track of a running instance. + + Every AWS-touching seam provision.main() has (resolve_ami, + ensure_key_pair, ensure_security_group, launch_instance, + wait_for_instance, and boto3.client itself) is monkeypatched with a + lightweight fake, mirroring test_orchestrate.py's ``orchestrate_fakes`` + approach -- this exercises provision.main()'s own sequencing/state-file + logic (what Fix 2 changed) without touching real AWS or needing a + Stubber response sequence for calls this test never lets happen for + real. + """ + + class _FakeBoto3: + @staticmethod + def client(service_name, region_name=None): + return f"fake-{service_name}-client[{region_name}]" + + monkeypatch.setattr(provision, "boto3", _FakeBoto3) + monkeypatch.setattr(provision, "resolve_ami", lambda ssm, param: "ami-fake0123456789") + + def fake_ensure_key_pair(ec2, key_name, key_dir): + key_dir.mkdir(parents=True, exist_ok=True) + return key_dir / f"{key_name}.pem" + + monkeypatch.setattr(provision, "ensure_key_pair", fake_ensure_key_pair) + monkeypatch.setattr( + provision, + "ensure_security_group", + lambda ec2, name, cidr, tag_project, run_id: "sg-fake0123456789", + ) + monkeypatch.setattr( + provision, + "launch_instance", + lambda ec2, cfg, ami_id, sg_id, key_name, dry_run=False: { + "instance_id": "i-fake0123456789", + "purchasing_used": "spot", + }, + ) + + state_dir = tmp_path / "state" + run_id = "test-run-orphan-window" + + def failing_wait_for_instance(ec2, instance_id): + # By the time wait_for_instance is called, state must already be on + # disk with public_ip still the None placeholder -- exactly the + # ordering Fix 2 requires (state written BEFORE the SSH wait, not + # only after it succeeds). + state_path = state_dir / f"{run_id}.json" + assert state_path.exists(), "state file must exist before wait_for_instance is called" + written = json.loads(state_path.read_text()) + assert written["instance_id"] == instance_id + assert written["public_ip"] is None + raise TimeoutError("simulated SSH-reachability timeout") + + monkeypatch.setattr(provision, "wait_for_instance", failing_wait_for_instance) + + argv = [ + "--yes", + "--run-id", + run_id, + "--key-dir", + str(tmp_path / "keys"), + "--state-dir", + str(state_dir), + "--ssh-cidr", + "203.0.113.5/32", + ] + + with pytest.raises(TimeoutError): + provision.main(argv) + + # The state file must survive the failure -- teardown.py needs it to + # find and tear down the still-running instance later. + state_path = state_dir / f"{run_id}.json" + assert state_path.exists() + + err = capsys.readouterr().err + assert "i-fake0123456789" in err + assert "STILL RUNNING AND BILLING" in err + assert str(state_path) in err + assert f"python teardown.py --run-id {run_id} --region" in err + + +# --------------------------------------------------------------------------- +# Fix 4a (MAJOR): teardown.resolve_regions -- pure region-resolution logic +# --------------------------------------------------------------------------- +# +# These tests make no AWS calls and use no Stubber, per resolve_regions()'s +# own design (a pure function of `args` and a plain dict of loaded state +# files) -- `args` is a minimal argparse.Namespace stand-in, not a fully +# parsed CLI invocation, since resolve_regions() only ever consults +# `.region` and `.run_id`. + + +def test_resolve_regions_explicit_flag_wins_over_state_region(): + """An explicit --region always overrides a state file's own recorded region.""" + args = argparse.Namespace(region="us-west-2", run_id="run-a") + states = {"run-a": {"region": "eu-central-1"}} + + assert teardown.resolve_regions(args, states) == {"us-west-2": ["run-a"]} + + +def test_resolve_regions_uses_state_file_region_when_no_flag(): + """With no --region, a single --run-id's own recorded region is used.""" + args = argparse.Namespace(region=None, run_id="run-a") + states = {"run-a": {"region": "ap-southeast-2"}} + + assert teardown.resolve_regions(args, states) == {"ap-southeast-2": ["run-a"]} + + +def test_resolve_regions_falls_back_to_config_default_when_unknown(): + """With no --region and no matching state file, Config's default region is used.""" + args = argparse.Namespace(region=None, run_id="run-with-no-state-file") + states: dict = {} + + assert teardown.resolve_regions(args, states) == { + Config().region: ["run-with-no-state-file"] + } + + +def test_resolve_regions_all_groups_by_recorded_region_and_keeps_default(): + """--all (run_id=None) groups every known run by its own region, plus the default.""" + args = argparse.Namespace(region=None, run_id=None) + states = { + "run-a": {"region": "us-west-2"}, + "run-b": {"region": "us-west-2"}, + "run-c": {"region": "eu-central-1"}, + "run-d": {}, # no recorded region at all -> falls back to the default + } + + result = teardown.resolve_regions(args, states) + + assert set(result["us-west-2"]) == {"run-a", "run-b"} + assert result["eu-central-1"] == ["run-c"] + # The default/fallback region (Config's own default) is always present + # as a key, even though only run-d actually landed there via fallback, + # so a caller iterating this mapping's keys always still searches it + # for tag-only discovery of state-less instances. + assert "run-d" in result[Config().region] + + +def test_resolve_regions_all_explicit_flag_overrides_every_run(): + """An explicit --region with --all overrides every individual run's own region.""" + args = argparse.Namespace(region="us-east-2", run_id=None) + states = { + "run-a": {"region": "us-west-2"}, + "run-b": {"region": "eu-central-1"}, + } + + result = teardown.resolve_regions(args, states) + + assert result == {"us-east-2": ["run-a", "run-b"]} + + +# --------------------------------------------------------------------------- +# Fix 4b (MAJOR): a stale state file (no matching live instance) is routed +# through teardown_run -- SG/key-pair cleanup included -- BEFORE its local +# state file is removed, instead of just being unlinked. +# --------------------------------------------------------------------------- + + +def test_teardown_one_region_stale_state_calls_delete_security_group_before_removing_file( + tmp_path, +): + """A stale run's security group is deleted before its state file disappears. + + Regression test for Fix 4b: the pre-fix behavior just unlinked a stale + state file without ever touching AWS, leaking the security group (and, + with --delete-key, the key pair) of any run whose instance died out + from under it -- e.g. via the on-instance dead-man timer firing -- before + teardown.py was ever run. Stubber's queued ``delete_security_group`` + response is only consumed if ``_teardown_one_region``'s internal + ``teardown_run`` call actually invokes it (``stubber.assert_no_pending_ + responses()`` below fails the test otherwise); combined with the state + file only being removed from disk AFTER that call returns without + raising, these two assertions together establish the required "SG + deleted before state file disappears" ordering. + """ + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + run_id = "stale-run" + instance_id = "i-0123456789abcdef0" + sg_id = "sg-0123456789abcdef0" + tag_project = "accelforge-correlation" + + state = { + "run_id": run_id, + "region": "us-east-1", + "instance_id": instance_id, + "sg_id": sg_id, + "key_name": "accelforge-correlation-stale-run", + "key_path": None, + } + state_path = tmp_path / f"{run_id}.json" + state_path.write_text(json.dumps(state)) + + # Discovery: no live instance matches this run -- makes it "stale". + stubber.add_response( + "describe_instances", + {"Reservations": []}, + { + "Filters": [ + {"Name": "tag:Project", "Values": [tag_project]}, + { + "Name": "instance-state-name", + "Values": ["pending", "running", "stopping", "stopped"], + }, + ] + }, + ) + # teardown_run's terminate step: the instance is already gone (e.g. the + # dead-man timer fired) -- tolerated as InvalidInstanceID.NotFound, per + # teardown_run's own existing contract. + stubber.add_client_error( + "terminate_instances", + service_error_code="InvalidInstanceID.NotFound", + service_message=f"The instance ID '{instance_id}' does not exist", + expected_params={"InstanceIds": [instance_id]}, + ) + # The assertion this test exists for: delete_security_group MUST be + # called (Stubber raises on assert_no_pending_responses() otherwise). + stubber.add_response("delete_security_group", {}, {"GroupId": sg_id}) + + with stubber: + exit_code = teardown._teardown_one_region( + ec2_client, + tag_project, + "us-east-1", + None, + {run_id: state}, + {run_id: state_path}, + delete_key=False, + yes=True, + ) + + stubber.assert_no_pending_responses() + assert exit_code == 0 + # The state file is gone only now that teardown_run (including + # delete_security_group) has already completed successfully. + assert not state_path.exists() + + +def test_teardown_one_region_stale_state_keeps_file_if_teardown_run_raises(tmp_path, monkeypatch): + """If the stale-cleanup teardown_run call itself raises, the state file survives. + + Complements the happy-path stale-cleanup test above: a failed cleanup + must not lose track of the leak by removing the state file anyway. + """ + ec2_client = _make_client("ec2") + stubber = Stubber(ec2_client) + + run_id = "stale-run-cleanup-fails" + sg_id = "sg-0123456789abcdef0" + tag_project = "accelforge-correlation" + + # No instance_id: teardown_run skips straight to security-group + # deletion, which is the call we make fail here. + state = {"run_id": run_id, "region": "us-east-1", "sg_id": sg_id, "key_name": None} + state_path = tmp_path / f"{run_id}.json" + state_path.write_text(json.dumps(state)) + + # Avoid actually sleeping between retries (see + # test_delete_security_group_retries_past_dependency_violation's + # identical rationale for patching teardown's own `import time`). + monkeypatch.setattr(teardown.time, "sleep", lambda seconds: None) + + stubber.add_response( + "describe_instances", + {"Reservations": []}, + { + "Filters": [ + {"Name": "tag:Project", "Values": [tag_project]}, + { + "Name": "instance-state-name", + "Values": ["pending", "running", "stopping", "stopped"], + }, + ] + }, + ) + # A persistent DependencyViolation exhausts _delete_security_group_with_retry's + # retries and surfaces as a RuntimeError from teardown_run. + for _ in range(teardown._SG_DELETE_MAX_RETRIES): + stubber.add_client_error( + "delete_security_group", + service_error_code="DependencyViolation", + service_message="resource has a dependent object", + expected_params={"GroupId": sg_id}, + ) + + with stubber: + exit_code = teardown._teardown_one_region( + ec2_client, + tag_project, + "us-east-1", + None, + {run_id: state}, + {run_id: state_path}, + delete_key=False, + yes=True, + ) + + stubber.assert_no_pending_responses() + assert exit_code == 1 + assert state_path.exists() diff --git a/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile b/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile new file mode 100644 index 00000000..a53fd392 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile @@ -0,0 +1,27 @@ +CUDA_HOME ?= /usr/local/cuda +NCCL_HOME ?= +NVCC ?= $(CUDA_HOME)/bin/nvcc +GXX ?= g++ +ARCH ?= -arch=sm_90 + +.PHONY: clean + +NCCL_INC := $(if $(NCCL_HOME),-I$(NCCL_HOME)/include) +NCCL_LIB := $(if $(NCCL_HOME),-L$(NCCL_HOME)/lib) + +# Default target (first in this file): the GPU/NCCL executor. Requires nvcc plus a CUDA +# toolkit and NCCL headers/libs; NOT buildable in this development environment (no nvcc here +# -- see the work-package report). Provided so the GPU target compiles cleanly, by construction, +# on the target AWS p5.48xlarge instance, where CUDA_HOME/NCCL_HOME should be set as needed. +torus_bench: torus_bench.cu + $(NVCC) -O3 $(ARCH) $(NCCL_INC) $(NCCL_LIB) -o $@ $< -lnccl + +# Simulator target: forces g++ to treat the .cu file as plain C++ (-x c++), not CUDA source. +# This is the local, GPU-free test vehicle for the schedule-building, edge-assertion, and +# per-collective reduction/routing logic -- see the file header of torus_bench.cu for the full +# schedule-as-data rationale. This is the target exercised by the acceptance criteria. +torus_bench_sim: torus_bench.cu + $(GXX) -x c++ -std=c++17 -O2 -Wall -Wextra -DTORUS_SIM -o $@ $< + +clean: + rm -f torus_bench torus_bench_sim diff --git a/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu b/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu new file mode 100644 index 00000000..ed17c0b9 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu @@ -0,0 +1,1681 @@ +// torus_bench.cu +// +// Single-process multi-GPU NCCL benchmark that runs collective algorithms restricted to the +// EDGES OF A LOGICAL TORUS, on hardware that is physically fully-connected (one AWS +// p5.48xlarge, 8x H100 over NVSwitch). +// +// SCIENTIFIC PURPOSE (drives the whole design) +// --------------------------------------------- +// This benchmark exists to correlate an analytical torus-network model against real +// measurements. The measurement is only valid if EVERY inter-GPU transfer travels between +// logical torus NEIGHBORS -- that constraint IS the experiment. A single misrouted transfer +// (e.g. a "shortcut" the NVSwitch fabric would happily allow but the torus topology would not) +// silently invalidates the correlation. We therefore do not trust ourselves to hand-write +// per-collective CUDA/NCCL call sequences and eyeball their correctness; instead: +// +// SCHEDULE-AS-DATA: pure host code (no CUDA, no GPU) builds an explicit, fully materialized +// step-by-step transfer schedule (`Schedule` = vector, each Step a set of concurrent +// `Xfer`s followed by local reduce/copy `LocalOp`s). A single choke point -- the edge +// assertion inside build_schedule() -- inspects every Xfer the schedule will ever contain and +// aborts the program if any of them is not a torus-neighbor transfer. This assertion is the +// scientific guarantee of this file and must never be disabled, in either build. +// +// TWO EXECUTORS, ONE SCHEDULE: the identical Schedule produced by build_schedule() is handed +// to either of two interchangeable executors with matching function signatures: +// - a host-memory SIMULATOR (compiled here, in this environment, with no GPU/nvcc +// available -- this is the local test vehicle and is exercised by the acceptance +// criteria), and +// - an NCCL/CUDA executor (built on the target GPU instance; cannot be compiled or run in +// this environment since no nvcc/CUDA toolkit is installed here -- see the project report +// for confirmation of this constraint). +// Both executors are driven by the exact same schedule-building and CLI code; only the +// "how do I actually move these bytes" implementation differs, selected at compile time via +// the TORUS_SIM macro. Compiling with `g++ -x c++ -DTORUS_SIM` yields the simulator binary; +// compiling with `nvcc` (TORUS_SIM undefined) yields the GPU binary. All CUDA/NCCL-specific +// code is fenced with `#ifndef TORUS_SIM` so a plain C++ compiler never sees CUDA syntax. +// +// TOPOLOGY CONVENTION +// -------------------- +// dims = [d_0, ..., d_{K-1}] is a K-dimensional torus with product(dims) = N ranks. Rank <-> +// coordinate mapping is row-major with the LAST dimension fastest-varying (i.e. like a C array +// of shape `dims`). neighbor(r, dim, +-1) wraps around (mod dims[dim]). For an extent-2 +// dimension, +1 and -1 land on the SAME neighbor -- the code below computes this generically via +// modular arithmetic and never special-cases extent-2 dimensions. +// +// DOCUMENTATION CONVENTIONS USED IN THIS FILE +// --------------------------------------------- +// Each function has a comment block with: a one-line summary, Parameters, Returns, and (where +// relevant) Preconditions/Notes -- the C++ analogue of NumPy-style docstrings. Each per-collective +// schedule-builder additionally documents, phase by phase, the data-placement INVARIANT that +// phase establishes; this is the load-bearing correctness argument for that collective and is +// exactly what a reader needs to convince themselves the algorithm is right. +// +// DERIVATION NOTE ON reduce_scatter (see build_reduce_scatter() below for full detail): the +// planning spec for this work package proposed a tentative per-step slot formula for +// reduce_scatter and then explicitly flagged uncertainty about it ("hold on, mirror the +// all_gather invariant exactly"). That tentative formula, worked through by hand and confirmed +// with a throwaway Python simulation across dims=[8],[2,4],[4,2],[2,2,2], places each fully +// reduced slot one hop short of its destination rank. The corrected formula and direction +// (documented at build_reduce_scatter) were derived from a time-reversal argument against the +// (spec-verified-correct) all_gather formula and independently confirmed by brute-force +// simulation before being encoded here; the C++ simulator's --check flag re-verifies this at +// runtime for every size in the sweep. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ===================================================================================== +// Buffer identifiers and collective enum +// ===================================================================================== + +// Logical per-rank buffer roles used by every schedule builder and both executors. Both +// executors allocate storage per (rank, BufId) according to buffer_sizes() below. +enum BufId : int { + BUF_SEND = 0, + BUF_RECV = 1, + BUF_TMP = 2, + BUF_WORK_A = 3, + BUF_WORK_B = 4, + NUM_BUFS = 5 +}; + +enum class Collective { ALL_REDUCE, ALL_GATHER, REDUCE_SCATTER, ALLTOALL, BROADCAST, SENDRECV }; + +// Returns the canonical CLI/output-format name for a collective. +// +// Parameters +// ---------- +// c : Collective +// +// Returns +// ------- +// const char* -- a string literal, e.g. "all_reduce". Never null. +const char* collective_name(Collective c) { + switch (c) { + case Collective::ALL_REDUCE: return "all_reduce"; + case Collective::ALL_GATHER: return "all_gather"; + case Collective::REDUCE_SCATTER: return "reduce_scatter"; + case Collective::ALLTOALL: return "alltoall"; + case Collective::BROADCAST: return "broadcast"; + case Collective::SENDRECV: return "sendrecv"; + } + return "?"; +} + +// Parses a --collective CLI argument. +// +// Parameters +// ---------- +// s : const std::string& -- one of the six recognized collective names. +// out : Collective& -- set on success; left unmodified on failure. +// +// Returns +// ------- +// bool -- true if `s` was recognized. +bool parse_collective(const std::string& s, Collective& out) { + if (s == "all_reduce") { out = Collective::ALL_REDUCE; return true; } + if (s == "all_gather") { out = Collective::ALL_GATHER; return true; } + if (s == "reduce_scatter") { out = Collective::REDUCE_SCATTER; return true; } + if (s == "alltoall") { out = Collective::ALLTOALL; return true; } + if (s == "broadcast") { out = Collective::BROADCAST; return true; } + if (s == "sendrecv") { out = Collective::SENDRECV; return true; } + return false; +} + +// Returns a human-readable name for a BufId, used only in --check diagnostic output. +const char* buf_name(int b) { + switch (b) { + case BUF_SEND: return "BUF_SEND"; + case BUF_RECV: return "BUF_RECV"; + case BUF_TMP: return "BUF_TMP"; + case BUF_WORK_A: return "BUF_WORK_A"; + case BUF_WORK_B: return "BUF_WORK_B"; + } + return "?"; +} + +// ===================================================================================== +// Schedule IR (pure data -- no CUDA dependency anywhere in this section) +// ===================================================================================== + +// One point-to-point transfer between two DIFFERENT ranks. Every Xfer that will ever be +// constructed by any builder in this file must satisfy is_torus_neighbor(src, dst) -- see +// check_edge_or_abort() and the final validation pass in build_schedule(). src == dst is +// forbidden by construction: same-rank data movement must be expressed as a LocalOp instead. +struct Xfer { + int src, dst; + int src_buf, dst_buf; + size_t src_off, dst_off, bytes; +}; + +// A same-rank operation: either a byte-for-byte copy (add == false) or an elementwise +// float accumulate dst[i] += src[i] (add == true), applied over `bytes` bytes (i.e. +// bytes/sizeof(float) floats -- bytes must be a multiple of 4 whenever add == true). +struct LocalOp { + int rank; + int src_buf; + size_t src_off; + int dst_buf; + size_t dst_off; + size_t bytes; + bool add; +}; + +// One synchronization step of a Schedule: all `xfers` are considered to execute concurrently +// (reading pre-step state), and only once every Xfer in the step has landed do the `post` +// LocalOps run (also concurrently with each other, since by construction no two post ops of +// the same step touch overlapping (rank,buf,offset) ranges). +struct Step { + std::vector xfers; + std::vector post; +}; + +using Schedule = std::vector; + +// ===================================================================================== +// Torus topology helpers +// ===================================================================================== + +// Returns N = product(dims), the total rank count of the torus. +inline int num_ranks(const std::vector& dims) { + int n = 1; + for (int d : dims) n *= d; + return n; +} + +// Converts a rank id to its torus coordinates. +// +// Parameters +// ---------- +// r : int -- rank id, 0 <= r < product(dims). +// dims : const std::vector& -- torus extents, dims[K-1] is the fastest-varying axis. +// +// Returns +// ------- +// std::vector -- coordinates, one per dimension, coords[j] in [0, dims[j]). +inline std::vector coords_of(int r, const std::vector& dims) { + std::vector c(dims.size()); + for (int j = (int)dims.size() - 1; j >= 0; --j) { + c[j] = r % dims[j]; + r /= dims[j]; + } + return c; +} + +// Inverse of coords_of(): converts torus coordinates back to a rank id (row-major, last +// dimension fastest). +inline int rank_of(const std::vector& c, const std::vector& dims) { + int r = 0; + for (size_t j = 0; j < dims.size(); ++j) r = r * dims[j] + c[j]; + return r; +} + +// Returns the rank reached from `r` by moving one hop of `delta` (+1 or -1) along dimension +// `dim`, wrapping around (mod dims[dim]). For an extent-2 dimension, delta=+1 and delta=-1 +// necessarily return the same rank -- this falls out of the modular arithmetic below with no +// special-casing, matching the spec's requirement that extent-2 degeneracy not be hard-coded. +inline int neighbor(int r, int dim, int delta, const std::vector& dims) { + std::vector c = coords_of(r, dims); + int e = dims[dim]; + c[dim] = ((c[dim] + delta) % e + e) % e; + return rank_of(c, dims); +} + +// Determines whether two ranks are torus neighbors: their coordinates must differ in exactly +// one dimension, and in that dimension by +-1 modulo the dimension's extent. +// +// Notes +// ----- +// For an extent-2 dimension this is trivially satisfied by any pair that differs there (both +// possible non-zero differences, 1 and (extent-1)=1, coincide), which is the intended behavior. +inline bool is_torus_neighbor(int a, int b, const std::vector& dims) { + std::vector ca = coords_of(a, dims), cb = coords_of(b, dims); + int diff_dim = -1, diff_count = 0; + for (size_t j = 0; j < dims.size(); ++j) { + if (ca[j] != cb[j]) { diff_dim = (int)j; ++diff_count; } + } + if (diff_count != 1) return false; + int e = dims[diff_dim]; + int d = ((ca[diff_dim] - cb[diff_dim]) % e + e) % e; + return d == 1 || d == e - 1; +} + +// Aborts the program with a file:line diagnostic if (src,dst) is not a torus edge, or if +// src == dst. This is THE scientific guarantee of this benchmark (see file header) and must +// remain active in both builds. +// +// Design decision: we use an explicit check + std::abort() rather than assert() from +// , because assert() compiles to a no-op under -DNDEBUG and we do not control every +// build environment this file might eventually be compiled in (e.g. a release-mode CI flag). +// An explicit check is unconditionally active regardless of optimization/NDEBUG flags. +inline void check_edge_or_abort(int src, int dst, const std::vector& dims) { + if (src == dst) { + std::fprintf(stderr, + "%s:%d: EDGE ASSERTION FAILED: Xfer has src==dst (rank %d); same-rank " + "movement must be expressed as a LocalOp, not an Xfer\n", + __FILE__, __LINE__, src); + std::abort(); + } + if (!is_torus_neighbor(src, dst, dims)) { + std::fprintf(stderr, + "%s:%d: EDGE ASSERTION FAILED: rank %d -> rank %d is not a torus " + "neighbor for the given dims; this transfer would not exist on the " + "logical torus and must not be scheduled\n", + __FILE__, __LINE__, src, dst); + std::abort(); + } +} + +// Enumerates every combination of coordinate values across dimensions [0, d), invoking `cb` +// once per combination with a coordinate vector `v` whose entries at indices >= d are left +// exactly as passed in (typically already pinned to a specific rank's own coordinates). +// +// Parameters +// ---------- +// d : int -- number of leading dimensions (0..d-1) to enumerate freely; if d == 0, `cb` is +// invoked exactly once, with `v` unchanged (the "no free dimensions" case). +// dims : const std::vector& -- torus extents. +// v : std::vector -- base coordinate vector (taken by value since we mutate indices < d +// during enumeration; entries at indices >= d are the caller's fixed values). +// cb : const std::function&)>& -- invoked once per combination. +// +// Notes +// ----- +// Enumeration order (dimension 0 varies slowest, in this implementation) is an arbitrary +// choice: every caller in this file treats each combination as an independent, order-agnostic +// unit of work (one Xfer/LocalOp pair per combination), so the traversal order used to reach +// the same combination SET has no effect on correctness. +inline void for_each_free_combo(int d, const std::vector& dims, std::vector v, + const std::function&)>& cb) { + std::function rec = [&](int j) { + if (j == d) { cb(v); return; } + for (int val = 0; val < dims[j]; ++val) { + v[j] = val; + rec(j + 1); + } + }; + rec(0); +} + +// ===================================================================================== +// Per-(collective,S) buffer sizing +// ===================================================================================== + +// Computes the per-rank byte size of each of the five logical buffers a given collective needs +// for total-collective-size S. Both executors call this identically to allocate storage (heap +// arrays for the simulator, cudaMalloc for the NCCL build) -- see file header for buffer roles. +// +// Parameters +// ---------- +// c : Collective +// dims : const std::vector& -- torus extents; N = product(dims). +// S : size_t -- total collective size in bytes for this sweep point. +// +// Returns +// ------- +// std::array -- indexed by BufId; unused buffers are size 0. +// +// Preconditions +// ------------- +// S must be divisible by N and by 4*N*N (whole-float shard/chunk boundaries); callers are +// expected to have already applied the sweep-level divisibility skip check (see main()) before +// calling this. +inline std::array buffer_sizes(Collective c, const std::vector& dims, + size_t S) { + int N = num_ranks(dims); + size_t m = S / (size_t)N; // per-rank shard, m = S/N, per the CLI spec's convention. + int K = (int)dims.size(); + std::array sz{}; + sz.fill(0); + switch (c) { + case Collective::SENDRECV: + sz[BUF_SEND] = m; + sz[BUF_RECV] = (size_t)K * m; // one m-byte region per dimension. + break; + case Collective::BROADCAST: + sz[BUF_SEND] = m; // only meaningful on root; allocated uniformly for simplicity. + sz[BUF_RECV] = m; + break; + case Collective::ALL_GATHER: + sz[BUF_SEND] = m; + sz[BUF_RECV] = (size_t)N * m; // N slots of m bytes each. + break; + case Collective::REDUCE_SCATTER: + sz[BUF_SEND] = (size_t)N * m; // N slots of m bytes each (destined for each rank). + sz[BUF_RECV] = m; + sz[BUF_WORK_A] = (size_t)N * m; // running accumulator, one slot per destination. + sz[BUF_TMP] = (size_t)N * m; // staging area for incoming adds. + break; + case Collective::ALL_REDUCE: + // Composed internally of reduce_scatter(shard=m/N) followed by all_gather + // (shard=m/N); see build_all_reduce() for the full derivation. Both sub-phases' + // internal buffer needs (N*(m/N) == m) collapse to a uniform m bytes here. + sz[BUF_SEND] = m; + sz[BUF_RECV] = m; + sz[BUF_WORK_A] = m; + sz[BUF_TMP] = m; + break; + case Collective::ALLTOALL: + sz[BUF_SEND] = m; // N chunks of c=m/N bytes each. + sz[BUF_RECV] = m; + sz[BUF_WORK_A] = 2 * m; // ping-pong in-flight staging; sized generously (2x) per + sz[BUF_WORK_B] = 2 * m; // spec, occupancy is asserted at schedule-build time. + break; + } + return sz; +} + +// ===================================================================================== +// Schedule builders (pure host code -- no CUDA dependency anywhere in this section) +// ===================================================================================== + +// Builds the schedule for `sendrecv`: one step per torus dimension, every rank exchanges its +// full BUF_SEND vector with its +1 neighbor in that dimension. +// +// Invariant per step d: after step d, every rank's BUF_RECV region [d*m, (d+1)*m) holds the +// data that neighbor(r, d, -1) sent -- i.e. its BUF_SEND contents -- since neighbor(r,d,-1)'s +// own send in this same step targets exactly rank r (send direction is always +1, and +// neighbor(neighbor(r,d,-1), d, +1) == r by construction of neighbor()). +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N is exchanged per dimension. +// +// Returns +// ------- +// Schedule -- K steps, N Xfers each, no LocalOps. +inline Schedule build_sendrecv(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + Schedule sched; + for (int d = 0; d < K; ++d) { + Step step; + for (int r = 0; r < N; ++r) { + int dst = neighbor(r, d, +1, dims); + check_edge_or_abort(r, dst, dims); + step.xfers.push_back({r, dst, BUF_SEND, BUF_RECV, 0, (size_t)d * m, m}); + } + sched.push_back(std::move(step)); + } + return sched; +} + +// Builds the schedule for `broadcast` from root rank 0. +// +// Algorithm: dimension-by-dimension forward-chain propagation. A host-side has_data[N] tracks, +// at schedule-BUILD time (not at run time), which ranks are already known to hold the +// broadcast data after each step; this bookkeeping exists purely to decide which Xfers to +// generate and is not part of the executed Schedule itself. +// +// Invariant: at the start of dimension d's phase, the set of ranks with has_data[r] == true is +// exactly the set of ranks that agree with rank 0 on every coordinate j >= d (this holds for +// d == 0 trivially: only rank 0 itself). Each of dimension d's (extent_d - 1) repeats extends +// every already-seeded "line" one hop further in the +1 direction; after all extent_d - 1 +// repeats, every rank agreeing with rank 0 on coordinates j > d has data (regardless of its +// coordinate d), establishing the invariant for phase d+1. After all K dimensions, every rank +// has data. +// +// Design decision: root's own copy is written by a dedicated step-0 LocalOp (BUF_SEND -> +// BUF_RECV) with NO concurrent Xfers in that same step. This lets every subsequent send (even +// root's very first "real" send) source uniformly from BUF_RECV: had root's first send shared +// step 0 with the LocalOp, it would need to special-case sourcing from BUF_SEND instead, since +// a Step's post-LocalOps run strictly after that step's Xfers land. +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N is the broadcast vector size. +// +// Returns +// ------- +// Schedule -- 1 (LocalOp-only) + sum_d(extent_d - 1) steps. +inline Schedule build_broadcast(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + Schedule sched; + { + Step step0; + step0.post.push_back({0, BUF_SEND, 0, BUF_RECV, 0, m, false}); + sched.push_back(std::move(step0)); + } + std::vector has_data(N, 0); + has_data[0] = 1; + for (int d = 0; d < K; ++d) { + int E = dims[d]; + for (int s = 0; s < E - 1; ++s) { + Step step; + std::vector new_has = has_data; // conditions evaluated against pre-step state. + for (int r = 0; r < N; ++r) { + if (!has_data[r]) continue; + int dst = neighbor(r, d, +1, dims); + if (has_data[dst]) continue; + check_edge_or_abort(r, dst, dims); + step.xfers.push_back({r, dst, BUF_RECV, BUF_RECV, 0, 0, m}); + new_has[dst] = 1; + } + has_data = new_has; + sched.push_back(std::move(step)); + } + } + return sched; +} + +// Builds the schedule for `all_gather`. +// +// Algorithm: standard ring all-gather, generalized to a mixed-radix torus by processing one +// dimension at a time (ascending order). Step 0 seeds each rank's own slot via a LocalOp. +// +// Invariant: at the START of dimension d's phase, rank r holds exactly the slots +// {u : u_j == r_j for all j >= d} (dimensions below d are already fully "free" -- rank r holds +// every value there -- while dimensions >= d are still pinned to r's own coordinate). This +// holds trivially at d == 0 (only slot r itself, u_j == r_j for ALL j) and, vacuously, means +// every rank holds every slot once d reaches K (all dimensions processed). +// +// Each phase d performs (extent_d - 1) ring-relay steps in the +1 direction: at step s, rank r +// forwards the slot batch it received on the PREVIOUS step (or, at s=1, the batch it started +// the phase with) to its +1 neighbor. This is the classic "forward only what you just received" +// ring relay, which avoids redundant retransmission and completes dimension d's expansion in +// exactly extent_d - 1 hops. +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N is each rank's own shard size. +// +// Returns +// ------- +// Schedule -- 1 + sum_d(extent_d - 1) steps. +inline Schedule build_all_gather(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + Schedule sched; + { + Step step0; + for (int r = 0; r < N; ++r) + step0.post.push_back({r, BUF_SEND, 0, BUF_RECV, (size_t)r * m, m, false}); + sched.push_back(std::move(step0)); + } + for (int d = 0; d < K; ++d) { + int E = dims[d]; + for (int s = 1; s <= E - 1; ++s) { + Step step; + for (int r = 0; r < N; ++r) { + std::vector base = coords_of(r, dims); + int dst = neighbor(r, d, +1, dims); + check_edge_or_abort(r, dst, dims); + base[d] = ((base[d] - (s - 1)) % E + E) % E; + for_each_free_combo(d, dims, base, [&](const std::vector& v) { + int vslot = rank_of(v, dims); + size_t off = (size_t)vslot * m; + step.xfers.push_back({r, dst, BUF_RECV, BUF_RECV, off, off, m}); + }); + } + sched.push_back(std::move(step)); + } + } + return sched; +} + +// Builds the schedule for `reduce_scatter`. +// +// DERIVATION NOTE (see also file header): the planning spec's own tentative formula for this +// collective was flagged mid-sentence as unreliable ("hold on, mirror the all_gather invariant +// exactly"). This implementation instead derives the per-step slot set and DIRECTION from a +// time-reversal argument against the (independently correct, spec-supplied) all_gather formula, +// and that derivation was confirmed by an exhaustive brute-force simulation (dims=[8],[2,4], +// [4,2],[2,2,2]) before being encoded here. Summary of the correction: the naive "mirror +// all_gather with the same (r_d - (s-1)) mod E slot formula and +1 direction" places each fully +// reduced slot ONE HOP SHORT of its destination rank (rank k's slot ends up fully summed at +// rank k-1, not rank k). The fix is both a different slot formula AND a different direction: +// sends go in the -1 direction, and at step t the sender's slot index is (r_d + t) mod E, not +// (r_d - (s-1)) mod E. +// +// Algorithm: work happens in BUF_WORK_A (an N-slot accumulator seeded from BUF_SEND at step 0). +// Dimensions are processed in DESCENDING order (K-1 down to 0), matching the spec. +// +// Invariant: at the END of dimension d's phase, rank r holds partial sums ONLY for slots +// {v : v_j == r_j for all j >= d} (dimensions below d are not yet reduced -- rank r's held +// slots still range over every value there -- while dimensions >= d have been fully reduced +// down to r's own coordinate). This is exactly the all_gather invariant with the phase-transition +// direction reversed: all_gather EXPANDS what a rank holds as d increases from 0 to K; +// reduce_scatter CONTRACTS what a rank holds as d decreases from K-1 to 0, so by symmetry it is +// stated as an END-of-phase (rather than start-of-phase) condition. +// +// Correctness of the per-step formula for a single dimension's ring (extent E, direction -1, +// slot index v_d = (r_d + t) mod E at step t = 1..E-1): consider dimension d as an isolated ring +// (the free dimensions jd ride along unchanged, in parallel, for +// every value). For a fixed target slot-index k (a value of v_d), the ring of E nodes must sum +// together each node's local contribution for chunk k, ending up entirely at node k. At reduce +// step t, the node currently entrusted with chunk k's running partial sum is node (k - t) mod E +// (t=1: node k-1 sends its OWN local value for chunk k to node k-2, which adds it in; t=2: node +// k-2, now holding a 2-term partial sum, forwards it to node k-3; ...; t=E-1: node (k - +// (E-1)) mod E == (k+1) mod E, holding an (E-1)-term partial sum -- every node except k itself +// -- forwards it to node k, which adds in its own remaining term to complete the sum of all E +// contributions). Restating "node (k - t) mod E sends chunk k to node (k - t - 1) mod E" from +// the SENDER's own coordinate r = (k - t) mod E gives: sender r sends chunk k = (r + t) mod E to +// receiver (r - 1) mod E == neighbor(r, d, -1). This is exactly the formula used below. +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N, BUF_SEND holds N slots of m bytes each. +// +// Returns +// ------- +// Schedule -- 1 (seed) + sum_d(extent_d - 1) (reduce) + 1 (final copy) steps. +inline Schedule build_reduce_scatter(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + Schedule sched; + { + Step step0; + for (int r = 0; r < N; ++r) + step0.post.push_back({r, BUF_SEND, 0, BUF_WORK_A, 0, (size_t)N * m, false}); + sched.push_back(std::move(step0)); + } + for (int d = K - 1; d >= 0; --d) { + int E = dims[d]; + for (int t = 1; t <= E - 1; ++t) { + Step step; + for (int r = 0; r < N; ++r) { + std::vector cr = coords_of(r, dims); + // Direction is -1 (see derivation above); this is the corrected direction, + // NOT the +1 direction all_gather uses. + int dst = neighbor(r, d, -1, dims); + check_edge_or_abort(r, dst, dims); + std::vector base = cr; + base[d] = ((cr[d] + t) % E + E) % E; + for_each_free_combo(d, dims, base, [&](const std::vector& v) { + int vslot = rank_of(v, dims); + size_t off = (size_t)vslot * m; + // Stage into the receiver's BUF_TMP at the same v*m offset (distinct + // offsets across combos within this step fall out automatically since each + // combo yields a distinct vslot), then a post LocalOp adds it into the + // receiver's running accumulator at that same slot. + step.xfers.push_back({r, dst, BUF_WORK_A, BUF_TMP, off, off, m}); + step.post.push_back({dst, BUF_TMP, off, BUF_WORK_A, off, m, true}); + }); + } + sched.push_back(std::move(step)); + } + } + { + Step stepf; + for (int r = 0; r < N; ++r) + stepf.post.push_back({r, BUF_WORK_A, (size_t)r * m, BUF_RECV, 0, m, false}); + sched.push_back(std::move(stepf)); + } + return sched; +} + +// Builds the schedule for `all_reduce` by composing the (unmodified) reduce_scatter and +// all_gather builders over sub-shards of size m/N, per the spec. +// +// Design: build_reduce_scatter(dims, m) is called with reduce_scatter's OWN "S" parameter set +// to all_reduce's per-rank vector size m (not to S itself). Internally, reduce_scatter then +// computes its own m_rs = m/N == the desired sub-shard size, and its own N*m_rs == m exactly +// matches the size of all_reduce's per-rank input buffer -- so all_reduce's existing BUF_SEND +// content can be fed to reduce_scatter completely unmodified, with no data rearrangement, +// because a flat m-byte vector split into N contiguous m/N-byte pieces is precisely +// reduce_scatter's own "N contiguous slots" input convention. Symmetrically, +// build_all_gather(dims, m) produces an N*(m/N) == m byte output, exactly all_reduce's needed +// result size. +// +// A single bridging LocalOp copies reduce_scatter's own (small, m/N-byte) output out of its +// BUF_RECV into BUF_SEND, where the reused all_gather schedule's own step 0 expects to find its +// input. This reuse is safe because reduce_scatter only ever READS BUF_SEND once, in its own +// step 0; by the time its schedule finishes, BUF_SEND is dead and free to reuse as scratch. +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N is each rank's full input/output vector size. +// +// Returns +// ------- +// Schedule -- reduce_scatter(dims,m)'s steps, then 1 bridging step, then all_gather(dims,m)'s +// steps. +inline Schedule build_all_reduce(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + size_t m = S / (size_t)N; + size_t shard = m / (size_t)N; + + Schedule sched = build_reduce_scatter(dims, m); + { + Step bridge; + for (int r = 0; r < N; ++r) + bridge.post.push_back({r, BUF_RECV, 0, BUF_SEND, 0, shard, false}); + sched.push_back(std::move(bridge)); + } + Schedule ag = build_all_gather(dims, m); + for (auto& step : ag) sched.push_back(std::move(step)); + return sched; +} + +// Builds the schedule for `alltoall` using dimension-ordered minimal routing. +// +// Each of the N*(N-1) non-self chunks (u,v), u != v, starts at rank u (BUF_SEND offset v*c, +// c = m/N) and must reach rank v (BUF_RECV offset u*c). Self chunks (u == v) never move; they +// are resolved by a single step-0 LocalOp per rank. +// +// Routing: dimensions are processed in ascending order. Within dimension d's phase, every chunk +// whose current holder disagrees with its target on coordinate d takes one hop per Step, in the +// direction (+1 or -1) that minimizes remaining distance around that dimension's ring; the +// phase repeats until no chunk needs to move in dimension d (bounded by extent_d - 1 +// iterations, the maximum possible remaining distance, with an explicit abort if that bound is +// ever exceeded -- it should not be, since minimal-direction hops need at most +// floor(extent_d/2) <= extent_d - 1 of them). +// +// In-flight buffer management: a chunk not currently at BUF_SEND or BUF_RECV lives in one of +// two ping-pong work buffers, BUF_WORK_A/BUF_WORK_B, alternating buffers each time it takes a +// hop (this guarantees a chunk's read-from buffer this step always differs from its +// write-to buffer this step, so simple sequential-then-concurrent memory semantics -- +// read-all-then-write-all within a Step -- can never alias a chunk's own old and new copies of +// itself). +// +// Slot allocation within a work buffer is a genuine per-step host-side bump allocator (a +// closed-form offset formula is not safe in general: two chunks that share a source rank and +// happen to have the same destination coordinate on every already-matched dimension travel +// together and can simultaneously occupy the same intermediate rank, so slots must be assigned, +// not computed). New allocations for step t are chosen to avoid every slot occupied at the +// START of step t (including slots about to be vacated this same step, since those are still +// being read from concurrently); old slots are freed only once the step's Xfers have been +// fully built, so freed capacity becomes available starting with the NEXT step, never the +// current one. Total occupancy per (rank, buffer) is asserted <= the buffer's slot capacity +// (2*N slots of c bytes each, matching the 2*m-byte buffer size from buffer_sizes()); the spec +// notes this should stay near N by symmetry for uniform all-to-all, which is also confirmed by +// a max-occupancy check in the throwaway Python simulation used to validate this algorithm +// before writing it here (observed max was well under half the budget for all tested dims). +// +// A chunk that arrives at its destination (all dimensions matched) is routed directly into +// BUF_RECV instead of a work buffer and is marked done, removing it from further consideration. +// +// Parameters +// ---------- +// dims : const std::vector& +// S : size_t -- total collective bytes; m = S/N per rank, c = m/N per chunk. +// +// Returns +// ------- +// Schedule -- 1 (self-chunk) step, followed by one step per routing hop actually taken. +inline Schedule build_alltoall(const std::vector& dims, size_t S) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + size_t c = m / (size_t)N; + Schedule sched; + { + Step step0; + for (int r = 0; r < N; ++r) + step0.post.push_back({r, BUF_SEND, (size_t)r * c, BUF_RECV, (size_t)r * c, c, false}); + sched.push_back(std::move(step0)); + } + + // Host-side (schedule-build-time only) bookkeeping of each in-flight chunk's current + // position. buf: -1 == still/again at BUF_SEND (never true after the first hop, but used + // as the initial state so the first hop's ping-pong toggle lands on BUF_WORK_A); 0 == + // BUF_WORK_A; 1 == BUF_WORK_B. slot is meaningful only when buf >= 0. + struct A2AChunk { + int u, v, holder; + int buf; + int slot; + bool done; + }; + std::vector chunks; + chunks.reserve((size_t)N * (N - 1)); + for (int u = 0; u < N; ++u) + for (int v = 0; v < N; ++v) + if (u != v) chunks.push_back({u, v, u, -1, -1, false}); + + const int CAP_SLOTS = 2 * N; // 2*m bytes / c bytes-per-slot, matching buffer_sizes(). + std::vector, 2>> occupied(N); + for (int r = 0; r < N; ++r) { + occupied[r][0].assign(CAP_SLOTS, 0); + occupied[r][1].assign(CAP_SLOTS, 0); + } + auto alloc_slot = [&](int rank, int buf) -> int { + for (int s = 0; s < CAP_SLOTS; ++s) { + if (!occupied[rank][buf][s]) { + occupied[rank][buf][s] = 1; + return s; + } + } + std::fprintf(stderr, + "%s:%d: alltoall work-buffer overflow at rank %d buf %d (occupancy " + "would exceed 2*m bytes)\n", + __FILE__, __LINE__, rank, buf); + std::abort(); + return -1; // unreachable + }; + + for (int d = 0; d < K; ++d) { + int E = dims[d]; + int iter = 0; + while (true) { + std::vector movers; + for (size_t i = 0; i < chunks.size(); ++i) { + if (chunks[i].done) continue; + std::vector cr = coords_of(chunks[i].holder, dims); + std::vector cv = coords_of(chunks[i].v, dims); + if (cr[d] != cv[d]) movers.push_back(i); + } + if (movers.empty()) break; + ++iter; + if (iter > E - 1) { + std::fprintf(stderr, + "%s:%d: alltoall routing failed to converge in dimension %d " + "within %d iterations\n", + __FILE__, __LINE__, d, E - 1); + std::abort(); + } + + struct Move { + size_t chunk_idx; + int dst_rank; + int new_buf, new_slot; + bool arrives; + }; + std::vector moves; + moves.reserve(movers.size()); + // Pass 1: decide direction/destination and allocate NEW slots against the + // occupancy snapshot as of the start of this step (see design note above: old + // slots are deliberately not freed until pass 3, so a slot being read from this + // step is never handed out as someone else's new landing spot this same step). + for (size_t i : movers) { + A2AChunk& ch = chunks[i]; + std::vector cr = coords_of(ch.holder, dims); + std::vector cv = coords_of(ch.v, dims); + int distp = ((cv[d] - cr[d]) % E + E) % E; + int distm = ((cr[d] - cv[d]) % E + E) % E; + int dirn = (distp <= distm) ? +1 : -1; + int dst = neighbor(ch.holder, d, dirn, dims); + check_edge_or_abort(ch.holder, dst, dims); + std::vector cdst = coords_of(dst, dims); + bool arrives = true; + for (int j = 0; j < K; ++j) + if (cdst[j] != cv[j]) { arrives = false; break; } + int new_buf = -1, new_slot = -1; + if (!arrives) { + new_buf = (ch.buf == 0) ? 1 : 0; // ping-pong: SEND or B -> A; A -> B. + new_slot = alloc_slot(dst, new_buf); + } + moves.push_back({i, dst, new_buf, new_slot, arrives}); + } + // Pass 2: emit Xfers from each chunk's OLD position to its NEW position. + Step step; + for (const Move& mv : moves) { + A2AChunk& ch = chunks[mv.chunk_idx]; + int src_buf = (ch.buf < 0) ? (int)BUF_SEND : (ch.buf == 0 ? (int)BUF_WORK_A : (int)BUF_WORK_B); + size_t src_off = (ch.buf < 0) ? (size_t)ch.v * c : (size_t)ch.slot * c; + int dst_buf; + size_t dst_off; + if (mv.arrives) { + dst_buf = BUF_RECV; + dst_off = (size_t)ch.u * c; + } else { + dst_buf = (mv.new_buf == 0) ? (int)BUF_WORK_A : (int)BUF_WORK_B; + dst_off = (size_t)mv.new_slot * c; + } + step.xfers.push_back({ch.holder, mv.dst_rank, src_buf, dst_buf, src_off, dst_off, c}); + } + // Pass 3: now that this step's Xfers are fully built, free vacated slots and + // update chunk bookkeeping for the next iteration. + for (const Move& mv : moves) { + A2AChunk& ch = chunks[mv.chunk_idx]; + if (ch.buf >= 0) occupied[ch.holder][ch.buf][ch.slot] = 0; + ch.holder = mv.dst_rank; + if (mv.arrives) { + ch.done = true; + ch.buf = -1; + ch.slot = -1; + } else { + ch.buf = mv.new_buf; + ch.slot = mv.new_slot; + } + } + sched.push_back(std::move(step)); + } + } + + for (const A2AChunk& ch : chunks) { + if (!ch.done) { + std::fprintf(stderr, + "%s:%d: alltoall chunk (u=%d,v=%d) failed to reach its destination\n", + __FILE__, __LINE__, ch.u, ch.v); + std::abort(); + } + } + return sched; +} + +// ===================================================================================== +// Intra-step aliasing validation (shared, pure host code -- runs as part of build_schedule()'s +// final validation pass, so it applies identically to whichever executor -- simulator or +// NCCL/CUDA -- ends up running the assembled Schedule). +// ===================================================================================== +// +// Design/WHY (see also run_schedule()'s Notes, in both the NCCL and simulator branches below, +// for the two executors' actual ordering guarantees this assertion is checked against): the +// host-memory simulator's run_schedule() gives every Step's Xfers STRONGER semantics than the +// NCCL/CUDA executor actually provides -- it snapshots every Xfer's source data before writing +// ANY Xfer's destination (see that function's own "Design decision" comment), so a schedule with +// overlapping src/dst ranges within one Step would silently produce a correct result there even +// though it would NOT on real hardware. The NCCL executor instead relies only on: (1) all of a +// Step's Xfers issued together inside one ncclGroupStart/End, with no ordering guarantee among +// DIFFERENT Xfers of that group beyond NCCL's own send/recv rendezvous (which pairs a specific +// send with its specific matching recv -- it says nothing about two unrelated Xfers of the same +// Step racing each other), and (2) per-rank CUDA stream order, which guarantees only that ops +// enqueued LATER on one rank's stream see the effects of ops enqueued EARLIER on that SAME +// stream -- in particular, a Step's post LocalOps (always enqueued, on every rank's stream, +// strictly after that Step's Xfers -- see run_schedule()) are guaranteed to see every effect of +// that Step's own Xfers, but no other cross-operation ordering is guaranteed by construction. +// +// This function makes that gap an assertion instead of a latent, hard-to-reproduce bug: every +// schedule this file ever builds is validated here to never depend on any same-step read/write +// ordering stronger than "a post-op reads a range a same-step Xfer just wrote" -- the one +// relationship the NCCL executor's fixed enqueue order (a Step's Xfers, then that Step's +// post-ops) always provides "for free", with no host synchronization required. Any OTHER +// same-step read/write overlap on the same (rank, buffer) -- e.g. one Xfer's source overlapping +// another Xfer's destination, or a post-op's destination overlapping another post-op's source -- +// would only be safe under the simulator's stronger snapshot semantics, and must never occur. + +// One (rank, buffer) byte range read or written by one Xfer or LocalOp within a single Step, used +// only by check_step_aliasing_or_abort() below. +struct _ByteRange { + int rank; + int buf; + size_t begin, end; // half-open [begin, end), in bytes. + bool is_post; // true if this range comes from a LocalOp (post); false if from an Xfer. +}; + +// Returns whether two half-open byte ranges overlap. +inline bool _ranges_overlap(size_t a_begin, size_t a_end, size_t b_begin, size_t b_end) { + return a_begin < b_end && b_begin < a_end; +} + +// Validates one Step against the aliasing rule described above; aborts with a diagnostic naming +// the step index and both offending ranges on the first violation found. +// +// Parameters +// ---------- +// step_idx : size_t -- index of `step` within its Schedule (diagnostics only). +// step : const Step& +// +// Notes +// ----- +// The single allowed exception -- a post-op's SOURCE range overlapping an Xfer's DESTINATION +// range, both of the same Step -- is exactly the reduce_scatter reduce-phase pattern (an Xfer +// lands a value in BUF_TMP, and that same Step's post-op immediately adds it out of BUF_TMP); +// see run_schedule()'s Notes for why that ordering (post-ops always after that Step's Xfers) is +// safe on both executors. Every other overlap combination aborts. +inline void check_step_aliasing_or_abort(size_t step_idx, const Step& step) { + std::vector<_ByteRange> reads, writes; + for (const Xfer& x : step.xfers) { + reads.push_back({x.src, x.src_buf, x.src_off, x.src_off + x.bytes, false}); + writes.push_back({x.dst, x.dst_buf, x.dst_off, x.dst_off + x.bytes, false}); + } + for (const LocalOp& op : step.post) { + reads.push_back({op.rank, op.src_buf, op.src_off, op.src_off + op.bytes, true}); + writes.push_back({op.rank, op.dst_buf, op.dst_off, op.dst_off + op.bytes, true}); + } + for (const _ByteRange& r : reads) { + for (const _ByteRange& w : writes) { + if (r.rank != w.rank || r.buf != w.buf) continue; + if (!_ranges_overlap(r.begin, r.end, w.begin, w.end)) continue; + // The one guaranteed-safe relationship: a post-op reading exactly what a same-step + // Xfer just wrote (see this function's Notes above and run_schedule()'s Notes). + if (r.is_post && !w.is_post) continue; + std::fprintf( + stderr, + "%s:%d: INTRA-STEP ALIAS ASSERTION FAILED at step %zu: rank %d buf %s read " + "range [%zu,%zu) (from %s) overlaps write range [%zu,%zu) (from %s); this " + "schedule relies on same-step read/write ordering the NCCL executor does not " + "guarantee (see check_step_aliasing_or_abort()'s Notes)\n", + __FILE__, __LINE__, step_idx, r.rank, buf_name(r.buf), r.begin, r.end, + r.is_post ? "a post-op" : "an Xfer", w.begin, w.end, + w.is_post ? "a post-op" : "an Xfer"); + std::abort(); + } + } +} + +// Dispatches to the appropriate per-collective builder and then re-validates every Xfer in the +// assembled schedule against the torus-edge guarantee, and every Step against the intra-step +// aliasing rule above. +// +// Notes +// ----- +// Every builder above already calls check_edge_or_abort() at each Xfer's construction site, +// which fails fastest and with the most local context. The bulk re-scan here is a deliberate +// belt-and-suspenders duplication: it makes build_schedule() itself -- not just its helpers -- +// the literal authority for the scientific guarantee described in the file header, matching the +// spec's requirement that "build_schedule asserts is_torus_neighbor(...) for every Xfer". The +// aliasing pass (check_step_aliasing_or_abort()) has no earlier per-Xfer equivalent -- it is +// inherently a whole-Step check -- so build_schedule() is the only place it can run. +// +// Parameters +// ---------- +// c : Collective +// dims : const std::vector& -- torus extents. +// S : size_t -- total collective bytes for this sweep point. +// +// Returns +// ------- +// Schedule -- fully built, edge-validated, and aliasing-validated. +inline Schedule build_schedule(Collective c, const std::vector& dims, size_t S) { + Schedule sched; + switch (c) { + case Collective::SENDRECV: sched = build_sendrecv(dims, S); break; + case Collective::BROADCAST: sched = build_broadcast(dims, S); break; + case Collective::ALL_GATHER: sched = build_all_gather(dims, S); break; + case Collective::REDUCE_SCATTER: sched = build_reduce_scatter(dims, S); break; + case Collective::ALL_REDUCE: sched = build_all_reduce(dims, S); break; + case Collective::ALLTOALL: sched = build_alltoall(dims, S); break; + } + for (size_t i = 0; i < sched.size(); ++i) { + for (const Xfer& x : sched[i].xfers) check_edge_or_abort(x.src, x.dst, dims); + check_step_aliasing_or_abort(i, sched[i]); + } + return sched; +} + +// ===================================================================================== +// --check data pattern: shared, pure-host generation and verification logic +// ===================================================================================== +// +// The --check input convention is uniform across all six collectives: every rank's BUF_SEND +// buffer (whatever its collective-specific size happens to be) is filled with f(rank, i) at +// flat float index i = 0 .. (buffer_bytes/4 - 1). This single convention subsumes every +// per-collective fill rule described in the spec (e.g. reduce_scatter's "rank u's BUF_SEND slot +// v holds f(u, v*elems_per_slot + i)" and alltoall's analogous per-chunk rule are both just this +// same flat-buffer fill, since a slot/chunk is by construction a contiguous sub-range of the +// flat buffer and v*elems_per_slot+i (or v*celems+i) is exactly that sub-range's flat index). +// +// Because f() is a pure, deterministic function of (source rank, index), verification never +// needs to read another rank's actual buffer contents: for every element of a rank's BUF_RECV, +// we know analytically which (source rank, source flat index) it is supposed to equal (or, for +// the reducing collectives, which SET of them it is supposed to sum) and simply recompute f() +// directly. This is implemented once, in verify_collective() below, and reused unmodified by +// both executors. + +// Deterministic --check fill value. +// +// Parameters +// ---------- +// r : int -- source rank. +// i : long long -- flat float index within that rank's buffer; must be >= 0 in practice (the +// modulo-97 reduction below is defensive against negative input but this file never +// constructs a negative index). +// +// Returns +// ------- +// float -- r*100 + (i mod 97). Exact in fp32 (see verify_collective() notes on why summing up +// to N such values remains exactly representable for the N, dims this benchmark targets). +inline float f_pattern(int r, long long i) { + long long im = i % 97; + if (im < 0) im += 97; + return (float)(r * 100 + (int)im); +} + +// Generates the full --check fill pattern for one rank's BUF_SEND buffer. +// +// Parameters +// ---------- +// rank : int +// nbytes : size_t -- BUF_SEND size for this rank (from buffer_sizes()); must be a multiple of +// sizeof(float) (guaranteed by the S % (4*N*N) == 0 sweep-level precondition). +// +// Returns +// ------- +// std::vector -- nbytes/4 floats, out[i] == f_pattern(rank, i). +inline std::vector gen_pattern(int rank, size_t nbytes) { + size_t nf = nbytes / sizeof(float); + std::vector out(nf); + for (size_t i = 0; i < nf; ++i) out[i] = f_pattern(rank, (long long)i); + return out; +} + +// Diagnostic record for the first mismatch found by verify_collective(), used to format the +// "# CHECK ... FAIL (first mismatch: ...)" line. +struct MismatchInfo { + int rank = -1; + int buf = BUF_RECV; + long long idx = -1; + float want = 0.f; + float got = 0.f; +}; + +// Verifies one rank's BUF_RECV contents against the analytically-known-correct result for the +// given collective, per the check rules in the spec (all reducible, per the discussion above, +// to recomputing f_pattern() directly rather than reading other ranks' data). +// +// Parameters +// ---------- +// c : Collective +// dims : const std::vector& +// S : size_t -- total collective bytes for this sweep point. +// rank : int -- the rank whose BUF_RECV is being checked. +// recv : const std::vector& -- that rank's full BUF_RECV contents, already copied to +// host memory by the caller (trivial for the simulator; a cudaMemcpy D2H for the NCCL +// build). +// mm : MismatchInfo& -- filled in on the first mismatch found; unmodified if this returns true. +// +// Returns +// ------- +// bool -- true iff every checked element matched exactly. +// +// Notes +// ----- +// Comparisons use exact float equality. This is intentional and safe here, not a bug: every +// f_pattern() value is a small non-negative integer (r*100 + i%97, comfortably under 2^24 for +// the rank counts and indices this benchmark exercises), and summing up to N such values (N <= +// a few hundred in any realistic torus) never leaves the range of exactly-representable +// integers in fp32 at any intermediate step -- so no rounding ever occurs, in the executor's +// float accumulation OR in this function's own reference summation, and exact comparison is +// mathematically justified rather than merely convenient. +inline bool verify_collective(Collective c, const std::vector& dims, size_t S, int rank, + const std::vector& recv, MismatchInfo& mm) { + int N = num_ranks(dims); + int K = (int)dims.size(); + size_t m = S / (size_t)N; + + auto check_eq = [&](size_t idx, float want, float got) -> bool { + if (got != want) { + mm.rank = rank; + mm.buf = BUF_RECV; + mm.idx = (long long)idx; + mm.want = want; + mm.got = got; + return false; + } + return true; + }; + + switch (c) { + case Collective::SENDRECV: { + size_t nf = m / sizeof(float); + for (int d = 0; d < K; ++d) { + int src = neighbor(rank, d, -1, dims); + for (size_t i = 0; i < nf; ++i) { + size_t idx = (size_t)d * nf + i; + if (!check_eq(idx, f_pattern(src, (long long)i), recv[idx])) return false; + } + } + return true; + } + case Collective::BROADCAST: { + size_t nf = m / sizeof(float); + for (size_t i = 0; i < nf; ++i) + if (!check_eq(i, f_pattern(0, (long long)i), recv[i])) return false; + return true; + } + case Collective::ALL_GATHER: { + size_t nf = m / sizeof(float); + for (int u = 0; u < N; ++u) { + for (size_t i = 0; i < nf; ++i) { + size_t idx = (size_t)u * nf + i; + if (!check_eq(idx, f_pattern(u, (long long)i), recv[idx])) return false; + } + } + return true; + } + case Collective::REDUCE_SCATTER: { + size_t elems_per_slot = m / sizeof(float); + for (size_t i = 0; i < elems_per_slot; ++i) { + float want = 0.f; + for (int u = 0; u < N; ++u) + want += f_pattern(u, (long long)((size_t)rank * elems_per_slot + i)); + if (!check_eq(i, want, recv[i])) return false; + } + return true; + } + case Collective::ALL_REDUCE: { + size_t nf = m / sizeof(float); + for (size_t i = 0; i < nf; ++i) { + float want = 0.f; + for (int u = 0; u < N; ++u) want += f_pattern(u, (long long)i); + if (!check_eq(i, want, recv[i])) return false; + } + return true; + } + case Collective::ALLTOALL: { + size_t c_bytes = m / (size_t)N; + size_t celems = c_bytes / sizeof(float); + for (int u = 0; u < N; ++u) { + for (size_t i = 0; i < celems; ++i) { + size_t idx = (size_t)u * celems + i; + float want = f_pattern(u, (long long)((size_t)rank * celems + i)); + if (!check_eq(idx, want, recv[idx])) return false; + } + } + return true; + } + } + return true; +} + +// Prints the single required "# CHECK ..." line for one sweep size, per the exact output +// format in the spec. +inline void print_check_line(Collective c, size_t S, bool pass, const MismatchInfo& mm) { + if (pass) { + std::printf("# CHECK collective=%s S=%zu: PASS\n", collective_name(c), S); + } else { + std::printf( + "# CHECK collective=%s S=%zu: FAIL (first mismatch: rank=%d buf=%s idx=%lld " + "want=%g got=%g)\n", + collective_name(c), S, mm.rank, buf_name(mm.buf), mm.idx, (double)mm.want, + (double)mm.got); + } +} + +// ===================================================================================== +// CLI helpers +// ===================================================================================== + +// Parses a --dims argument of the form "2x2x2", "8", or "2x4" into per-dimension extents. +// +// Parameters +// ---------- +// s : const std::string& -- 'x'-separated positive integers. +// +// Returns +// ------- +// std::vector -- one entry per dimension, each > 0. Empty on any parse failure (missing +// token, non-digit character, or non-positive value), which callers treat as a CLI error. +inline std::vector parse_dims(const std::string& s) { + std::vector out; + size_t pos = 0; + while (pos <= s.size()) { + size_t next = s.find('x', pos); + std::string tok = (next == std::string::npos) ? s.substr(pos) : s.substr(pos, next - pos); + if (tok.empty()) return {}; + for (char ch : tok) + if (!std::isdigit((unsigned char)ch)) return {}; + int val = std::atoi(tok.c_str()); + if (val <= 0) return {}; + out.push_back(val); + if (next == std::string::npos) break; + pos = next + 1; + } + return out; +} + +// ===================================================================================== +// Executors: identical function signatures, divergent implementations. +// +// Everything above this point is pure host C++ with no CUDA dependency and is shared, +// unmodified, by both builds. Everything below is fenced per the spec: CUDA/NCCL code lives +// under `#ifndef TORUS_SIM`; the host-memory simulator (this environment's test vehicle, since +// no nvcc/GPU is available here) lives in the `#else` branch. Both branches implement the exact +// same set of function names and signatures -- alloc_buffers, free_buffers, sync_all_devices, +// fill_input_pattern, run_schedule, run_check, global_teardown -- so main() below is written +// once and never itself needs an #ifdef. +// ===================================================================================== + +#ifndef TORUS_SIM +// -------------------------------------------------------------------------------- +// NCCL/CUDA executor. Requires nvcc + a CUDA toolkit + NCCL; NOT buildable or runnable in this +// development environment (no nvcc/CUDA installed here -- see project report). Written to +// mirror every simulator-side function signature exactly, per the spec, so this branch never +// references anything sim-only. +// -------------------------------------------------------------------------------- +#include +#include + +// CUDA error-check macro: prints file:line and aborts on any non-success cudaError_t. +#define CUDA_CHECK(call) \ + do { \ + cudaError_t _e = (call); \ + if (_e != cudaSuccess) { \ + std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, __LINE__, \ + cudaGetErrorString(_e)); \ + std::abort(); \ + } \ + } while (0) + +// NCCL error-check macro: prints file:line and aborts on any non-success ncclResult_t. +#define NCCL_CHECK(call) \ + do { \ + ncclResult_t _r = (call); \ + if (_r != ncclSuccess) { \ + std::fprintf(stderr, "%s:%d: NCCL error: %s\n", __FILE__, __LINE__, \ + ncclGetErrorString(_r)); \ + std::abort(); \ + } \ + } while (0) + +// Elementwise in-place float accumulate: dst[i] += src[i] for i in [0, n). Used to implement +// LocalOp with add == true on the device (the host-side simulator does the equivalent with a +// plain loop; see the #else branch below). +__global__ void add_inplace(float* dst, const float* src, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) dst[i] += src[i]; +} + +static int g_N = 0; +static std::vector> g_dev_bufs; +static std::array g_buf_sizes{}; +static ncclComm_t* g_comms = nullptr; +static cudaStream_t* g_streams = nullptr; + +// Allocates per-(rank,buffer) device storage for the upcoming sweep size, and lazily +// initializes the (persistent, reused-across-sizes) NCCL communicators and per-device streams +// on the first call. +// +// Parameters +// ---------- +// N : int -- device/rank count (must match every subsequent call until the matching +// free_buffers()). +// sizes : const size_t[NUM_BUFS] -- from buffer_sizes(); a 0 entry allocates nothing (null ptr). +inline void alloc_buffers(int N, const size_t sizes[NUM_BUFS]) { + g_N = N; + for (int b = 0; b < NUM_BUFS; ++b) g_buf_sizes[b] = sizes[b]; + if (!g_comms) { + g_comms = new ncclComm_t[N]; + std::vector devs(N); + for (int i = 0; i < N; ++i) devs[i] = i; + NCCL_CHECK(ncclCommInitAll(g_comms, N, devs.data())); + g_streams = new cudaStream_t[N]; + for (int r = 0; r < N; ++r) { + CUDA_CHECK(cudaSetDevice(r)); + CUDA_CHECK(cudaStreamCreate(&g_streams[r])); + } + } + g_dev_bufs.assign(N, {nullptr, nullptr, nullptr, nullptr, nullptr}); + for (int r = 0; r < N; ++r) { + CUDA_CHECK(cudaSetDevice(r)); + for (int b = 0; b < NUM_BUFS; ++b) { + if (sizes[b] > 0) CUDA_CHECK(cudaMalloc(&g_dev_bufs[r][b], sizes[b])); + } + } +} + +// Frees the per-(rank,buffer) device storage allocated by the matching alloc_buffers() call. +// Communicators/streams are intentionally NOT destroyed here (they are reused across sweep +// sizes); see global_teardown() for final cleanup. +inline void free_buffers(int N) { + for (int r = 0; r < N; ++r) { + CUDA_CHECK(cudaSetDevice(r)); + for (int b = 0; b < NUM_BUFS; ++b) + if (g_dev_bufs[r][b]) CUDA_CHECK(cudaFree(g_dev_bufs[r][b])); + } + g_dev_bufs.clear(); +} + +// Blocks the host until every device has completed all work previously enqueued on its stream. +inline void sync_all_devices() { + for (int r = 0; r < g_N; ++r) { + CUDA_CHECK(cudaSetDevice(r)); + CUDA_CHECK(cudaDeviceSynchronize()); + } +} + +// Fills every rank's BUF_SEND with the deterministic --check pattern (see gen_pattern()), +// H2D-copying a host-generated array rather than launching a fill kernel (simplest correct +// option given the CUDA path cannot be tested locally; avoids a second untestable kernel). +inline void fill_input_pattern(int N) { + size_t send_bytes = g_buf_sizes[BUF_SEND]; + for (int r = 0; r < N; ++r) { + std::vector pat = gen_pattern(r, send_bytes); + CUDA_CHECK(cudaSetDevice(r)); + CUDA_CHECK(cudaMemcpy(g_dev_bufs[r][BUF_SEND], pat.data(), send_bytes, + cudaMemcpyHostToDevice)); + } +} + +// Executes one full pass of `sched` across all N devices. +// +// Notes +// ----- +// Correctness argument for the (deliberate) ABSENCE of any per-step host synchronization here +// (see the file header's SCHEDULE-AS-DATA section for the two-executor contract this satisfies): +// +// - RANK-LOCAL ordering is preserved by CUDA stream order alone, with no host round-trip +// needed. Every one of a given rank r's operations -- its ncclSend/ncclRecv calls (issued on +// g_streams[r] via g_comms[r]), its cudaMemcpyAsync D2D post-ops, and its add_inplace kernel +// launches -- are ALL enqueued on that same single stream g_streams[r], in the exact order +// this function enqueues them (every Step's Xfers, in loop order, followed by that Step's +// post LocalOps, in loop order, before moving to the next Step). CUDA guarantees operations +// enqueued on one stream execute in that enqueue order; a later-enqueued op on a stream is +// therefore guaranteed to see the effects of every earlier op on that SAME stream without +// any explicit sync between them. This is exactly what build_schedule()'s intra-step alias +// assertion (see that function) verifies is sufficient: it never lets a schedule reach this +// executor if it would require a per-rank read/write ordering stronger than "post-op reads +// what an Xfer of the same step already wrote" -- precisely the one relationship stream +// order already provides here, for free. +// - CROSS-RANK ordering (rank A's send must be matched by rank B's matching recv before either +// side's dependent work proceeds) is enforced by NCCL itself, not by any host synchronization +// this function performs: every Step's Xfers are issued inside one ncclGroupStart/End, and +// NCCL's own send/recv rendezvous protocol is what guarantees a recv only completes (on its +// own stream) once its matching send has actually transferred the data -- that handshake is +// GPU-side, asynchronous, and requires no host cudaStreamSynchronize call to be correct. +// +// Given both of the above, a host round-trip is not needed between every Step, nor between a +// Step's Xfers and its post LocalOps -- it is needed exactly ONCE per call to this function, to +// give the HOST a defined point at which every device's work for this entire schedule execution +// is known to have completed (callers that need to read results back, e.g. run_check() via +// cudaMemcpy, or that are timing this call, already provide that host sync themselves -- +// sync_all_devices() in main()'s check/warmup/timed-loop call sites -- but performing it once +// here too keeps this function's own postcondition self-contained rather than relying on every +// caller to remember to do it). This drops per-iteration host-sync round-trips from ~2x the +// number of Steps (the pre-fix per-step synchronize-after-Xfers + +// synchronize-after-post-LocalOps pattern) to exactly 1, with no change to the timed loop's +// measured semantics (main()'s warmup/timed loops already sync once before/after the whole w+n +// iteration count, not between individual run_schedule() calls). +inline void run_schedule(const Schedule& sched, int N) { + for (const Step& step : sched) { + ncclGroupStart(); + for (const Xfer& x : step.xfers) { + char* sptr = (char*)g_dev_bufs[x.src][x.src_buf] + x.src_off; + char* dptr = (char*)g_dev_bufs[x.dst][x.dst_buf] + x.dst_off; + NCCL_CHECK(ncclSend(sptr, x.bytes, ncclChar, x.dst, g_comms[x.src], g_streams[x.src])); + NCCL_CHECK(ncclRecv(dptr, x.bytes, ncclChar, x.src, g_comms[x.dst], g_streams[x.dst])); + } + ncclGroupEnd(); + for (const LocalOp& op : step.post) { + CUDA_CHECK(cudaSetDevice(op.rank)); + char* sptr = (char*)g_dev_bufs[op.rank][op.src_buf] + op.src_off; + char* dptr = (char*)g_dev_bufs[op.rank][op.dst_buf] + op.dst_off; + if (!op.add) { + CUDA_CHECK(cudaMemcpyAsync(dptr, sptr, op.bytes, cudaMemcpyDeviceToDevice, + g_streams[op.rank])); + } else { + size_t nf = op.bytes / sizeof(float); + int threads = 256; + int blocks = (int)((nf + (size_t)threads - 1) / (size_t)threads); + add_inplace<<>>((float*)dptr, + (const float*)sptr, nf); + } + } + } + // Host synchronization happens ONCE per schedule execution, here, after the last Step -- + // see the Notes above for why nothing between Steps (or between a Step's Xfers and its + // post-ops) needs it. + for (int r = 0; r < N; ++r) { + CUDA_CHECK(cudaSetDevice(r)); + CUDA_CHECK(cudaStreamSynchronize(g_streams[r])); + } +} + +// Runs the --check verification for every rank, D2H-copying each rank's BUF_RECV before +// delegating to the shared verify_collective(). Prints the single required "# CHECK ..." line. +// +// Returns +// ------- +// bool -- true iff every rank's BUF_RECV matched the expected result. +inline bool run_check(Collective c, const std::vector& dims, size_t S, int N) { + bool overall_pass = true; + MismatchInfo first_mm; + for (int r = 0; r < N; ++r) { + size_t nbytes = g_buf_sizes[BUF_RECV]; + size_t nf = nbytes / sizeof(float); + std::vector recv(nf); + CUDA_CHECK(cudaSetDevice(r)); + CUDA_CHECK(cudaMemcpy(recv.data(), g_dev_bufs[r][BUF_RECV], nbytes, cudaMemcpyDeviceToHost)); + MismatchInfo mm; + bool ok = verify_collective(c, dims, S, r, recv, mm); + if (!ok && overall_pass) { + overall_pass = false; + first_mm = mm; + } + } + print_check_line(c, S, overall_pass, first_mm); + return overall_pass; +} + +// Destroys the persistent NCCL communicators and CUDA streams created lazily by the first +// alloc_buffers() call. Safe to call even if alloc_buffers() was never called. +inline void global_teardown() { + if (g_comms) { + for (int r = 0; r < g_N; ++r) ncclCommDestroy(g_comms[r]); + delete[] g_comms; + g_comms = nullptr; + } + if (g_streams) { + for (int r = 0; r < g_N; ++r) { + cudaSetDevice(r); + cudaStreamDestroy(g_streams[r]); + } + delete[] g_streams; + g_streams = nullptr; + } +} + +#else +// -------------------------------------------------------------------------------- +// Host-memory simulator executor (TORUS_SIM). No CUDA/NCCL dependency whatsoever -- this is +// the local, GPU-free test vehicle exercised by the acceptance criteria in this environment. +// Timings produced here are, per the spec, not scientifically meaningful (there is no real +// interconnect being modeled); the schedule-building, edge-assertion, and per-collective +// correctness logic under test is identical to what the NCCL branch would execute. +// -------------------------------------------------------------------------------- + +static std::vector, NUM_BUFS>> g_host_bufs; + +// Allocates zero-initialized per-(rank,buffer) host storage for the upcoming sweep size. +// +// Parameters +// ---------- +// N : int -- rank count. +// sizes : const size_t[NUM_BUFS] -- from buffer_sizes(). +inline void alloc_buffers(int N, const size_t sizes[NUM_BUFS]) { + g_host_bufs.assign((size_t)N, std::array, NUM_BUFS>{}); + for (int r = 0; r < N; ++r) + for (int b = 0; b < NUM_BUFS; ++b) g_host_bufs[r][b].assign(sizes[b], 0); +} + +// Releases the host storage allocated by the matching alloc_buffers() call. +inline void free_buffers(int N) { + (void)N; + g_host_bufs.clear(); +} + +// No-op: the simulator is single-threaded host code, so every operation is already +// synchronous by construction. Present only so main()'s driver loop can call the same function +// name in both builds. +inline void sync_all_devices() {} + +// Fills every rank's BUF_SEND with the deterministic --check pattern (see gen_pattern()). +inline void fill_input_pattern(int N) { + for (int r = 0; r < N; ++r) { + std::vector pat = gen_pattern(r, g_host_bufs[r][BUF_SEND].size()); + std::memcpy(g_host_bufs[r][BUF_SEND].data(), pat.data(), pat.size() * sizeof(float)); + } +} + +// Executes one full pass of `sched` over host memory. +// +// Design decision: within a Step, ALL Xfer sources are first snapshotted into temporary +// buffers, and only then are all destinations written. This makes the "all xfers execute +// concurrently, reading pre-step state" semantics of Step literally true regardless of +// iteration order or any potential (believed absent, but not asserted) offset aliasing between +// a step's reads and writes -- a small, cheap robustness margin given how load-bearing exact +// schedule semantics are for this benchmark's scientific validity. +inline void run_schedule(const Schedule& sched, int N) { + (void)N; + for (const Step& step : sched) { + std::vector> staged(step.xfers.size()); + for (size_t i = 0; i < step.xfers.size(); ++i) { + const Xfer& x = step.xfers[i]; + const std::vector& src = g_host_bufs[x.src][x.src_buf]; + staged[i].assign(src.begin() + (long)x.src_off, src.begin() + (long)(x.src_off + x.bytes)); + } + for (size_t i = 0; i < step.xfers.size(); ++i) { + const Xfer& x = step.xfers[i]; + std::memcpy(g_host_bufs[x.dst][x.dst_buf].data() + x.dst_off, staged[i].data(), x.bytes); + } + for (const LocalOp& op : step.post) { + if (!op.add) { + std::memcpy(g_host_bufs[op.rank][op.dst_buf].data() + op.dst_off, + g_host_bufs[op.rank][op.src_buf].data() + op.src_off, op.bytes); + } else { + float* dst = reinterpret_cast(g_host_bufs[op.rank][op.dst_buf].data() + op.dst_off); + const float* src = reinterpret_cast(g_host_bufs[op.rank][op.src_buf].data() + op.src_off); + size_t nf = op.bytes / sizeof(float); + for (size_t i = 0; i < nf; ++i) dst[i] += src[i]; + } + } + } +} + +// Runs the --check verification for every rank directly against host memory (no device copy +// needed) and prints the single required "# CHECK ..." line. +// +// Returns +// ------- +// bool -- true iff every rank's BUF_RECV matched the expected result. +inline bool run_check(Collective c, const std::vector& dims, size_t S, int N) { + bool overall_pass = true; + MismatchInfo first_mm; + for (int r = 0; r < N; ++r) { + const std::vector& raw = g_host_bufs[r][BUF_RECV]; + size_t nf = raw.size() / sizeof(float); + std::vector recv(nf); + std::memcpy(recv.data(), raw.data(), raw.size()); + MismatchInfo mm; + bool ok = verify_collective(c, dims, S, r, recv, mm); + if (!ok && overall_pass) { + overall_pass = false; + first_mm = mm; + } + } + print_check_line(c, S, overall_pass, first_mm); + return overall_pass; +} + +// No-op in the simulator: there is no persistent device/communicator state to tear down. +inline void global_teardown() {} + +#endif // TORUS_SIM + +// ===================================================================================== +// main(): CLI parsing, sweep loop, and output. Shared verbatim by both builds -- everything it +// calls (alloc_buffers, free_buffers, fill_input_pattern, run_schedule, sync_all_devices, +// run_check, global_teardown, build_schedule, buffer_sizes) has an identical signature in both +// the CUDA and simulator branches above, so no #ifdef is needed here at all. +// ===================================================================================== + +int main(int argc, char** argv) { + std::string collective_str; + std::string dims_str; + long long b = -1, e = -1; + long long f = 2; + long long w = 5; + long long n = 20; + bool check = false; + + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto need_val = [&](const char* flag) -> std::string { + if (i + 1 >= argc) { + std::fprintf(stderr, "torus_bench: missing value for %s\n", flag); + std::exit(2); + } + return std::string(argv[++i]); + }; + if (a == "--collective") collective_str = need_val("--collective"); + else if (a == "--dims") dims_str = need_val("--dims"); + else if (a == "-b") b = std::atoll(need_val("-b").c_str()); + else if (a == "-e") e = std::atoll(need_val("-e").c_str()); + else if (a == "-f") f = std::atoll(need_val("-f").c_str()); + else if (a == "-w") w = std::atoll(need_val("-w").c_str()); + else if (a == "-n") n = std::atoll(need_val("-n").c_str()); + else if (a == "--check") check = true; + else { + std::fprintf(stderr, "torus_bench: unknown argument '%s'\n", a.c_str()); + return 2; + } + } + + if (collective_str.empty() || dims_str.empty() || b <= 0 || e <= 0) { + std::fprintf(stderr, + "usage: torus_bench --collective {all_reduce,all_gather,reduce_scatter," + "alltoall,broadcast,sendrecv} --dims -b " + "-e [-f 2] [-w 5] [-n 20] [--check]\n"); + return 2; + } + if (f <= 1) { + std::fprintf(stderr, "torus_bench: -f must be >= 2 (got %lld)\n", f); + return 2; + } + + Collective c; + if (!parse_collective(collective_str, c)) { + std::fprintf(stderr, "torus_bench: unknown --collective '%s'\n", collective_str.c_str()); + return 2; + } + std::vector dims = parse_dims(dims_str); + if (dims.empty()) { + std::fprintf(stderr, "torus_bench: invalid --dims '%s'\n", dims_str.c_str()); + return 2; + } + int N = num_ranks(dims); + +#ifdef TORUS_SIM + std::printf("# torus_bench [SIMULATOR build -- host memory only, no GPU/NCCL]\n"); +#else + std::printf("# torus_bench [NCCL/CUDA build]\n"); +#endif + std::printf( + "# collective=%s dims=%s N=%d range=[%lld,%lld] factor=%lld warmup=%lld iters=%lld " + "check=%d\n", + collective_name(c), dims_str.c_str(), N, b, e, f, w, n, check ? 1 : 0); + + bool any_check_failure = false; + for (long long S = b; S <= e; S *= f) { + size_t divisor = 4ull * (size_t)N * (size_t)N; + if ((size_t)S % divisor != 0) { + std::printf( + "# SKIP S=%lld not divisible by 4*N*N=%zu (N=%d): shard/chunk boundaries " + "would not align to whole floats\n", + S, divisor, N); + continue; + } + + Schedule sched = build_schedule(c, dims, (size_t)S); + std::array sizes = buffer_sizes(c, dims, (size_t)S); + alloc_buffers(N, sizes.data()); + + char check_char = '-'; + if (check) { + fill_input_pattern(N); + run_schedule(sched, N); + sync_all_devices(); + bool pass = run_check(c, dims, (size_t)S, N); + if (!pass) any_check_failure = true; + check_char = pass ? '1' : '0'; + } + + for (long long it = 0; it < w; ++it) run_schedule(sched, N); + sync_all_devices(); + auto t0 = std::chrono::steady_clock::now(); + for (long long it = 0; it < n; ++it) run_schedule(sched, N); + sync_all_devices(); + auto t1 = std::chrono::steady_clock::now(); + double total_us = std::chrono::duration(t1 - t0).count(); + double avg_us = (n > 0) ? (total_us / (double)n) : 0.0; + + free_buffers(N); + + std::printf("TORUSBENCH,%s,%s,%lld,%.2f,%c\n", collective_name(c), dims_str.c_str(), S, + avg_us, check_char); + std::fflush(stdout); + } + + global_teardown(); + return any_check_failure ? 1 : 0; +} + From de6cb33bfc19183b96a47f687cb2f465159a7ed0 Mon Sep 17 00:00:00 2001 From: Reng Zheng Date: Thu, 16 Jul 2026 15:47:25 -0400 Subject: [PATCH 18/19] stripped dead imports --- accelforge/model/_looptree/accesses.py | 2 +- accelforge/model/_looptree/latency/latency.py | 2 +- accelforge/model/_looptree/latency/memory.py | 2 +- .../model/_looptree/reuse/isl/__init__.py | 5 -- accelforge/model/_looptree/reuse/isl/des.py | 3 +- .../_looptree/reuse/isl/distributed/README.md | 2 - .../isl/distributed/distributed_buffers.py | 14 +--- .../distributed_models_demo.ipynb | 64 ++++++++----------- tests/isl/distributed/helpers.py | 51 +++------------ tests/isl/distributed/test_contract.py | 50 +++++++-------- tests/isl/distributed/test_edge_pressure.py | 40 +++--------- 11 files changed, 74 insertions(+), 161 deletions(-) diff --git a/accelforge/model/_looptree/accesses.py b/accelforge/model/_looptree/accesses.py index d54f2999..a643a5d6 100755 --- a/accelforge/model/_looptree/accesses.py +++ b/accelforge/model/_looptree/accesses.py @@ -5,7 +5,7 @@ import islpy as isl -from accelforge.model._looptree.reuse.isl import IslReuseAnalysisOutput +from accelforge.model._looptree.reuse.isl.des import IslReuseAnalysisOutput from accelforge.util._frozenset import oset from accelforge.model._looptree.reuse.symbolic import ( BuffetStats, diff --git a/accelforge/model/_looptree/latency/latency.py b/accelforge/model/_looptree/latency/latency.py index ab1810bd..9cdfadc4 100755 --- a/accelforge/model/_looptree/latency/latency.py +++ b/accelforge/model/_looptree/latency/latency.py @@ -4,7 +4,7 @@ # from accelforge.model._looptree._isl.singular import get_value_from_singular_qpolynomial from accelforge.frontend.arch import Compute from accelforge.model._looptree.latency.processors import LATENCY_PROCESSORS -from accelforge.model._looptree.reuse.isl import IslReuseAnalysisOutput +from accelforge.model._looptree.reuse.isl.des import IslReuseAnalysisOutput from accelforge.model._looptree.reuse import SymbolicAnalysisOutput from accelforge.util._sympy.broadcast_max import max_nonzero diff --git a/accelforge/model/_looptree/latency/memory.py b/accelforge/model/_looptree/latency/memory.py index ee97fbff..f90c395d 100755 --- a/accelforge/model/_looptree/latency/memory.py +++ b/accelforge/model/_looptree/latency/memory.py @@ -8,7 +8,7 @@ from accelforge.model._looptree.accesses import isl_buffer_accesses_from_buffet_actions from accelforge.model._looptree.mapping_utilities import get_leaves -from accelforge.model._looptree.reuse.isl import IslReuseAnalysisOutput +from accelforge.model._looptree.reuse.isl.des import IslReuseAnalysisOutput from accelforge.model._looptree.reuse import SymbolicAnalysisOutput from accelforge.model._looptree.types import Buffet diff --git a/accelforge/model/_looptree/reuse/isl/__init__.py b/accelforge/model/_looptree/reuse/isl/__init__.py index e295e773..e69de29b 100755 --- a/accelforge/model/_looptree/reuse/isl/__init__.py +++ b/accelforge/model/_looptree/reuse/isl/__init__.py @@ -1,5 +0,0 @@ -from .des import IslReuseAnalysisOutput - -__all__ = [ - "IslReuseAnalysisOutput", -] diff --git a/accelforge/model/_looptree/reuse/isl/des.py b/accelforge/model/_looptree/reuse/isl/des.py index a7890af3..7b8d2de4 100755 --- a/accelforge/model/_looptree/reuse/isl/des.py +++ b/accelforge/model/_looptree/reuse/isl/des.py @@ -1,6 +1,5 @@ """ -TODO: Is this file still necessary? It is referenced elsewhere but is no longer - the format we are looking for. +Deserializes LoopTree reuse-analysis output into ISL objects. """ from dataclasses import dataclass, field diff --git a/accelforge/model/_looptree/reuse/isl/distributed/README.md b/accelforge/model/_looptree/reuse/isl/distributed/README.md index cdd8c561..696c7f77 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/README.md +++ b/accelforge/model/_looptree/reuse/isl/distributed/README.md @@ -34,8 +34,6 @@ implementations: > per-link-load abstraction and its scalar-extraction helpers live in > [`edge_pressure.py`](edge_pressure.py); multicast-network construction (`identify_mesh_casts` > and the helpers built on its result) lives in [`mesh_casts.py`](mesh_casts.py). -> `distributed_buffers.py` re-exports the moved names, so existing imports of the four models and -> `_eval_const` from `distributed_buffers.py` keep working (see §9 for the full map). --- diff --git a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py index 601f4d5a..92614f56 100644 --- a/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py +++ b/accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py @@ -7,9 +7,7 @@ concrete model below contributes only its cost kernel via `_transfer_cost`. The underlying primitives -- `EdgePressure` and its scalar-extraction helpers, and the multicast-network construction (`identify_mesh_casts` and friends) -- -live in `edge_pressure.py` and `mesh_casts.py` respectively, and are -re-exported below for backward compatibility (see the Note at the import -site). +live in `edge_pressure.py` and `mesh_casts.py` respectively. """ from abc import abstractmethod @@ -27,18 +25,8 @@ TransferModel, ) -# Note: these re-exports preserve this module's historical public surface. -# `EdgePressure`/`_eval_const`/`_const_pwq`/`_edge_pressure_from_links` and -# `identify_mesh_casts`/`calculate_extents_per_dim`/`_covered_fills`/ -# `_mesh_node_tuple` used to be defined directly in this file; the test suite -# and `correlation.ipynb` import them from this module path, so the split -# into `edge_pressure.py` / `mesh_casts.py` must not break that import. -# `_union_pwqs`, `_per_src`, and `_fabric_crossing` are pulled in for this -# module's own internal use (Star/XY/FullyConnected kernels below), not part -# of the historical surface. from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import ( EdgePressure, - _eval_const, _const_pwq, _edge_pressure_from_links, _union_pwqs, diff --git a/notebooks/astrasim2_correlation/distributed_models_demo.ipynb b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb index 368c00b4..e107ea4b 100644 --- a/notebooks/astrasim2_correlation/distributed_models_demo.ipynb +++ b/notebooks/astrasim2_correlation/distributed_models_demo.ipynb @@ -54,19 +54,15 @@ "## How to use this notebook as a review guide\n", "\n", "Each numbered section states which functions it exercises and cites their\n", - "current `file:line` ranges, recomputed against this working tree. Two passes\n", - "followed the two commits above: a documentation/comment style pass (comments\n", - "and docstrings only, verified AST-equivalent to `HEAD` with docstrings\n", - "stripped), and then a structural `/simplify` module split that moved\n", - "`EdgePressure` and its scalar-extraction helpers into `edge_pressure.py` and\n", - "`identify_mesh_casts` and its helpers into `mesh_casts.py`, leaving\n", - "`distributed_buffers.py` with a new `MulticastModel` base class (the shared\n", - "`__init__`/`apply` shape every model below now uses) and each model's own\n", - "cost kernel. Both passes are behavior-preserving -- verified against the\n", - "unmodified test suite and the asserted numbers below -- so the citations\n", - "reflect current file locations and line numbers, not the original commit\n", - "diffs. Every number asserted in a code cell below (`56`, `64`, `448`, ...) was\n", - "independently re-verified against this exact code before being written here.\n", + "current `file:line` ranges, recomputed against this working tree:\n", + "`EdgePressure` and its scalar-extraction helpers live in `edge_pressure.py`,\n", + "`identify_mesh_casts` and its helpers in `mesh_casts.py`, and the\n", + "`MulticastModel` base class (the shared `__init__`/`apply` shape every model\n", + "below uses) with each model's own cost kernel in `distributed_buffers.py`.\n", + "The citations reflect current file locations and line numbers, not the\n", + "original commit diffs. Every number asserted in a code cell below (`56`, `64`,\n", + "`448`, ...) was independently re-verified against this exact code before\n", + "being written here.\n", "Section 6 collects the section -> function -> file:line-range -> commit\n", "mapping into one table so a reviewer can walk the diff systematically.\n" ] @@ -93,6 +89,8 @@ " FullyConnectedMulticastModel,\n", " StarMulticastModel,\n", " XYRoutingMulticastModel,\n", + ")\n", + "from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import (\n", " _eval_const,\n", ")\n", "from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import (\n", @@ -135,8 +133,7 @@ "(`distributed_buffers.py:523-599`, commit `142722f5`; its `hops`/\n", "`edge_pressure` aggregation is now the shared `MulticastModel.apply` above,\n", "fed by `StarMulticastModel._transfer_cost`, `distributed_buffers.py:546-547`);\n", - "`EdgePressure` (`edge_pressure.py:21-133`, commit `142722f5`, moved out of\n", - "`distributed_buffers.py` by the `/simplify` module split).\n" + "`EdgePressure` (`edge_pressure.py:21-133`, commit `142722f5`).\n" ] }, { @@ -351,8 +348,7 @@ "(`distributed_buffers.py:414-415`, commit `81be62a4`); `_directed_mesh_links`\n", "(`distributed_buffers.py:417-515`, commit `142722f5`, touched again in\n", "`81be62a4`); `EdgePressure.bottleneck` / `EdgePressure.eval_edge`\n", - "(`edge_pressure.py:89-133`, commit `142722f5`, moved out of\n", - "`distributed_buffers.py` by the `/simplify` module split)." + "(`edge_pressure.py:89-133`, commit `142722f5`)." ] }, { @@ -699,12 +695,9 @@ "(`fills.map_.subtract(fills.map_)`).\n", "\n", "**Code exercised**: `_covered_fills`\n", - "(`mesh_casts.py:114-137`, commit `81be62a4`, moved out of\n", - "`distributed_buffers.py` by the `/simplify` module split); the partition\n", - "usage itself is now the shared `MulticastModel.apply`\n", - "(`distributed_buffers.py:97-147`, commit `81be62a4`, folded from\n", - "`FullyConnectedMulticastModel.apply` and the other three models' identical\n", - "`apply` tails during the module split).\n" + "(`mesh_casts.py:114-137`, commit `81be62a4`); the partition\n", + "usage itself is the shared `MulticastModel.apply`\n", + "(`distributed_buffers.py:97-147`, commit `81be62a4`).\n" ] }, { @@ -775,8 +768,7 @@ "identical hops and bottleneck.\n", "\n", "**Code exercised**: `_mesh_node_tuple`\n", - "(`mesh_casts.py:140-164`, commit `81be62a4`, moved out of\n", - "`distributed_buffers.py` by the `/simplify` module split); its caller\n", + "(`mesh_casts.py:140-164`, commit `81be62a4`); its caller\n", "`XYRoutingMulticastModel._directed_mesh_links`\n", "(`distributed_buffers.py:417-515`, commit `142722f5`, touched again in\n", "`81be62a4`).\n" @@ -878,21 +870,15 @@ "| (all sections) | `_eval_const`, `_const_pwq` (shared scalar-extraction helpers) | `edge_pressure.py` | 159-201 | `81be62a4` |\n", "| 2, 3 | `_edge_pressure_from_links` | `edge_pressure.py` | 230-251 | `142722f5` |\n", "\n", - "Notes on mixed attribution: `XYRoutingMulticastModel.apply` and\n", - "`StarMulticastModel.apply` were first written in `de426ae0`/`142722f5` and had\n", - "their `hops`/`edge_pressure` derivation refactored to a single\n", - "`identify_mesh_casts` call in `81be62a4` (rows above cite `81be62a4` for that\n", - "reason). A later, behavior-preserving `/simplify` module split folded every\n", - "model's `apply` into one `MulticastModel.apply` (`distributed_buffers.py:\n", - "97-147`) and moved `EdgePressure`/`identify_mesh_casts` and their helpers into\n", - "`edge_pressure.py`/`mesh_casts.py` respectively; the table above cites that\n", - "current location rather than `apply` on each subclass, since the per-model\n", - "`apply` no longer exists post-split -- only each model's `_transfer_cost` (the\n", - "one-line cost-kernel hook `apply` now calls) and cost-specific helper remain\n", - "on the subclass. `FullyConnectedMulticastModel`'s cost kernel\n", + "Notes on mixed attribution: the XY and star models' `hops`/`edge_pressure`\n", + "derivation was refactored to a single `identify_mesh_casts` call in\n", + "`81be62a4` (rows above cite `81be62a4` for that reason); the shared\n", + "derivation lives in `MulticastModel.apply` (`distributed_buffers.py:97-147`),\n", + "with each model contributing only its `_transfer_cost` cost-kernel hook and\n", + "cost-specific helper. `FullyConnectedMulticastModel`'s cost kernel\n", "(`_cost_fully_connected`) is unchanged since `de426ae0` and predates the\n", - "reviewed range; only its fulfilled/unfulfilled partition usage (now inside\n", - "the shared `MulticastModel.apply`) is new in `81be62a4`.\n" + "reviewed range; only its fulfilled/unfulfilled partition usage (inside the\n", + "shared `MulticastModel.apply`) is new in `81be62a4`.\n" ] } ], diff --git a/tests/isl/distributed/helpers.py b/tests/isl/distributed/helpers.py index 78de3e3d..2fed1f87 100644 --- a/tests/isl/distributed/helpers.py +++ b/tests/isl/distributed/helpers.py @@ -1,28 +1,21 @@ """ Shared test helpers for the ISL distributed-buffer multicast model test suite. -Factors out two pieces of logic that used to be byte-for-byte duplicated across -``test_multicast.py``, ``test_fully_connected.py``, and ``test_xy_routing.py``: - 1. ``construct_spacetime`` -- turning a yaml ``dims`` list into ``Tag`` objects. 2. ``run_hops_gamut`` -- the "load a yaml of test cases, build a ``Fill`` /``Occupancy``/``dist_fn`` triple, run one ``TransferModel``, and compare - ``.hops`` against an expected key" loop, which is identical across the - three hop-oracle test files and differs only in which model class, which - yaml directory, and which ``expected`` key (``hypercube_hops`` / - ``fully_connected_hops`` / ``xy_routing_hops``) is used. - -With this module in place, ``test_multicast.py``, ``test_fully_connected.py``, -and ``test_xy_routing.py`` reduce to thin parametrizations: a ``TestCase`` whose -single ``test_gamut`` method calls ``run_hops_gamut`` with its model class, yaml -path, and expected-key string. + ``.hops`` against an expected key" loop shared by ``test_multicast.py``, + ``test_fully_connected.py``, and ``test_xy_routing.py``, which differ only + in which model class, which yaml directory, and which ``expected`` key + (``hypercube_hops`` / ``fully_connected_hops`` / ``xy_routing_hops``) is + used. Import note ----------- This module (and every test module in this package) imports the canonical -``load_solutions`` helper via the *absolute* path ``tests.isl.util`` rather than -a local copy. That import only resolves if the repository root is on -``sys.path``, which is the case when the suite is invoked from the repo root as +``load_solutions`` helper via the *absolute* path ``tests.isl.util``. That +import only resolves if the repository root is on ``sys.path``, which is the +case when the suite is invoked from the repo root as PATH="$HOME/.local/bin:$PATH" .venv/bin/python -m pytest tests/isl/distributed/ -q @@ -36,17 +29,7 @@ import islpy as isl -from accelforge.model._looptree.reuse.isl.distributed.distributed_buffers import ( - # Design (D6): the "evaluate a parameter-free PwQPolynomial at the zero - # point" idiom lives in exactly one place now -- see `_eval_const`'s - # docstring for the history of it drifting across four call sites with - # inconsistent return types. This module used to inline its own copy - # (`info.hops.eval(isl.Point.zero(...))`, which returns an `isl.Val` - # despite looking `int`-shaped); importing the canonical helper instead - # keeps this the single non-test call site left outside - # `distributed_buffers.py` and guarantees a real Python `int`. - _eval_const, -) +from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import _eval_const from accelforge.model._looptree.reuse.isl.mapping_to_isl.types import ( # Data movement descriptors. Fill, @@ -58,11 +41,6 @@ ) from accelforge.model._looptree.reuse.isl.spatial import TransferInfo, TransferModel -# Design: this package (formerly `tests/not_working/distribuffers/`) used to -# carry a byte-identical copy of `tests/isl/util.py` as its own `util.py`. -# Rather than keep two copies of `load_solutions` in sync by hand, it imports -# the canonical one directly -- see the "Import note" above for the -# run-from-repo-root requirement this implies. from tests.isl.util import load_solutions @@ -97,14 +75,6 @@ def run_hops_gamut(model_cls: type[TransferModel], yaml_path: Path, expected_key Run every test case in a yaml gamut file through ``model_cls`` and assert the resulting ``TransferInfo.hops`` matches the case's expected value. - # Design: `test_multicast.py`, `test_fully_connected.py`, and - # `test_xy_routing.py` each had their own copy of this loop; the only - # differences were the model class being exercised, the yaml directory the - # cases were loaded from, and which key of `test["expected"]` held the - # oracle value (`hypercube_hops` / `fully_connected_hops` / - # `xy_routing_hops`). Parametrizing on those three makes the loop itself - # single-sourced. - Parameters ---------- model_cls: @@ -118,8 +88,7 @@ def run_hops_gamut(model_cls: type[TransferModel], yaml_path: Path, expected_key The key within each case's ``expected`` dict holding the oracle hop count for this model (e.g. ``"xy_routing_hops"``). A ``None`` value at this key marks a case as still in progress: rather than failing, the - case's inputs/output are printed for manual inspection (preserving the - original tests' "unimplemented case" debugging affordance). + case's inputs/output are printed for manual inspection. Raises ------ diff --git a/tests/isl/distributed/test_contract.py b/tests/isl/distributed/test_contract.py index 1bc30faa..1b6c6b59 100644 --- a/tests/isl/distributed/test_contract.py +++ b/tests/isl/distributed/test_contract.py @@ -1,33 +1,31 @@ """ -Regression tests for the D1/D2/D3 contract fixes to -``distributed_buffers.py``'s multicast models (see the module's design notes -and the module-level ``_mesh_node_tuple``/``_covered_fills``/ -``identify_mesh_casts`` docstrings for the full rationale). Each class below -pins down one previously-broken-or-undocumented contract so a future change -cannot silently regress it: +Regression tests for contract fixes to ``distributed_buffers.py``'s multicast +models (see the ``_mesh_node_tuple``/``_covered_fills``/ +``identify_mesh_casts`` docstrings in ``mesh_casts.py`` for the full +rationale). Each class below pins down one previously-broken-or-undocumented +contract so a future change cannot silently regress it: -- ``TestTupleNameGenericity`` (D1): a spacetime/node tuple named anything +- ``TestTupleNameGenericity``: a spacetime/node tuple named anything other than the literal ``'noc'`` (e.g. ``pe[x, y]``) used to crash deep inside an opaque ISL assertion in ``XYRoutingMulticastModel``'s and ``StarMulticastModel``'s helpers, which hardcoded ``'noc'`` into every map string they built. Both now read the tuple name off the caller's own maps. -- ``TestXYDimensionalityGuard`` (D1): a non-2-D node tuple through +- ``TestXYDimensionalityGuard``: a non-2-D node tuple through ``XYRoutingMulticastModel`` now raises a clear ``ValueError`` instead of the same kind of opaque ISL assertion. -- ``TestUnfulfilledPartition`` (D2): ``fulfilled_fill``/``unfulfilled_fill`` +- ``TestUnfulfilledPartition``: ``fulfilled_fill``/``unfulfilled_fill`` are a true partition of ``fills`` by whether ``identify_mesh_casts`` found a matched source, for all four models -- previously ``unfulfilled_fill`` was unconditionally empty regardless of whether a fill actually had a source. -- ``TestAsymmetricDistFnOrientation`` (D3): ``identify_mesh_casts``'s - ``dist_fn`` is documented (after the D3 fix) to be applied in the +- ``TestAsymmetricDistFnOrientation``: ``identify_mesh_casts``'s + ``dist_fn`` is documented to be applied in the ``{ [dst -> src] -> [hops] }`` orientation; this pins down that an asymmetric ``dist_fn`` (direction-dependent cost) actually selects the source that is *cheapest to reach from the destination*, not some other pairing that a swapped orientation would silently produce. See accelforge/model/_looptree/reuse/isl/distributed/README.md for the models' -background, and accelforge/model/_looptree/reuse/isl/distributed/distributed_buffers.py -for the D1/D2/D3 design comments this file exercises. +background. """ import unittest @@ -44,8 +42,8 @@ FullyConnectedMulticastModel, XYRoutingMulticastModel, StarMulticastModel, - _eval_const, ) +from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import _eval_const _CTX = isl.DEFAULT_CONTEXT @@ -56,7 +54,7 @@ def _manhattan_2d(name: str) -> isl.Map: node tuple named ``name`` (e.g. ``"noc"`` or ``"pe"``), identical in shape to the ``2d_manhattan`` fixture in ``xy_routing/test_cases.yaml`` but with the tuple name parametrized so it can be reused for the tuple-name - genericity regression tests (D1) below. + genericity regression tests below. """ return isl.Map.read_from_str( _CTX, @@ -94,7 +92,7 @@ def _all_to_all(name: str, n: int) -> tuple[Fill, Occupancy, isl.Map]: Node ``g`` starts holding only ``data[g]`` and requests every other node's datum, exactly the ``_all_to_all`` fixture in ``test_edge_pressure.py`` - but with the tuple name parametrized for the D1 genericity checks. + but with the tuple name parametrized for the genericity checks. """ tags = [SpatialTag(0, 0)] occ = Occupancy( @@ -115,7 +113,7 @@ def _all_to_all(name: str, n: int) -> tuple[Fill, Occupancy, isl.Map]: class TestTupleNameGenericity(unittest.TestCase): """ - D1: a node tuple named anything other than ``'noc'`` must produce the same + A node tuple named anything other than ``'noc'`` must produce the same numbers as the ``'noc'``-named equivalent, for every model whose helpers used to hardcode the literal ``'noc'`` into ISL map strings. """ @@ -124,7 +122,7 @@ def test_xy_non_noc_tuple_name_matches_noc(self): """ XY routing case B (source (1,0) -> (0,2),(2,2), 6 hops -- see ``xy_routing/test_cases.yaml``) run under both ``noc[x, y]`` and - ``pe[x, y]`` must produce the identical hop count. Before D1, `'pe'` + ``pe[x, y]`` must produce the identical hop count. Previously, `'pe'` crashed inside ``_directed_mesh_links``'s hardcoded ``'noc[...]'`` map strings with a raw ISL tuple-name-mismatch assertion. """ @@ -153,7 +151,7 @@ def test_star_and_fully_connected_non_noc_tuple_name_match_noc(self): """ The 8-node all-to-all oracle (FullyConnected 56, Star 64 -- see ``TestStarSpokePressure`` in ``test_edge_pressure.py``) run under both - ``noc[g]`` and ``pe[g]`` must produce identical numbers. Before D1, + ``noc[g]`` and ``pe[g]`` must produce identical numbers. Previously, ``StarMulticastModel._spoke_loads`` hardcoded ``'noc'`` the same way ``_directed_mesh_links`` did (``FullyConnectedMulticastModel`` never hardcoded a tuple name, so it is included here only as a same-input @@ -174,7 +172,7 @@ def test_star_and_fully_connected_non_noc_tuple_name_match_noc(self): class TestXYDimensionalityGuard(unittest.TestCase): """ - D1: ``XYRoutingMulticastModel`` only supports exactly 2-D node tuples, and + ``XYRoutingMulticastModel`` only supports exactly 2-D node tuples, and must say so with a clear ``ValueError`` rather than an opaque ISL abort. """ @@ -184,7 +182,7 @@ def test_three_dimensional_node_tuple_raises_value_error(self): ``apply`` (via ``_directed_mesh_links``), naming the offending tuple and its dimensionality, instead of failing deep inside an ``isl.Map.read_from_str``/``apply_range`` call the way it did before - D1's ``_mesh_node_tuple`` dimensionality check was added. The + ``_mesh_node_tuple``'s dimensionality check was added. The `dist_fn` here is a trivial constant (correctness of routing over a 3-D mesh is out of scope -- this test only exercises the guard). """ @@ -213,11 +211,11 @@ def test_three_dimensional_node_tuple_raises_value_error(self): class TestUnfulfilledPartition(unittest.TestCase): """ - D2: ``fulfilled_fill``/``unfulfilled_fill`` must be a true partition of + ``fulfilled_fill``/``unfulfilled_fill`` must be a true partition of ``fills`` by whether ``identify_mesh_casts`` matched a source, for every model. Shared geometry: node 0 holds only ``data[0]``; node 1 requests both ``data[0]`` (has a source -- fulfilled) and ``data[1]`` (no source - anywhere -- unfulfilled). Before D2, every model reported + anywhere -- unfulfilled). Previously, every model reported ``unfulfilled_fill`` as unconditionally empty (``fills - fills``) and ``fulfilled_fill`` as the entire (uncovered) fill map. """ @@ -313,10 +311,10 @@ def test_partition_across_models(self): class TestAsymmetricDistFnOrientation(unittest.TestCase): """ - D3: ``identify_mesh_casts`` applies ``dist_fn`` in the + ``identify_mesh_casts`` applies ``dist_fn`` in the ``{ [dst -> src] -> [hops] }`` orientation (the docstring previously said - the opposite, though the code itself was never wrong -- D3 was a - documentation-only fix). This pins down the now-correctly-documented + the opposite, though the code itself was never wrong -- the fix was + documentation-only). This pins down the now-correctly-documented behavior with a ``dist_fn`` that is *directionally* asymmetric (cost depends on which side is source vs. destination, not just on distance), so a caller who mis-orients their own ``dist_fn`` -- or a future change diff --git a/tests/isl/distributed/test_edge_pressure.py b/tests/isl/distributed/test_edge_pressure.py index 6df5e1ad..29d5df01 100644 --- a/tests/isl/distributed/test_edge_pressure.py +++ b/tests/isl/distributed/test_edge_pressure.py @@ -5,15 +5,14 @@ per *physical* directed edge: how many multicast trees cross each link. Two models are covered: -- ``XYRoutingMulticastModel``'s ``edge_pressure`` (now reached via - ``model.apply(0, fill, occ).edge_pressure``, not a standalone method -- see - the API-change note on ``test_load_sums_to_hops`` below) -- directed mesh +- ``XYRoutingMulticastModel``'s ``edge_pressure`` (reached via + ``model.apply(0, fill, occ).edge_pressure``) -- directed mesh links (``xedge_r``/``xedge_l``/``yedge_u``/``yedge_d``). ``test_load_sums_to_hops`` checks the per-edge loads sum back to the already-validated A-F hop totals (4/6/6/3/4/448). Bottlenecks and a couple of individual edge loads (hand-derived, geometry documented inline) pin the shape. -- ``StarMulticastModel``'s ``edge_pressure`` (same access-pattern change) -- +- ``StarMulticastModel``'s ``edge_pressure`` -- the spokes realization of a fully-connected fabric (``spoke_in[n]`` ingress, ``spoke_out[n]`` egress). For an N-way all-to-all each node receives N-1 and sources 1, so the ingress spokes are hottest at N-1. Tied to @@ -37,17 +36,9 @@ XYRoutingMulticastModel, StarMulticastModel, FullyConnectedMulticastModel, - # Design: this used to be a local copy (identical signature/behavior) kept - # in sync by hand; now that the source module exposes one module-level, - # `int`-returning `_eval_const` (D6), the test suite imports it directly - # instead of re-deriving the "evaluate a parameter-free PwQPolynomial at - # the space's zero point" idiom a second time. - _eval_const, ) +from accelforge.model._looptree.reuse.isl.distributed.edge_pressure import _eval_const -# `construct_spacetime` and `load_solutions` are hoisted into `helpers.py` -# (single-sourced there, see that module's docstring for the run-from-repo-root -# import requirement) rather than redefined/re-imported locally here. from .helpers import construct_spacetime, load_solutions @@ -68,23 +59,12 @@ def test_load_sums_to_hops(self): """ Summed per-edge load matches the trusted A-F hop totals. - # NOTE (post-D5 API/semantics change): before D5, `XYRoutingMulticastModel` - # computed `hops` via a standalone `_cost_xy` and `edge_pressure` via a - # *second*, independent traversal of `mcns`; this test's "sum(load) == - # hops" check was therefore a genuine cross-implementation invariant -- - # a bug in either path's edge decomposition would surface here even if - # each path's own totals looked locally correct. D5 replaced that with - # one aggregation path: `apply()` now computes `hops = _const_pwq( - # pressure.total())` *from* the same `EdgePressure`, so `pressure.total() - # == info.hops` is true by construction, not by cross-checked - # computation -- the invariant, as originally framed, is now - # tautological. What is *not* tautological is comparing `pressure.total()` - # against the yaml's hand-derived `xy_routing_hops` oracle (an - # independent, human-computed number, unrelated to how the code is - # wired) -- that is what this test still checks, so it keeps its - # regression value as a per-case oracle check even though it can no - # longer catch a hops/edge_pressure disagreement (there structurally - # isn't one to catch anymore). + `MulticastModel.apply` computes `hops = _const_pwq(pressure.total())` + from the same `EdgePressure` it reports, so `pressure.total() == + info.hops` is true by construction. The regression value here is the + comparison against the yaml's hand-derived `xy_routing_hops` oracle -- + an independent, human-computed number, unrelated to how the code is + wired -- checked per case. """ for test in self.testcases: expected = test["expected"]["xy_routing_hops"] From 1fbbb328cfec85dc2749c0185800a84dfa9c6436 Mon Sep 17 00:00:00 2001 From: rengzhengcodes Date: Mon, 7 Sep 2026 08:30:27 +0000 Subject: [PATCH 19/19] Replace in-tree correlation infra with aws-gpu-profiler submodule The EC2 profiling scripts under notebooks/astrasim2_correlation/correlation/ now live in their own repo (rengzhengcodes/aws-gpu-profiler), generalized from an nccl-tests-only orchestrator into a manifest-driven GPU profiler. Wire it in as a submodule at the same path, tracking the profiling-data-20260904 branch, which carries the 2026-09-04 8x H100 all-to-all sweep (run correl-20260904-154601) that the notebook reads. Notebook: read empirical CSVs from correlation/data (inside the submodule) with the repo-root fallback, cite the aws-gpu-profiler run command instead of orchestrate.py, and re-execute in place. Fit unchanged: alpha 74.23 us, BW_eff 339.5 GB/s, R^2 0.9996. Co-Authored-By: Claude Fable 5.1 --- .gitmodules | 4 + notebooks/astrasim2_correlation/correlation | 1 + .../astrasim2_correlation/correlation.ipynb | 1779 ++++++++++++++++- .../correlation/.gitignore | 4 - .../correlation/README.md | 188 -- .../correlation/config.py | 666 ------ .../correlation/data/.gitkeep | 0 .../correlation/orchestrate.py | 935 --------- .../correlation/parse_nccl.py | 483 ----- .../correlation/provision.py | 895 --------- .../correlation/run_profile.sh | 182 -- .../correlation/setup_node.sh | 137 -- .../correlation/teardown.py | 850 -------- .../correlation/tests/__init__.py | 0 .../correlation/tests/test_orchestrate.py | 580 ------ .../correlation/tests/test_parse_nccl.py | 288 --- .../tests/test_provision_teardown.py | 802 -------- .../correlation/torus_bench/Makefile | 27 - .../correlation/torus_bench/torus_bench.cu | 1681 ---------------- 19 files changed, 1688 insertions(+), 7814 deletions(-) create mode 100644 .gitmodules create mode 160000 notebooks/astrasim2_correlation/correlation delete mode 100644 notebooks/astrasim2_correlation/correlation/.gitignore delete mode 100644 notebooks/astrasim2_correlation/correlation/README.md delete mode 100644 notebooks/astrasim2_correlation/correlation/config.py delete mode 100644 notebooks/astrasim2_correlation/correlation/data/.gitkeep delete mode 100644 notebooks/astrasim2_correlation/correlation/orchestrate.py delete mode 100644 notebooks/astrasim2_correlation/correlation/parse_nccl.py delete mode 100644 notebooks/astrasim2_correlation/correlation/provision.py delete mode 100644 notebooks/astrasim2_correlation/correlation/run_profile.sh delete mode 100644 notebooks/astrasim2_correlation/correlation/setup_node.sh delete mode 100644 notebooks/astrasim2_correlation/correlation/teardown.py delete mode 100644 notebooks/astrasim2_correlation/correlation/tests/__init__.py delete mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py delete mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py delete mode 100644 notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py delete mode 100644 notebooks/astrasim2_correlation/correlation/torus_bench/Makefile delete mode 100644 notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..51a2a6c4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "notebooks/astrasim2_correlation/correlation"] + path = notebooks/astrasim2_correlation/correlation + url = git@github.com:rengzhengcodes/aws-gpu-profiler.git + branch = profiling-data-20260904 diff --git a/notebooks/astrasim2_correlation/correlation b/notebooks/astrasim2_correlation/correlation new file mode 160000 index 00000000..0b5c2ff7 --- /dev/null +++ b/notebooks/astrasim2_correlation/correlation @@ -0,0 +1 @@ +Subproject commit 0b5c2ff7f109655020d4fd93cab4cdbac34d695b diff --git a/notebooks/astrasim2_correlation/correlation.ipynb b/notebooks/astrasim2_correlation/correlation.ipynb index 014fd35f..2446800e 100644 --- a/notebooks/astrasim2_correlation/correlation.ipynb +++ b/notebooks/astrasim2_correlation/correlation.ipynb @@ -16,10 +16,10 @@ "id": "e3a04b7c", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:47.222682Z", - "iopub.status.busy": "2026-06-11T18:41:47.222573Z", - "iopub.status.idle": "2026-06-11T18:41:48.167242Z", - "shell.execute_reply": "2026-06-11T18:41:48.166287Z" + "iopub.execute_input": "2026-09-06T20:56:22.030796Z", + "iopub.status.busy": "2026-09-06T20:56:22.030692Z", + "iopub.status.idle": "2026-09-06T20:56:22.743319Z", + "shell.execute_reply": "2026-09-06T20:56:22.741658Z" } }, "outputs": [], @@ -54,10 +54,10 @@ "id": "c5b2c7bf", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:48.169250Z", - "iopub.status.busy": "2026-06-11T18:41:48.169045Z", - "iopub.status.idle": "2026-06-11T18:41:48.177262Z", - "shell.execute_reply": "2026-06-11T18:41:48.176277Z" + "iopub.execute_input": "2026-09-06T20:56:22.745772Z", + "iopub.status.busy": "2026-09-06T20:56:22.745590Z", + "iopub.status.idle": "2026-09-06T20:56:22.754056Z", + "shell.execute_reply": "2026-09-06T20:56:22.752638Z" } }, "outputs": [], @@ -85,10 +85,10 @@ "id": "e343cac3", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:48.178504Z", - "iopub.status.busy": "2026-06-11T18:41:48.178391Z", - "iopub.status.idle": "2026-06-11T18:41:48.183047Z", - "shell.execute_reply": "2026-06-11T18:41:48.182522Z" + "iopub.execute_input": "2026-09-06T20:56:22.756278Z", + "iopub.status.busy": "2026-09-06T20:56:22.756176Z", + "iopub.status.idle": "2026-09-06T20:56:22.761574Z", + "shell.execute_reply": "2026-09-06T20:56:22.760255Z" } }, "outputs": [], @@ -138,14 +138,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "a2a2df5f", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:48.184434Z", - "iopub.status.busy": "2026-06-11T18:41:48.184302Z", - "iopub.status.idle": "2026-06-11T18:41:48.188107Z", - "shell.execute_reply": "2026-06-11T18:41:48.187196Z" + "iopub.execute_input": "2026-09-06T20:56:22.764145Z", + "iopub.status.busy": "2026-09-06T20:56:22.764048Z", + "iopub.status.idle": "2026-09-06T20:56:22.767664Z", + "shell.execute_reply": "2026-09-06T20:56:22.766765Z" } }, "outputs": [], @@ -184,10 +184,10 @@ "id": "06d40ac3", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:48.189890Z", - "iopub.status.busy": "2026-06-11T18:41:48.189717Z", - "iopub.status.idle": "2026-06-11T18:41:48.193103Z", - "shell.execute_reply": "2026-06-11T18:41:48.192370Z" + "iopub.execute_input": "2026-09-06T20:56:22.769403Z", + "iopub.status.busy": "2026-09-06T20:56:22.769325Z", + "iopub.status.idle": "2026-09-06T20:56:22.771339Z", + "shell.execute_reply": "2026-09-06T20:56:22.770817Z" } }, "outputs": [], @@ -205,10 +205,10 @@ "id": "60364739", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:48.194288Z", - "iopub.status.busy": "2026-06-11T18:41:48.194178Z", - "iopub.status.idle": "2026-06-11T18:41:49.888588Z", - "shell.execute_reply": "2026-06-11T18:41:49.888067Z" + "iopub.execute_input": "2026-09-06T20:56:22.773097Z", + "iopub.status.busy": "2026-09-06T20:56:22.773027Z", + "iopub.status.idle": "2026-09-06T20:56:23.944649Z", + "shell.execute_reply": "2026-09-06T20:56:23.943653Z" } }, "outputs": [ @@ -263,14 +263,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "6d5e39c9", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:49.890046Z", - "iopub.status.busy": "2026-06-11T18:41:49.889928Z", - "iopub.status.idle": "2026-06-11T18:41:49.909301Z", - "shell.execute_reply": "2026-06-11T18:41:49.908706Z" + "iopub.execute_input": "2026-09-06T20:56:23.947214Z", + "iopub.status.busy": "2026-09-06T20:56:23.947091Z", + "iopub.status.idle": "2026-09-06T20:56:23.957229Z", + "shell.execute_reply": "2026-09-06T20:56:23.957034Z" } }, "outputs": [ @@ -299,8 +299,8 @@ " collective_size_bytes\n", " per_rank_bytes\n", " per_gpu_wire_bytes\n", - " hops_per_element\n", - " fabric_chunks_per_element\n", + " hops_per_tile_coordinate\n", + " fabric_chunks_per_tile_coordinate\n", " model_latency_s\n", " algbw_GBps\n", " busbw_GBps\n", @@ -457,31 +457,31 @@ "9 8 536870912 67108864 58720256 \n", "10 8 1073741824 134217728 117440512 \n", "\n", - " hops_per_element fabric_chunks_per_element model_latency_s algbw_GBps \\\n", - "0 168 56 7.645867e-07 171.428571 \n", - "1 168 56 1.529173e-06 171.428571 \n", - "2 168 56 3.058347e-06 171.428571 \n", - "3 168 56 6.116693e-06 171.428571 \n", - "4 168 56 1.223339e-05 171.428571 \n", - "5 168 56 2.446677e-05 171.428571 \n", - "6 168 56 4.893355e-05 171.428571 \n", - "7 168 56 9.786709e-05 171.428571 \n", - "8 168 56 1.957342e-04 171.428571 \n", - "9 168 56 3.914684e-04 171.428571 \n", - "10 168 56 7.829367e-04 171.428571 \n", + " hops_per_tile_coordinate fabric_chunks_per_tile_coordinate \\\n", + "0 168 56 \n", + "1 168 56 \n", + "2 168 56 \n", + "3 168 56 \n", + "4 168 56 \n", + "5 168 56 \n", + "6 168 56 \n", + "7 168 56 \n", + "8 168 56 \n", + "9 168 56 \n", + "10 168 56 \n", "\n", - " busbw_GBps \n", - "0 150.0 \n", - "1 150.0 \n", - "2 150.0 \n", - "3 150.0 \n", - "4 150.0 \n", - "5 150.0 \n", - "6 150.0 \n", - "7 150.0 \n", - "8 150.0 \n", - "9 150.0 \n", - "10 150.0 " + " model_latency_s algbw_GBps busbw_GBps \n", + "0 7.645867e-07 171.428571 150.0 \n", + "1 1.529173e-06 171.428571 150.0 \n", + "2 3.058347e-06 171.428571 150.0 \n", + "3 6.116693e-06 171.428571 150.0 \n", + "4 1.223339e-05 171.428571 150.0 \n", + "5 2.446677e-05 171.428571 150.0 \n", + "6 4.893355e-05 171.428571 150.0 \n", + "7 9.786709e-05 171.428571 150.0 \n", + "8 1.957342e-04 171.428571 150.0 \n", + "9 3.914684e-04 171.428571 150.0 \n", + "10 7.829367e-04 171.428571 150.0 " ] }, "execution_count": 7, @@ -538,10 +538,10 @@ "id": "cbdaefb9", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:49.910815Z", - "iopub.status.busy": "2026-06-11T18:41:49.910686Z", - "iopub.status.idle": "2026-06-11T18:41:49.913503Z", - "shell.execute_reply": "2026-06-11T18:41:49.913097Z" + "iopub.execute_input": "2026-09-06T20:56:23.958916Z", + "iopub.status.busy": "2026-09-06T20:56:23.958819Z", + "iopub.status.idle": "2026-09-06T20:56:23.961162Z", + "shell.execute_reply": "2026-09-06T20:56:23.960642Z" } }, "outputs": [ @@ -565,10 +565,10 @@ "id": "11751aab", "metadata": { "execution": { - "iopub.execute_input": "2026-06-11T18:41:49.914710Z", - "iopub.status.busy": "2026-06-11T18:41:49.914592Z", - "iopub.status.idle": "2026-06-11T18:41:50.198575Z", - "shell.execute_reply": "2026-06-11T18:41:50.197933Z" + "iopub.execute_input": "2026-09-06T20:56:23.962824Z", + "iopub.status.busy": "2026-09-06T20:56:23.962750Z", + "iopub.status.idle": "2026-09-06T20:56:24.189484Z", + "shell.execute_reply": "2026-09-06T20:56:24.188129Z" } }, "outputs": [ @@ -598,62 +598,1649 @@ { "cell_type": "markdown", "id": "449a7991", - "source": "## Torus leg: 2×2×2 logical torus (3-cube) all-to-all\n\nGPU $r$ sits at binary coordinates $(g_0, g_1, g_2) \\in \\{0, 1\\}^3$ of a 3-dimensional `noc` space, one coordinate per torus dimension. Rank is recovered row-major with the **last** dimension fastest-varying, $r = 4g_0 + 2g_1 + g_2$ — matching the `torus_bench` benchmark's rank$\\leftrightarrow$coordinate convention.\n\nEvery dimension here has extent 2, so the $\\pm 1 \\pmod 2$ torus wraparound edge coincides with the only other value that coordinate can take: wraparound degenerates exactly to the hypercube edge, and `HypercubeMulticastModel` applies directly. Its precondition — a Manhattan, translation-invariant `dist_fn` — is satisfied by Hamming distance on the binary coordinates (ISL has no `!=`/`<>` operator, so per-axis inequality is written as the disjunction `(a < b or a > b)`).\n\nCost per $(s, d)$ pair at Hamming distance $k$ is $2^k - 1$ (the bounding-box multicast-tree cost `HypercubeMulticastModel` computes), summing to **152** hops over all 64 `data[s, d]` chunks. A dimension-ordered *minimal unicast* routing — one packet per hop instead of a shared multicast tree, which is what the empirical torus benchmark actually implements — would instead traverse $\\sum_{s,d} \\mathrm{hamming}(s, d) = $ **96** hops. The two series bracket real routing behavior; the gap between them is itself a correlation observable, quantifying how far a multicast-aware model diverges from point-to-point unicast.", - "metadata": {} + "metadata": {}, + "source": [ + "## Torus leg: 2×2×2 logical torus (3-cube) all-to-all\n", + "\n", + "GPU $r$ sits at binary coordinates $(g_0, g_1, g_2) \\in \\{0, 1\\}^3$ of a 3-dimensional `noc` space, one coordinate per torus dimension. Rank is recovered row-major with the **last** dimension fastest-varying, $r = 4g_0 + 2g_1 + g_2$ — matching the `torus_bench` benchmark's rank$\\leftrightarrow$coordinate convention.\n", + "\n", + "Every dimension here has extent 2, so the $\\pm 1 \\pmod 2$ torus wraparound edge coincides with the only other value that coordinate can take: wraparound degenerates exactly to the hypercube edge, and `HypercubeMulticastModel` applies directly. Its precondition — a Manhattan, translation-invariant `dist_fn` — is satisfied by Hamming distance on the binary coordinates (ISL has no `!=`/`<>` operator, so per-axis inequality is written as the disjunction `(a < b or a > b)`).\n", + "\n", + "Cost per $(s, d)$ pair at Hamming distance $k$ is $2^k - 1$ (the bounding-box multicast-tree cost `HypercubeMulticastModel` computes), summing to **152** hops over all 64 `data[s, d]` chunks. A dimension-ordered *minimal unicast* routing — one packet per hop instead of a shared multicast tree, which is what the empirical torus benchmark actually implements — would instead traverse $\\sum_{s,d} \\mathrm{hamming}(s, d) = $ **96** hops. The two series bracket real routing behavior; the gap between them is itself a correlation observable, quantifying how far a multicast-aware model diverges from point-to-point unicast." + ] }, { "cell_type": "code", + "execution_count": 10, "id": "d166f6c4", - "source": "def binary_bounds(prefix: str, k: int) -> str:\n \"\"\"Bit constraints over dims ``{prefix}0..{prefix}{k-1}``.\"\"\"\n return \" and \".join(f\"0 <= {prefix}{i} <= 1\" for i in range(k))\n\n\ndef rank_expr(prefix: str, k: int) -> str:\n \"\"\"Affine recovery of the rank from binary torus coordinates.\n\n Row-major with the LAST coordinate fastest-varying:\n ``rank = sum_i 2**(k-1-i) * g_i``, matching the ``torus_bench``\n benchmark's rank<->coordinate convention (see the markdown above).\n \"\"\"\n return \" + \".join(f\"{2 ** (k - 1 - i)}*{prefix}{i}\" for i in range(k))\n\n\ndef torus_all_to_all_maps(\n dims: tuple[int, ...] = (2, 2, 2),\n) -> tuple[isl.Map, isl.Map, isl.Map]:\n \"\"\"Build (occupancy, fill, dist_fn) for a logical-torus all-to-all.\n\n Mirrors ``all_to_all_maps`` above but keys GPUs by binary torus\n coordinates instead of a one-hot id. ``data[s, d]`` is the chunk sent\n by GPU ``s`` to GPU ``d``; each GPU holds the chunks it sends (occ)\n and requests the chunks addressed to it (fill).\n\n Parameters\n ----------\n dims : tuple of int, default (2, 2, 2)\n Per-dimension extents of the logical torus. Every entry must\n equal 2 (see Raises); ``K = len(dims)`` sets the dimensionality\n of the ``noc`` space and the number of ``SpatialTag`` axes the\n caller must build to match.\n\n Returns\n -------\n occ : isl.Map\n ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n holds the chunk it is the source of, for every destination ``d``.\n fill : isl.Map\n ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n requests the chunk addressed to it, for every source ``s``.\n dist_fn : isl.Map\n Piecewise Hamming distance between two ``noc`` points, encoded as\n a disjunction over the ``2**K`` equal/differ patterns per axis.\n\n Raises\n ------\n ValueError\n If any entry of ``dims`` is not 2. General (non-power-of-two)\n torus extents need a genuine wraparound-aware Manhattan distance\n function, not the hypercube-degeneracy shortcut used here — see\n the markdown cell above for why extent 2 is special.\n \"\"\"\n if any(d != 2 for d in dims):\n raise ValueError(\n f\"torus_all_to_all_maps only supports extent-2 dims (hypercube \"\n f\"wraparound degeneracy), got dims={dims!r}; general torus \"\n f\"extents need a wraparound-aware Manhattan dist_fn\"\n )\n k = len(dims)\n n = 2 ** k\n\n gs = \", \".join(f\"gs{i}\" for i in range(k))\n gd = \", \".join(f\"gd{i}\" for i in range(k))\n occ = isl.Map.read_from_str(\n CTX,\n f\"{{ noc[{gs}] -> data[s, d] : {binary_bounds('gs', k)} \"\n f\"and s = {rank_expr('gs', k)} and 0 <= d < {n} }}\",\n )\n fill = isl.Map.read_from_str(\n CTX,\n f\"{{ noc[{gd}] -> data[s, d] : {binary_bounds('gd', k)} \"\n f\"and d = {rank_expr('gd', k)} and 0 <= s < {n} }}\",\n )\n\n # Piecewise Hamming distance: one disjunct per subset of axes that\n # differ (2**K sign patterns), cost = popcount of the subset. ISL has\n # no `!=`, so per-axis inequality is the disjunction (a < b or a > b).\n pieces = []\n for mask in range(2 ** k):\n dist = bin(mask).count(\"1\")\n conds = [\n f\"(gd{i} < gs{i} or gd{i} > gs{i})\" if mask & (1 << i) else f\"gd{i} = gs{i}\"\n for i in range(k)\n ]\n pieces.append(\n f\"[noc[{gd}] -> noc[{gs}]] -> hops[{dist}] : \" + \" and \".join(conds)\n )\n dist_fn = isl.Map.read_from_str(CTX, \"{ \" + \"; \".join(pieces) + \" }\")\n\n return occ, fill, dist_fn", - "metadata": {}, - "execution_count": null, - "outputs": [] + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:24.191947Z", + "iopub.status.busy": "2026-09-06T20:56:24.191854Z", + "iopub.status.idle": "2026-09-06T20:56:24.197412Z", + "shell.execute_reply": "2026-09-06T20:56:24.196135Z" + } + }, + "outputs": [], + "source": [ + "def binary_bounds(prefix: str, k: int) -> str:\n", + " \"\"\"Bit constraints over dims ``{prefix}0..{prefix}{k-1}``.\"\"\"\n", + " return \" and \".join(f\"0 <= {prefix}{i} <= 1\" for i in range(k))\n", + "\n", + "\n", + "def rank_expr(prefix: str, k: int) -> str:\n", + " \"\"\"Affine recovery of the rank from binary torus coordinates.\n", + "\n", + " Row-major with the LAST coordinate fastest-varying:\n", + " ``rank = sum_i 2**(k-1-i) * g_i``, matching the ``torus_bench``\n", + " benchmark's rank<->coordinate convention (see the markdown above).\n", + " \"\"\"\n", + " return \" + \".join(f\"{2 ** (k - 1 - i)}*{prefix}{i}\" for i in range(k))\n", + "\n", + "\n", + "def torus_all_to_all_maps(\n", + " dims: tuple[int, ...] = (2, 2, 2),\n", + ") -> tuple[isl.Map, isl.Map, isl.Map]:\n", + " \"\"\"Build (occupancy, fill, dist_fn) for a logical-torus all-to-all.\n", + "\n", + " Mirrors ``all_to_all_maps`` above but keys GPUs by binary torus\n", + " coordinates instead of a one-hot id. ``data[s, d]`` is the chunk sent\n", + " by GPU ``s`` to GPU ``d``; each GPU holds the chunks it sends (occ)\n", + " and requests the chunks addressed to it (fill).\n", + "\n", + " Parameters\n", + " ----------\n", + " dims : tuple of int, default (2, 2, 2)\n", + " Per-dimension extents of the logical torus. Every entry must\n", + " equal 2 (see Raises); ``K = len(dims)`` sets the dimensionality\n", + " of the ``noc`` space and the number of ``SpatialTag`` axes the\n", + " caller must build to match.\n", + "\n", + " Returns\n", + " -------\n", + " occ : isl.Map\n", + " ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n", + " holds the chunk it is the source of, for every destination ``d``.\n", + " fill : isl.Map\n", + " ``noc[g0..g_{K-1}] -> data[s, d]``: the GPU at ``(g0..g_{K-1})``\n", + " requests the chunk addressed to it, for every source ``s``.\n", + " dist_fn : isl.Map\n", + " Piecewise Hamming distance between two ``noc`` points, encoded as\n", + " a disjunction over the ``2**K`` equal/differ patterns per axis.\n", + "\n", + " Raises\n", + " ------\n", + " ValueError\n", + " If any entry of ``dims`` is not 2. General (non-power-of-two)\n", + " torus extents need a genuine wraparound-aware Manhattan distance\n", + " function, not the hypercube-degeneracy shortcut used here — see\n", + " the markdown cell above for why extent 2 is special.\n", + " \"\"\"\n", + " if any(d != 2 for d in dims):\n", + " raise ValueError(\n", + " f\"torus_all_to_all_maps only supports extent-2 dims (hypercube \"\n", + " f\"wraparound degeneracy), got dims={dims!r}; general torus \"\n", + " f\"extents need a wraparound-aware Manhattan dist_fn\"\n", + " )\n", + " k = len(dims)\n", + " n = 2 ** k\n", + "\n", + " gs = \", \".join(f\"gs{i}\" for i in range(k))\n", + " gd = \", \".join(f\"gd{i}\" for i in range(k))\n", + " occ = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gs}] -> data[s, d] : {binary_bounds('gs', k)} \"\n", + " f\"and s = {rank_expr('gs', k)} and 0 <= d < {n} }}\",\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"{{ noc[{gd}] -> data[s, d] : {binary_bounds('gd', k)} \"\n", + " f\"and d = {rank_expr('gd', k)} and 0 <= s < {n} }}\",\n", + " )\n", + "\n", + " # Piecewise Hamming distance: one disjunct per subset of axes that\n", + " # differ (2**K sign patterns), cost = popcount of the subset. ISL has\n", + " # no `!=`, so per-axis inequality is the disjunction (a < b or a > b).\n", + " pieces = []\n", + " for mask in range(2 ** k):\n", + " dist = bin(mask).count(\"1\")\n", + " conds = [\n", + " f\"(gd{i} < gs{i} or gd{i} > gs{i})\" if mask & (1 << i) else f\"gd{i} = gs{i}\"\n", + " for i in range(k)\n", + " ]\n", + " pieces.append(\n", + " f\"[noc[{gd}] -> noc[{gs}]] -> hops[{dist}] : \" + \" and \".join(conds)\n", + " )\n", + " dist_fn = isl.Map.read_from_str(CTX, \"{ \" + \"; \".join(pieces) + \" }\")\n", + "\n", + " return occ, fill, dist_fn" + ] }, { "cell_type": "code", + "execution_count": 11, "id": "256ea183", - "source": "dims_torus = (2, 2, 2)\nk = len(dims_torus)\nn_t = 2 ** k # = 8 GPUs, same physical count as the FC leg above\n\nocc_t, fill_t, dist_fn_t = torus_all_to_all_maps(dims_torus)\ntags_t = [SpatialTag(i, 0) for i in range(k)]\nmodel_t = HypercubeMulticastModel(dist_fn_t)\n\n\ndef torus_pair_hops(src: int, dst: int) -> int:\n \"\"\"Hops for a single (src, dst) chunk under the torus (Hamming) dist_fn.\"\"\"\n chunk = isl.Set.read_from_str(CTX, f\"{{ data[{src}, {dst}] }}\")\n info = model_t.apply(\n 0,\n Fill(tags_t, fill_t.intersect_range(chunk)),\n Occupancy(tags_t, occ_t.intersect_range(chunk)),\n )\n return eval_total(info.hops)\n\n\ndef hamming(a: int, b: int) -> int:\n \"\"\"Hamming distance between the binary representations of two ints.\"\"\"\n return bin(a ^ b).count(\"1\")\n\n\n# --- Run the tool over the full all-to-all and validate against the\n# verified reference numbers (see torus_hops_check.py pre-verification). ---\ninfo_t = model_t.apply(0, Fill(tags_t, fill_t), Occupancy(tags_t, occ_t))\ntorus_hops = eval_total(info_t.hops)\nassert torus_hops == 152, f\"model returned {torus_hops} total hops, expected 152\"\n\nprobes = {(0, 1): 1, (0, 3): 3, (0, 7): 7, (5, 5): 0}\nfor (src, dst), expected in probes.items():\n got = torus_pair_hops(src, dst)\n assert got == expected, f\"pair ({src}->{dst}) cost {got} != {expected}\"\n\n# Dimension-ordered minimal unicast routing (what the empirical torus\n# benchmark implements): one hop per differing bit, summed over every\n# ordered (s, d) pair. Computed programmatically -- not hardcoded -- so\n# it stays correct if dims_torus ever changes.\nminroute_hops = sum(hamming(s, d) for s in range(n_t) for d in range(n_t))\nassert minroute_hops == 96, f\"minroute_hops {minroute_hops} != 96\"\n\nprint(f\"ISL network model: {n_t}-GPU 2x2x2 logical torus all-to-all (binary coords)\")\nprint(f\" bounding-box multicast hops: {torus_hops} (Sigma 2^hamming(s,d) - 1)\")\nprint(f\" min-route unicast hops : {minroute_hops} (Sigma hamming(s,d))\")\nprint(f\" routing gap : {torus_hops - minroute_hops} hops \"\n f\"({torus_hops / minroute_hops:.2f}x)\")", - "metadata": {}, - "execution_count": null, - "outputs": [] + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:24.199703Z", + "iopub.status.busy": "2026-09-06T20:56:24.199616Z", + "iopub.status.idle": "2026-09-06T20:56:24.344722Z", + "shell.execute_reply": "2026-09-06T20:56:24.344199Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ISL network model: 8-GPU 2x2x2 logical torus all-to-all (binary coords)\n", + " bounding-box multicast hops: 152 (Sigma 2^hamming(s,d) - 1)\n", + " min-route unicast hops : 96 (Sigma hamming(s,d))\n", + " routing gap : 56 hops (1.58x)\n" + ] + } + ], + "source": [ + "dims_torus = (2, 2, 2)\n", + "k = len(dims_torus)\n", + "n_t = 2 ** k # = 8 GPUs, same physical count as the FC leg above\n", + "\n", + "occ_t, fill_t, dist_fn_t = torus_all_to_all_maps(dims_torus)\n", + "tags_t = [SpatialTag(i, 0) for i in range(k)]\n", + "model_t = HypercubeMulticastModel(dist_fn_t)\n", + "\n", + "\n", + "def torus_pair_hops(src: int, dst: int) -> int:\n", + " \"\"\"Hops for a single (src, dst) chunk under the torus (Hamming) dist_fn.\"\"\"\n", + " chunk = isl.Set.read_from_str(CTX, f\"{{ data[{src}, {dst}] }}\")\n", + " info = model_t.apply(\n", + " 0,\n", + " Fill(tags_t, fill_t.intersect_range(chunk)),\n", + " Occupancy(tags_t, occ_t.intersect_range(chunk)),\n", + " )\n", + " return eval_total(info.hops)\n", + "\n", + "\n", + "def hamming(a: int, b: int) -> int:\n", + " \"\"\"Hamming distance between the binary representations of two ints.\"\"\"\n", + " return bin(a ^ b).count(\"1\")\n", + "\n", + "\n", + "# --- Run the tool over the full all-to-all and validate against the\n", + "# verified reference numbers (see torus_hops_check.py pre-verification). ---\n", + "info_t = model_t.apply(0, Fill(tags_t, fill_t), Occupancy(tags_t, occ_t))\n", + "torus_hops = eval_total(info_t.hops)\n", + "assert torus_hops == 152, f\"model returned {torus_hops} total hops, expected 152\"\n", + "\n", + "probes = {(0, 1): 1, (0, 3): 3, (0, 7): 7, (5, 5): 0}\n", + "for (src, dst), expected in probes.items():\n", + " got = torus_pair_hops(src, dst)\n", + " assert got == expected, f\"pair ({src}->{dst}) cost {got} != {expected}\"\n", + "\n", + "# Dimension-ordered minimal unicast routing (what the empirical torus\n", + "# benchmark implements): one hop per differing bit, summed over every\n", + "# ordered (s, d) pair. Computed programmatically -- not hardcoded -- so\n", + "# it stays correct if dims_torus ever changes.\n", + "minroute_hops = sum(hamming(s, d) for s in range(n_t) for d in range(n_t))\n", + "assert minroute_hops == 96, f\"minroute_hops {minroute_hops} != 96\"\n", + "\n", + "print(f\"ISL network model: {n_t}-GPU 2x2x2 logical torus all-to-all (binary coords)\")\n", + "print(f\" bounding-box multicast hops: {torus_hops} (Sigma 2^hamming(s,d) - 1)\")\n", + "print(f\" min-route unicast hops : {minroute_hops} (Sigma hamming(s,d))\")\n", + "print(f\" routing gap : {torus_hops - minroute_hops} hops \"\n", + " f\"({torus_hops / minroute_hops:.2f}x)\")" + ] }, { "cell_type": "code", + "execution_count": 12, "id": "8f3f4946", - "source": "# 8 nodes * 3 distinct neighbors/node. Extent-2 wraparound means the +1\n# and -1 neighbor along a dim coincide (mod 2), so each node has exactly\n# one neighbor per torus dimension, not two -- hence 3, not 6.\nDIRECTED_LINKS_TORUS = n_t * k\n\nrows_t = []\nmib = MIN_MIB\nwhile mib <= MAX_MIB:\n size = mib * (1 << 20)\n total = size * n_t if PER_RANK else size\n per_rank = total / n_t\n chunk_bytes = total / (n_t * n_t)\n # By symmetry, all-to-all traffic loads all 24 directed torus links\n # uniformly: total hop-bytes (chunk_bytes * hops, summed over the 64\n # chunks) divides evenly across the links. Two series bracket\n # routing: `torus_hops` (bounding-box multicast) vs. `minroute_hops`\n # (dimension-ordered unicast) -- see the markdown cell above.\n per_link_bytes_model = (torus_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n per_link_bytes_minroute = (minroute_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n model_latency_s = ALPHA_S + per_link_bytes_model / bw\n minroute_latency_s = ALPHA_S + per_link_bytes_minroute / bw\n # Closed-form cross-check: 152/(64*24) = 19/192, 96/(64*24) = 1/16.\n assert math.isclose(model_latency_s, ALPHA_S + total * 19 / (192 * bw), rel_tol=1e-12)\n assert math.isclose(minroute_latency_s, ALPHA_S + total / (16 * bw), rel_tol=1e-12)\n rows_t.append((n_t, int(total), int(per_rank), per_link_bytes_model,\n model_latency_s, minroute_latency_s))\n mib *= 2\n\nsweep_torus = pd.DataFrame(rows_t, columns=[\n \"nodes\", \"collective_size_bytes\", \"per_rank_bytes\", \"per_link_bytes_model\",\n \"model_latency_s\", \"minroute_latency_s\",\n])\nsweep_torus", - "metadata": {}, - "execution_count": null, - "outputs": [] + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:24.347705Z", + "iopub.status.busy": "2026-09-06T20:56:24.347581Z", + "iopub.status.idle": "2026-09-06T20:56:24.355006Z", + "shell.execute_reply": "2026-09-06T20:56:24.354396Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodescollective_size_bytesper_rank_bytesper_link_bytes_modelmodel_latency_sminroute_latency_s
0810485761310721.037653e+056.917689e-074.369067e-07
1820971522621442.075307e+051.383538e-068.738133e-07
2841943045242884.150613e+052.767076e-061.747627e-06
38838860810485768.301227e+055.534151e-063.495253e-06
481677721620971521.660245e+061.106830e-056.990507e-06
583355443241943043.320491e+062.213660e-051.398101e-05
686710886483886086.640981e+064.427321e-052.796203e-05
78134217728167772161.328196e+078.854642e-055.592405e-05
88268435456335544322.656393e+071.770928e-041.118481e-04
98536870912671088645.312785e+073.541857e-042.236962e-04
10810737418241342177281.062557e+087.083713e-044.473924e-04
\n", + "
" + ], + "text/plain": [ + " nodes collective_size_bytes per_rank_bytes per_link_bytes_model \\\n", + "0 8 1048576 131072 1.037653e+05 \n", + "1 8 2097152 262144 2.075307e+05 \n", + "2 8 4194304 524288 4.150613e+05 \n", + "3 8 8388608 1048576 8.301227e+05 \n", + "4 8 16777216 2097152 1.660245e+06 \n", + "5 8 33554432 4194304 3.320491e+06 \n", + "6 8 67108864 8388608 6.640981e+06 \n", + "7 8 134217728 16777216 1.328196e+07 \n", + "8 8 268435456 33554432 2.656393e+07 \n", + "9 8 536870912 67108864 5.312785e+07 \n", + "10 8 1073741824 134217728 1.062557e+08 \n", + "\n", + " model_latency_s minroute_latency_s \n", + "0 6.917689e-07 4.369067e-07 \n", + "1 1.383538e-06 8.738133e-07 \n", + "2 2.767076e-06 1.747627e-06 \n", + "3 5.534151e-06 3.495253e-06 \n", + "4 1.106830e-05 6.990507e-06 \n", + "5 2.213660e-05 1.398101e-05 \n", + "6 4.427321e-05 2.796203e-05 \n", + "7 8.854642e-05 5.592405e-05 \n", + "8 1.770928e-04 1.118481e-04 \n", + "9 3.541857e-04 2.236962e-04 \n", + "10 7.083713e-04 4.473924e-04 " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 8 nodes * 3 distinct neighbors/node. Extent-2 wraparound means the +1\n", + "# and -1 neighbor along a dim coincide (mod 2), so each node has exactly\n", + "# one neighbor per torus dimension, not two -- hence 3, not 6.\n", + "DIRECTED_LINKS_TORUS = n_t * k\n", + "\n", + "rows_t = []\n", + "mib = MIN_MIB\n", + "while mib <= MAX_MIB:\n", + " size = mib * (1 << 20)\n", + " total = size * n_t if PER_RANK else size\n", + " per_rank = total / n_t\n", + " chunk_bytes = total / (n_t * n_t)\n", + " # By symmetry, all-to-all traffic loads all 24 directed torus links\n", + " # uniformly: total hop-bytes (chunk_bytes * hops, summed over the 64\n", + " # chunks) divides evenly across the links. Two series bracket\n", + " # routing: `torus_hops` (bounding-box multicast) vs. `minroute_hops`\n", + " # (dimension-ordered unicast) -- see the markdown cell above.\n", + " per_link_bytes_model = (torus_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n", + " per_link_bytes_minroute = (minroute_hops / DIRECTED_LINKS_TORUS) * chunk_bytes\n", + " model_latency_s = ALPHA_S + per_link_bytes_model / bw\n", + " minroute_latency_s = ALPHA_S + per_link_bytes_minroute / bw\n", + " # Closed-form cross-check: 152/(64*24) = 19/192, 96/(64*24) = 1/16.\n", + " assert math.isclose(model_latency_s, ALPHA_S + total * 19 / (192 * bw), rel_tol=1e-12)\n", + " assert math.isclose(minroute_latency_s, ALPHA_S + total / (16 * bw), rel_tol=1e-12)\n", + " rows_t.append((n_t, int(total), int(per_rank), per_link_bytes_model,\n", + " model_latency_s, minroute_latency_s))\n", + " mib *= 2\n", + "\n", + "sweep_torus = pd.DataFrame(rows_t, columns=[\n", + " \"nodes\", \"collective_size_bytes\", \"per_rank_bytes\", \"per_link_bytes_model\",\n", + " \"model_latency_s\", \"minroute_latency_s\",\n", + "])\n", + "sweep_torus" + ] }, { "cell_type": "markdown", - "id": "5e209b22", - "source": "## Empirical overlay: EC2 nccl-tests + torus_bench\n\nCSV files under `correlation/data/` (e.g. `correlation/data//csv/*.csv`) are fetched by `correlation/orchestrate.py` (sibling infrastructure, not part of this notebook) and share a unified schema: `source, topology, dims, collective, size_bytes, count, dtype, time_us, algbw_GBps, busbw_GBps, wrong`.\n\nSize normalization to the total collective size $S$ used by `sweep` / `sweep_torus` above differs by source: for `source == \"torus_bench\"` rows, `size_bytes` is **already** the total $S$. For `source == \"nccl-tests\"` rows with `collective == \"alltoall\"`, `size_bytes` is the *per-rank* message size, so $S = N \\times \\texttt{size\\_bytes}$ — verify this against the nccl-tests version actually in use, since the size semantics nccl-tests reports differ per collective. Only `alltoall` is normalized and correlated against the model here; other collectives are inventoried below for later work.", - "metadata": {} + "id": "38679dd0", + "metadata": {}, + "source": [ + "## All-scatter of a matmul operand: tensor sizes and the ISL model\n", + "\n", + "*All-scatter* (every GPU performs its own scatter; also called all-to-all personalized exchange or complete exchange, NCCL's `alltoall`) is the collective that re-shards a matmul operand. For $C = A \\cdot B$ with $A$ of shape $M \\times K$ spread over $N = 8$ GPUs, switching $A$ from **row-sharded** (GPU $s$ holds rows $[s M_b, (s+1) M_b)$ with $M_b = M/N$) to **column-sharded** (GPU $d$ needs columns $[d K_b, (d+1) K_b)$ with $K_b = K/N$) is an all-scatter: every GPU keeps the $M_b \\times K_b$ block on its own diagonal and sends the other $N-1$ blocks of its shard to their column owners.\n", + "\n", + "Below the ISL model is run on **this actual data space** `A[m, k]` (not the abstract per-pair `data[s, d]` used above) with the shard sizes $M_b, K_b$ left as ISL **parameters**, so `HypercubeMulticastModel` returns the hop count as a closed-form polynomial in $(M_b, K_b)$ (an ISL `PwQPolynomial`) that is then evaluated at each matmul shape. Bytes moved: total $S = M K \\cdot \\mathrm{bytes(dtype)}$, per-rank NCCL buffer $S/N$, and each GPU's port drains $S(N-1)/N^2$ bytes.\n", + "\n", + "The profiled node is a `p5.48xlarge` (8x H100 SXM behind NVSwitch), whose NVLink is rated at 450 GB/s per direction per GPU (`NVLINK_H100_GBPS` below). `LINK_BW_GBPS` above (150 GB/s, the V100-era value matching `mapping/v100_torus.arch.yaml`) is left untouched for the earlier sweeps; the empirical fit further down calibrates the effective bandwidth anyway, so the nominal value only positions the reference line." + ] }, { "cell_type": "code", - "id": "4715980a", - "source": "from pathlib import Path\n\n# Design: prefer the path relative to the notebook's own directory (the\n# normal case when the notebook is run in place), but fall back to the\n# repo-root-relative path so `jupyter execute` from a different cwd (e.g.\n# CI running from the repo root) still finds real data when it exists,\n# instead of silently taking the empty-data path below.\nDATA_DIR = (\n Path(\"correlation/data\") if Path(\"correlation\").is_dir()\n else Path(\"notebooks/astrasim2_correlation/correlation/data\")\n)\n\nUNIFIED_CSV_COLUMNS = [\n \"source\", \"topology\", \"dims\", \"collective\", \"size_bytes\", \"count\",\n \"dtype\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\", \"wrong\",\n]\n_NUMERIC_COLUMNS = [\"size_bytes\", \"count\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\"]\n\nfiles = sorted(DATA_DIR.glob(\"**/*.csv\"))\nif files:\n emp = pd.concat(\n [pd.read_csv(f, dtype={\"wrong\": str}) for f in files],\n ignore_index=True,\n )\n for col in _NUMERIC_COLUMNS:\n emp[col] = pd.to_numeric(emp[col], errors=\"coerce\")\nelse:\n # Design: keep the unified schema even when no files were found, so\n # downstream cells can reference emp[...] columns inside their\n # `if HAVE_EMPIRICAL:` guards without ever hitting a KeyError.\n emp = pd.DataFrame(columns=UNIFIED_CSV_COLUMNS)\n\nHAVE_EMPIRICAL = len(files) > 0 and not emp.empty\n\nif not HAVE_EMPIRICAL:\n print(\"no empirical data yet -- run correlation/orchestrate.py; \"\n \"overlay cells below will no-op\")\nelse:\n inventory = emp.groupby([\"topology\", \"collective\"]).agg(\n rows=(\"size_bytes\", \"count\"),\n min_size=(\"size_bytes\", \"min\"),\n max_size=(\"size_bytes\", \"max\"),\n )\n display(inventory)", - "metadata": {}, - "execution_count": null, - "outputs": [] + "execution_count": 13, + "id": "9eed83d8", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:24.356896Z", + "iopub.status.busy": "2026-09-06T20:56:24.356805Z", + "iopub.status.idle": "2026-09-06T20:56:24.359181Z", + "shell.execute_reply": "2026-09-06T20:56:24.358689Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "18 matmul operand shapes, (128, 256) .. (65536, 65536)\n" + ] + } + ], + "source": [ + "NVLINK_H100_GBPS = 450.0 # H100 SXM5 NVLink: 900 GB/s bidirectional = 450 GB/s per direction per GPU\n", + "DTYPE_BYTES = {\"bf16\": 2, \"fp16\": 2, \"fp32\": 4}\n", + "MATMUL_DTYPE = \"bf16\"\n", + "\n", + "# One operand shape per nccl-tests sweep point: M*K = 2**e for e = 15..32, split as\n", + "# M = 2**(e//2), K = 2**(e - e//2) (square for even e, 1:2 rectangular for odd e).\n", + "# Total all-scatter bytes S = M*K*2 then runs 64 KiB .. 8 GiB, i.e. per-rank 8 KiB .. 1 GiB,\n", + "# which is exactly the -f 2 sweep produced by\n", + "# aws-gpu-profiler run --workload nccl-tests --param collective=alltoall --param min_bytes=8192 --param max_bytes=1073741824 --data-dir correlation/data\n", + "MATMUL_SHAPES = [(2 ** (e // 2), 2 ** (e - e // 2)) for e in range(15, 33)]\n", + "print(f\"{len(MATMUL_SHAPES)} matmul operand shapes, {MATMUL_SHAPES[0]} .. {MATMUL_SHAPES[-1]}\")" + ] }, { "cell_type": "code", - "id": "a2e080ce", - "source": "fig, ax = plt.subplots(figsize=(7, 5))\n\nax.loglog(sweep[\"collective_size_bytes\"], sweep[\"model_latency_s\"] * 1e6, \"o-\",\n label=f\"FC model ({LINK_BW_GBPS:g} GB/s links)\")\nax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"model_latency_s\"] * 1e6, \"s-\",\n label=\"torus model, 152 hops (bounding-box multicast)\")\nax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"minroute_latency_s\"] * 1e6, \"s--\",\n label=\"torus min-route, 96 hops (dimension-ordered unicast)\")\n\nif HAVE_EMPIRICAL:\n alltoall = emp[emp[\"collective\"] == \"alltoall\"]\n fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n if not fc_rows.empty:\n # nccl-tests alltoall size_bytes is the PER-RANK message size (see\n # markdown above); scale to total S to match `sweep`'s convention.\n ax.scatter(fc_rows[\"size_bytes\"] * NODES, fc_rows[\"time_us\"],\n marker=\"x\", color=\"tab:red\", label=\"EC2 nccl-tests alltoall (fc)\")\n if not torus_rows.empty:\n # torus_bench size_bytes is ALREADY total collective size S.\n ax.scatter(torus_rows[\"size_bytes\"], torus_rows[\"time_us\"],\n marker=\"+\", color=\"tab:green\", label=\"EC2 torus_bench alltoall (torus)\")\n\nax.set_xlabel(\"total collective size (bytes)\")\nax.set_ylabel(\"all-to-all latency (µs)\")\nax.set_title(\"Fully-connected vs. 2x2x2 torus all-to-all: model vs. empirical\")\nax.grid(True, which=\"both\", alpha=0.3)\nax.legend()\nplt.show()", - "metadata": {}, - "execution_count": null, - "outputs": [] + "execution_count": 14, + "id": "65b7418e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:24.361292Z", + "iopub.status.busy": "2026-09-06T20:56:24.361220Z", + "iopub.status.idle": "2026-09-06T20:56:38.286959Z", + "shell.execute_reply": "2026-09-06T20:56:38.285091Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ISL model hop count for the matmul re-shard, parametric in the shard size:\n", + " [Mb, Kb] -> { 168 * Mb * Kb : Mb > 0 and Kb > 0 }\n", + " matches 3 * (N-1)/N * M*K = 168 * Mb * Kb at every probe\n" + ] + } + ], + "source": [ + "def matmul_reshard_maps(n: int) -> tuple[isl.Map, isl.Map]:\n", + " \"\"\"Build (occupancy, fill) for re-sharding A[m, k] from row blocks to column blocks.\n", + "\n", + " GPU ``s`` (one-hot coordinate e_s) holds rows ``[s*Mb, (s+1)*Mb)`` of ``A``\n", + " (occupancy); GPU ``d`` requests columns ``[d*Kb, (d+1)*Kb)`` (fill). ``Mb``\n", + " and ``Kb`` are ISL *parameters*, so the model's hop count comes back as a\n", + " quasi-polynomial in them.\n", + "\n", + " Parameters\n", + " ----------\n", + " n : int\n", + " Number of GPUs on the fully-connected fabric (one one-hot dimension\n", + " each).\n", + "\n", + " Returns\n", + " -------\n", + " occ : isl.Map\n", + " ``[Mb, Kb] -> { noc[g0..g_{n-1}] -> A[m, k] }``, the rows each GPU holds.\n", + " fill : isl.Map\n", + " ``[Mb, Kb] -> { noc[g0..g_{n-1}] -> A[m, k] }``, the columns each GPU needs.\n", + "\n", + " Notes\n", + " -----\n", + " Ownership is written as a disjunction over the ``n`` constant GPU ids\n", + " (``gs_i = 1 and i*Mb <= m < (i+1)*Mb``) because ``id * Mb`` with ``id`` a\n", + " variable would be a bilinear term, which ISL rejects.\n", + " \"\"\"\n", + " gs = \", \".join(f\"gs{i}\" for i in range(n))\n", + " gd = \", \".join(f\"gd{i}\" for i in range(n))\n", + " own_rows = \" or \".join(f\"(gs{i} = 1 and {i}*Mb <= m < {i + 1}*Mb)\" for i in range(n))\n", + " own_cols = \" or \".join(f\"(gd{i} = 1 and {i}*Kb <= k < {i + 1}*Kb)\" for i in range(n))\n", + " occ = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"[Mb, Kb] -> {{ noc[{gs}] -> A[m, k] : Mb >= 1 and Kb >= 1 and \"\n", + " f\"{onehot_constraints('gs', n)} and 0 <= k < {n}*Kb and ({own_rows}) }}\",\n", + " )\n", + " fill = isl.Map.read_from_str(\n", + " CTX,\n", + " f\"[Mb, Kb] -> {{ noc[{gd}] -> A[m, k] : Mb >= 1 and Kb >= 1 and \"\n", + " f\"{onehot_constraints('gd', n)} and 0 <= m < {n}*Mb and ({own_cols}) }}\",\n", + " )\n", + " return occ, fill\n", + "\n", + "\n", + "def eval_params(poly: isl.PwQPolynomial, **params: int) -> int:\n", + " \"\"\"Evaluate a parametric, fully-summed PwQPolynomial at integer parameter values.\"\"\"\n", + " space = poly.domain().get_space()\n", + " point = isl.Point.zero(space)\n", + " for name, value in params.items():\n", + " idx = space.find_dim_by_name(isl.dim_type.param, name)\n", + " point = point.set_coordinate_val(\n", + " isl.dim_type.param, idx, isl.Val.int_from_si(CTX, value)\n", + " )\n", + " return int(poly.eval(point).to_python())\n", + "\n", + "\n", + "occ_mm, fill_mm = matmul_reshard_maps(n)\n", + "_, _, dist_fn_fc = all_to_all_maps(n) # same unit-cost dist_fn as the FC leg above\n", + "tags_mm = [SpatialTag(i, 0) for i in range(n)]\n", + "info_mm = HypercubeMulticastModel(dist_fn_fc).apply(\n", + " 0, Fill(tags_mm, fill_mm), Occupancy(tags_mm, occ_mm)\n", + ")\n", + "assert info_mm.unfulfilled_fill.map_.is_empty(), \"every requested block must have a holder\"\n", + "hops_mm = info_mm.hops\n", + "print(\"ISL model hop count for the matmul re-shard, parametric in the shard size:\")\n", + "print(\" \", hops_mm)\n", + "\n", + "# Closed form: every element whose row owner differs from its column owner crosses the\n", + "# fabric at the uniform 3-hop FC cost -> 3 * (N-1)/N * M*K = 168 * Mb * Kb for N = 8.\n", + "HOPS_PER_SHARD_ELEMENT = 3 * (n - 1) * n\n", + "for Mb, Kb in [(1, 1), (2, 2), (128, 128), (512, 2048)]:\n", + " got = eval_params(hops_mm, Mb=Mb, Kb=Kb)\n", + " expected = HOPS_PER_SHARD_ELEMENT * Mb * Kb\n", + " assert got == expected, f\"hops(Mb={Mb}, Kb={Kb}) = {got} != {expected}\"\n", + "print(f\" matches 3 * (N-1)/N * M*K = {HOPS_PER_SHARD_ELEMENT} * Mb * Kb at every probe\")" + ] }, { "cell_type": "code", - "id": "a8ec5d63", - "source": "import numpy as np\n\n\ndef linear_fit(S: np.ndarray, t: np.ndarray) -> tuple[float, float, float]:\n \"\"\"Ordinary-least-squares fit of ``t = intercept + slope * S``.\n\n Parameters\n ----------\n S : np.ndarray\n Total collective size in bytes for each empirical sample.\n t : np.ndarray\n Measured latency in seconds for each empirical sample. Must be\n the same length as `S`.\n\n Returns\n -------\n intercept : float\n Fitted per-operation latency overhead (the model's alpha), seconds.\n slope : float\n Fitted dt/dS, seconds/byte.\n r_squared : float\n Coefficient of determination of the linear fit; NaN if `t` has\n zero total variance (R^2 is undefined in that degenerate case).\n \"\"\"\n slope, intercept = np.polyfit(S, t, 1)\n t_pred = slope * S + intercept\n ss_res = float(np.sum((t - t_pred) ** 2))\n ss_tot = float(np.sum((t - np.mean(t)) ** 2))\n r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else float(\"nan\")\n return float(intercept), float(slope), r_squared\n\n\ndef bw_from_slope(slope: float, slope_factor: float) -> float:\n \"\"\"Translate a fitted t-vs-S slope into an effective bandwidth, bytes/s.\n\n Given the model form ``t = alpha + S * slope_factor / BW``, the\n fitted slope is ``dt/dS = slope_factor / BW``, so\n ``BW = slope_factor / slope``.\n \"\"\"\n return slope_factor / slope\n\n\nif HAVE_EMPIRICAL and (emp[\"collective\"] == \"alltoall\").any():\n alltoall = emp[emp[\"collective\"] == \"alltoall\"].dropna(\n subset=[\"size_bytes\", \"time_us\"]\n )\n\n fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n if len(fc_rows) >= 2:\n # fc slope_factor = (N-1)/N^2 = 7/64 (N=8), from the closed form\n # in the \"Size and latency conventions\" cell above.\n S_fc = fc_rows[\"size_bytes\"].to_numpy(dtype=float) * NODES\n t_fc = fc_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n alpha_fc, slope_fc, r2_fc = linear_fit(S_fc, t_fc)\n bw_fc = bw_from_slope(slope_fc, 7 / 64)\n print(f\"FC calibration : alpha = {alpha_fc * 1e6:.3f} us, \"\n f\"BW_eff = {bw_fc / 1e9:.2f} GB/s, R^2 = {r2_fc:.4f}\")\n else:\n print(\"FC calibration : fewer than 2 alltoall rows, skipping fit\")\n\n torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n if len(torus_rows) >= 2:\n # torus slope_factor = hops/(64*24) = hops/1536; report BW under\n # both routing hypotheses (h=152 bounding-box, h=96 min-route)\n # from a single fit, since alpha/R^2 don't depend on slope_factor\n # -- only the slope-to-BW translation does.\n S_t = torus_rows[\"size_bytes\"].to_numpy(dtype=float)\n t_t = torus_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n alpha_t, slope_t, r2_t = linear_fit(S_t, t_t)\n bw_t_152 = bw_from_slope(slope_t, 152 / 1536)\n bw_t_96 = bw_from_slope(slope_t, 96 / 1536)\n print(f\"torus calibration : alpha = {alpha_t * 1e6:.3f} us, R^2 = {r2_t:.4f}\")\n print(f\" BW_eff @ h=152 (bounding-box model): {bw_t_152 / 1e9:.2f} GB/s\")\n print(f\" BW_eff @ h=96 (dimension-ordered) : {bw_t_96 / 1e9:.2f} GB/s\")\n else:\n print(\"torus calibration : fewer than 2 alltoall rows, skipping fit\")\nelse:\n print(\"calibration: no empirical alltoall rows yet\")", - "metadata": {}, - "execution_count": null, - "outputs": [] + "execution_count": 15, + "id": "55ff9b48", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.289471Z", + "iopub.status.busy": "2026-09-06T20:56:38.289351Z", + "iopub.status.idle": "2026-09-06T20:56:38.297942Z", + "shell.execute_reply": "2026-09-06T20:56:38.297028Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
MKdtypeelementscollective_size_bytesper_rank_bytesmodel_hopsper_gpu_wire_bytesmodel_latency_us
0128256bf16327686553681928601671680.015929
1256256bf166553613107216384172032143360.031858
2256512bf1613107226214432768344064286720.063716
3512512bf1626214452428865536688128573440.127431
45121024bf16524288104857613107213762561146880.254862
510241024bf161048576209715226214427525122293760.509724
610242048bf162097152419430452428855050244587521.019449
720482048bf16419430483886081048576110100489175042.038898
820484096bf1683886081677721620971522202009618350084.077796
940964096bf16167772163355443241943044404019236700168.155591
1040968192bf163355443267108864838860888080384734003216.311182
1181928192bf1667108864134217728167772161761607681468006432.622364
12819216384bf16134217728268435456335544323523215362936012865.244729
131638416384bf162684354565368709126710886470464307258720256130.489458
141638432768bf1653687091210737418241342177281409286144117440512260.978916
153276832768bf16107374182421474836482684354562818572288234881024521.957831
163276865536bf162147483648429496729653687091256371445764697620481043.915662
176553665536bf16429496729685899345921073741824112742891529395240962087.831324
\n", + "
" + ], + "text/plain": [ + " M K dtype elements collective_size_bytes per_rank_bytes \\\n", + "0 128 256 bf16 32768 65536 8192 \n", + "1 256 256 bf16 65536 131072 16384 \n", + "2 256 512 bf16 131072 262144 32768 \n", + "3 512 512 bf16 262144 524288 65536 \n", + "4 512 1024 bf16 524288 1048576 131072 \n", + "5 1024 1024 bf16 1048576 2097152 262144 \n", + "6 1024 2048 bf16 2097152 4194304 524288 \n", + "7 2048 2048 bf16 4194304 8388608 1048576 \n", + "8 2048 4096 bf16 8388608 16777216 2097152 \n", + "9 4096 4096 bf16 16777216 33554432 4194304 \n", + "10 4096 8192 bf16 33554432 67108864 8388608 \n", + "11 8192 8192 bf16 67108864 134217728 16777216 \n", + "12 8192 16384 bf16 134217728 268435456 33554432 \n", + "13 16384 16384 bf16 268435456 536870912 67108864 \n", + "14 16384 32768 bf16 536870912 1073741824 134217728 \n", + "15 32768 32768 bf16 1073741824 2147483648 268435456 \n", + "16 32768 65536 bf16 2147483648 4294967296 536870912 \n", + "17 65536 65536 bf16 4294967296 8589934592 1073741824 \n", + "\n", + " model_hops per_gpu_wire_bytes model_latency_us \n", + "0 86016 7168 0.015929 \n", + "1 172032 14336 0.031858 \n", + "2 344064 28672 0.063716 \n", + "3 688128 57344 0.127431 \n", + "4 1376256 114688 0.254862 \n", + "5 2752512 229376 0.509724 \n", + "6 5505024 458752 1.019449 \n", + "7 11010048 917504 2.038898 \n", + "8 22020096 1835008 4.077796 \n", + "9 44040192 3670016 8.155591 \n", + "10 88080384 7340032 16.311182 \n", + "11 176160768 14680064 32.622364 \n", + "12 352321536 29360128 65.244729 \n", + "13 704643072 58720256 130.489458 \n", + "14 1409286144 117440512 260.978916 \n", + "15 2818572288 234881024 521.957831 \n", + "16 5637144576 469762048 1043.915662 \n", + "17 11274289152 939524096 2087.831324 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bw_h100 = NVLINK_H100_GBPS * 1e9 # bytes/s\n", + "dtype_bytes = DTYPE_BYTES[MATMUL_DTYPE]\n", + "\n", + "rows_mm = []\n", + "for M, K in MATMUL_SHAPES:\n", + " assert M % n == 0 and K % n == 0, f\"{M}x{K} does not shard evenly over {n} GPUs\"\n", + " Mb, Kb = M // n, K // n\n", + " hops = eval_params(hops_mm, Mb=Mb, Kb=Kb)\n", + " elements = M * K\n", + " total_bytes = elements * dtype_bytes # S\n", + " crossing_elements = hops // 3 # uniform 3 hops per fabric-crossing element\n", + " per_gpu_wire_bytes = (crossing_elements // n) * dtype_bytes # (N-1)/N of each GPU's shard\n", + " t_model = ALPHA_S + per_gpu_wire_bytes / bw_h100\n", + " # Closed-form cross-check: t = alpha + S (N-1) / (N^2 BW)\n", + " assert math.isclose(\n", + " t_model, ALPHA_S + total_bytes * (n - 1) / (n * n * bw_h100), rel_tol=1e-12\n", + " )\n", + " rows_mm.append((M, K, MATMUL_DTYPE, elements, total_bytes, total_bytes // n, hops,\n", + " per_gpu_wire_bytes, t_model * 1e6))\n", + "\n", + "matmul_sweep = pd.DataFrame(rows_mm, columns=[\n", + " \"M\", \"K\", \"dtype\", \"elements\", \"collective_size_bytes\", \"per_rank_bytes\",\n", + " \"model_hops\", \"per_gpu_wire_bytes\", \"model_latency_us\",\n", + "])\n", + "matmul_sweep" + ] + }, + { + "cell_type": "markdown", + "id": "5e209b22", + "metadata": {}, + "source": [ + "## Empirical overlay: EC2 nccl-tests + torus_bench\n", + "\n", + "CSV files under `correlation/data/` (inside the `aws-gpu-profiler` submodule, where fetched results are versioned: `data///results.csv` for new runs; the 2026-09-04 run keeps the older `data//fc/csv/*.csv` layout, and the glob below reads both) are produced by the `aws-gpu-profiler` tool (`aws-gpu-profiler run --workload nccl-tests ... --data-dir correlation/data`, checked out as a git submodule at `correlation/`; not part of this notebook) and share a unified schema: `source, topology, dims, collective, size_bytes, count, dtype, time_us, algbw_GBps, busbw_GBps, wrong`.\n", + "\n", + "Size normalization to the total collective size $S$ used by `sweep` / `sweep_torus` above differs by source: for `source == \"torus_bench\"` rows, `size_bytes` is **already** the total $S$. For `source == \"nccl-tests\"` rows with `collective == \"alltoall\"`, `size_bytes` is the *per-rank* message size, so $S = N \\times \\texttt{size\\_bytes}$ — verify this against the nccl-tests version actually in use, since the size semantics nccl-tests reports differ per collective. Only `alltoall` is normalized and correlated against the model here; other collectives are inventoried below for later work." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "4715980a", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.300045Z", + "iopub.status.busy": "2026-09-06T20:56:38.299960Z", + "iopub.status.idle": "2026-09-06T20:56:38.313710Z", + "shell.execute_reply": "2026-09-06T20:56:38.311945Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
rowsmin_sizemax_size
topologycollective
fcall_gather1881921073741824
all_reduce1881921073741824
alltoall1881921073741824
broadcast1881921073741824
reduce_scatter1881921073741824
sendrecv1881921073741824
\n", + "
" + ], + "text/plain": [ + " rows min_size max_size\n", + "topology collective \n", + "fc all_gather 18 8192 1073741824\n", + " all_reduce 18 8192 1073741824\n", + " alltoall 18 8192 1073741824\n", + " broadcast 18 8192 1073741824\n", + " reduce_scatter 18 8192 1073741824\n", + " sendrecv 18 8192 1073741824" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "# Design: prefer the path relative to the notebook's own directory (the\n", + "# normal case when the notebook is run in place), but fall back to the\n", + "# repo-root-relative path so `jupyter execute` from a different cwd (e.g.\n", + "# CI running from the repo root) still finds real data when it exists,\n", + "# instead of silently taking the empty-data path below.\n", + "DATA_DIR = (\n", + " Path(\"correlation/data\") if Path(\"correlation/data\").is_dir()\n", + " else Path(\"notebooks/astrasim2_correlation/correlation/data\")\n", + ")\n", + "\n", + "UNIFIED_CSV_COLUMNS = [\n", + " \"source\", \"topology\", \"dims\", \"collective\", \"size_bytes\", \"count\",\n", + " \"dtype\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\", \"wrong\",\n", + "]\n", + "_NUMERIC_COLUMNS = [\"size_bytes\", \"count\", \"time_us\", \"algbw_GBps\", \"busbw_GBps\"]\n", + "\n", + "files = sorted(DATA_DIR.glob(\"**/*.csv\"))\n", + "if files:\n", + " emp = pd.concat(\n", + " [pd.read_csv(f, dtype={\"wrong\": str}) for f in files],\n", + " ignore_index=True,\n", + " )\n", + " for col in _NUMERIC_COLUMNS:\n", + " emp[col] = pd.to_numeric(emp[col], errors=\"coerce\")\n", + "else:\n", + " # Design: keep the unified schema even when no files were found, so\n", + " # downstream cells can reference emp[...] columns inside their\n", + " # `if HAVE_EMPIRICAL:` guards without ever hitting a KeyError.\n", + " emp = pd.DataFrame(columns=UNIFIED_CSV_COLUMNS)\n", + "\n", + "HAVE_EMPIRICAL = len(files) > 0 and not emp.empty\n", + "\n", + "if not HAVE_EMPIRICAL:\n", + " print(\"no empirical data yet -- run aws-gpu-profiler run --workload nccl-tests --data-dir correlation/data; \"\n", + " \"overlay cells below will no-op\")\n", + "else:\n", + " inventory = emp.groupby([\"topology\", \"collective\"]).agg(\n", + " rows=(\"size_bytes\", \"count\"),\n", + " min_size=(\"size_bytes\", \"min\"),\n", + " max_size=(\"size_bytes\", \"max\"),\n", + " )\n", + " display(inventory)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "a2e080ce", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.316016Z", + "iopub.status.busy": "2026-09-06T20:56:38.315920Z", + "iopub.status.idle": "2026-09-06T20:56:38.495065Z", + "shell.execute_reply": "2026-09-06T20:56:38.494459Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAHbCAYAAACQmw0xAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXdYFMcbx7/X6L2DICiIAvbeu4IFe4sFsPfegg3sMZbYW2LBlhhjLzH23nsFG4gKCgrS4dr8/iC3v9vrB3dKdD7Psw/sO+++8847O7tzuzOzHEIIAYVCoVAoFAqlRMH92g5QKBQKhUKhUJShnTQKhUKhUCiUEgjtpFEoFAqFQqGUQGgnjUKhUCgUCqUEQjtpFAqFQqFQKCUQ2kmjUCgUCoVCKYHQThqFQqFQKBRKCYR20igUCoVCoVBKILSTRqFQKBQKhVICoZ20Esa5c+fA4XBw7tw5RhYREQEfH5+v5hNFNarqikIxNP/1a8J/3X9VNG3aFE2bNi3SsT4+PoiIiDCoP98y+sRr69at4HA4SEhIMIov0dHR4HA4RrGtDtpJMwCyE0PV9uOPP35t9757FixYgAMHDnxtN74osbGxmDJlCqpWrQpra2u4u7ujXbt2uHXrVpHsSaVSbN26FR06dICXlxcsLS1RsWJFzJs3D/n5+V/FxytXriA6OhqfP38uUv7fA0lJSYiOjsa9e/e+tisUCqUI8L+2A98Sc+bMQZkyZViyihUrfiVvKDIWLFiAbt26oVOnTl/blS/Gb7/9hk2bNqFr164YMWIEMjIysGHDBtStWxfHjx9Hy5Yt9bKXm5uL/v37o27duhg2bBhcXFxw9epVREVF4fTp0zhz5ozevzCL6+OVK1cwe/ZsREREwM7OTq+8vxeSkpIwe/Zs+Pj4oGrVql/bHQpFb+Li4sDl6vY8qV+/fujVqxdMTU2N7NWXg3bSDEibNm1Qs2bNr+0GhYIffvgB0dHRsLKyYmQDBgxAQEAAoqOj9e6kmZiY4PLly6hfvz4jGzx4MHx8fJiOmr42De2jISCEID8/H+bm5l88bwqFoowuHa6cnBxYWlqCx+OBx+N9Aa++HPR15xeCw+EgOjpaSa7v+ARCCHx8fNCxY0eltPz8fNja2mLo0KFa7cTGxqJHjx5wdnaGubk5ypcvj+nTp7N07t69izZt2sDGxgZWVlZo0aIFrl27xtKRveq9fPkyJkyYAGdnZ1haWqJz585ITU1VKmv79u1x6dIl1K5dG2ZmZihbtiy2bdum5N/nz58xbtw4eHl5wdTUFH5+fli0aBGkUilLTyqVYsWKFahUqRLMzMzg7OyMkJAQ5pUZh8NBTk4OYmJimFfQ8vF+9+4dBgwYAFdXV5iamiIoKAibN29W8uft27fo1KkTLC0t4eLigvHjx6OgoEBrnP/66y9wOBycP39eKW3Dhg3gcDh49OgRAOD9+/fo378/PD09YWpqCnd3d3Ts2LFI4ytq1KjB6vwAgKOjIxo1aoSnT58ysqdPn8Lc3BxhYWEs3UuXLoHH42Hq1KkACjtp8h00GZ07d2bsyAgPD4eZmRlLBgDBwcGwt7dHUlKSXj6qIjo6GpMnTwYAlClThqlbWazEYjHmzp0LX19fmJqawsfHB9OmTVOqM9k5+c8//6BmzZowNzfHhg0bkJCQAA6Hg61btyrlrdiWs7KyMG7cOPj4+MDU1BQuLi5o1aoV7ty5o7EMr1+/xogRI1C+fHmYm5vD0dER3bt3N9h4mnPnzqFWrVoAgP79+zMxki/Tnj17UKNGDZibm8PJyQl9+/bFu3fvdLJvaP+Tk5MRGxsLkUikUU9WN0uWLMGaNWtQtmxZWFhYoHXr1njz5g0IIZg7dy48PT1hbm6Ojh07Ii0tTcnO2rVrERQUBFNTU3h4eGDkyJEqX51v3LgRvr6+MDc3R+3atXHx4kWVfhUUFCAqKgp+fn4wNTWFl5cXpkyZotN1Qh6RSAQHBwf0799fKS0zMxNmZmaYNGkSI1u1ahWCgoJgYWEBe3t71KxZE7t27dIrT3l27NjBnBMODg7o1asX3rx5w9Jp2rQpKlasiAcPHqBJkyawsLCAn58f/vrrLwDA+fPnUadOHebecurUKdbxsrFdsvuQjY0NHB0dMXbsWKXhE4r3SNk95/z58xgxYgRcXFzg6enJSlM8B//++280adIE1tbWsLGxQa1atVgxunjxIrp3747SpUszdTd+/Hjk5eUVOY6Ggj5JMyAZGRn4+PEjS+bk5GTQPDgcDvr27Yuff/4ZaWlpcHBwYNIOHz6MzMxM9O3bV6ONBw8eoFGjRhAIBBgyZAh8fHzw8uVLHD58GPPnzwcAPH78GI0aNYKNjQ2mTJkCgUCADRs2oGnTpkwDlGf06NGwt7dHVFQUEhISsHz5cowaNQq7d+9m6b148QLdunXDwIEDER4ejs2bNyMiIgI1atRAUFAQgMJXa02aNMG7d+8wdOhQlC5dGleuXEFkZCSSk5OxfPlyxt7AgQOxdetWtGnTBoMGDYJYLMbFixdx7do11KxZE9u3b8egQYNQu3ZtDBkyBADg6+sLAPjw4QPq1q0LDoeDUaNGwdnZGX///TcGDhyIzMxMjBs3DgCQl5eHFi1aIDExEWPGjIGHhwe2b9+OM2fOaK2vdu3awcrKCn/++SeaNGnCStu9ezeCgoKYV+Jdu3bF48ePMXr0aPj4+CAlJQUnT55EYmKiwQZZv3//nnVOBgQEYO7cuZg8eTK6deuGDh06ICcnBxEREahQoQLmzJmj1R7APs9XrFiBM2fOIDw8HFevXgWPx8OGDRtw4sQJbN++HR4eHnr5qIouXbrg2bNn+P333/HLL78w+s7OzgCAQYMGISYmBt26dcPEiRNx/fp1LFy4EE+fPsX+/ftZtuLi4vDDDz9g6NChGDx4MMqXL68xb0WGDRuGv/76C6NGjUJgYCA+ffqES5cu4enTp6hevbra427evIkrV66gV69e8PT0REJCAtatW4emTZviyZMnsLCw0MsPRQICAjBnzhzMmjULQ4YMQaNGjQCA6Wxv3boV/fv3R61atbBw4UJ8+PABK1aswOXLl3H37l2tr5AN7X9kZCRiYmIQHx+v0/m+c+dOCIVCjB49Gmlpafj555/Ro0cPNG/eHOfOncPUqVPx4sULrFq1CpMmTWL9+IqOjsbs2bPRsmVLDB8+HHFxcVi3bh1u3ryJy5cvQyAQAAA2bdqEoUOHon79+hg3bhxevXqFDh06wMHBAV5eXow9qVSKDh064NKlSxgyZAgCAgLw8OFD/PLLL3j27JleY2IFAgE6d+6Mffv2YcOGDTAxMWHSDhw4gIKCAvTq1QsA8Ouvv2LMmDHo1q0b08F58OABrl+/jt69e+ucp4z58+dj5syZ6NGjBwYNGoTU1FSsWrUKjRs3Vjon0tPT0b59e/Tq1Qvdu3fHunXr0KtXL+zcuRPjxo3DsGHD0Lt3byxevBjdunXDmzdvYG1tzcqvR48e8PHxwcKFC3Ht2jWsXLkS6enpKn+4KzJixAg4Oztj1qxZyMnJUau3detWDBgwAEFBQYiMjISdnR3u3r2L48ePMzHas2cPcnNzMXz4cDg6OuLGjRtYtWoV3r59iz179ugdR4NCKMVmy5YtBIDKTQYAEhUVpXSst7c3CQ8PZ/bPnj1LAJCzZ88ysvDwcOLt7c3sx8XFEQBk3bp1LFsdOnQgPj4+RCqVavS3cePGxNramrx+/Zollz+uU6dOxMTEhLx8+ZKRJSUlEWtra9K4cWOlsrds2ZJ1/Pjx4wmPxyOfP39mlRUAuXDhAiNLSUkhpqamZOLEiYxs7ty5xNLSkjx79ozl348//kh4PB5JTEwkhBBy5swZAoCMGTNGqYzyvlhaWrJiLGPgwIHE3d2dfPz4kSXv1asXsbW1Jbm5uYQQQpYvX04AkD///JPRycnJIX5+fkp1pYoffviBuLi4ELFYzMiSk5MJl8slc+bMIYQQkp6eTgCQxYsXa7RVHC5cuEA4HA6ZOXMmSy6RSEjDhg2Jq6sr+fjxIxk5ciTh8/nk5s2bWm22bNmS2NjYkPT0dJb8n3/+IQDIvHnzyKtXr4iVlRXp1KlTkX1UxeLFiwkAEh8fz5Lfu3ePACCDBg1iySdNmkQAkDNnzjAy2Tl5/Phxlm58fDwBQLZs2aKUr2JbtrW1JSNHjtTqryKy80ueq1evEgBk27ZtjEyXa4I6bt68qbIcQqGQuLi4kIoVK5K8vDxGfuTIEQKAzJo164v7Hx4errI+FZHVjbOzM+v6EhkZSQCQKlWqEJFIxMh/+OEHYmJiQvLz8wkhhdccExMT0rp1ayKRSBi91atXEwBk8+bNhJD/x6hq1aqkoKCA0du4cSMBQJo0acLItm/fTrhcLrl48SLL1/Xr1xMA5PLly4xM8ZqvCln7OXz4MEvetm1bUrZsWWa/Y8eOJCgoSKMtXUlISCA8Ho/Mnz+fJX/48CHh8/kseZMmTQgAsmvXLkYWGxtLABAul0uuXbumVBb5czAqKooAIB06dGDlNWLECAKA3L9/n5Epxkt2z2nYsCHrmiqfJjuHPn/+TKytrUmdOnVY5zkh7HuEqnN54cKFhMPhsO6TMr+/JPR1pwFZs2YNTp48ydqMgb+/P+rUqYOdO3cysrS0NPz999/o06ePxgHcqampuHDhAgYMGIDSpUuz0mTHSSQSnDhxAp06dULZsmWZdHd3d/Tu3RuXLl1CZmYm69ghQ4aw8m3UqBEkEglev37N0gsMDGR+0QOFTz7Kly+PV69eMbI9e/agUaNGsLe3x8ePH5mtZcuWkEgkuHDhAgBg79694HA4iIqKUiqntkHshBDs3bsXoaGhIISw8gkODkZGRgbzuurYsWNwd3dHt27dmOMtLCyYJ3Pa6NmzJ1JSUlhLEPz111+QSqXo2bMnAMDc3BwmJiY4d+4c0tPTdbKrDykpKejduzfKlCmDKVOmsNK4XC62bt2K7OxstGnTBmvXrkVkZKTW8ZULFizAqVOn8NNPPyk9dWndujWGDh2KOXPmoEuXLjAzM8OGDRuK7KM+HDt2DAAwYcIElnzixIkAgKNHj7LkZcqUQXBwcJHzs7Ozw/Xr15nXuLoiP+5NJBLh06dP8PPzg52dndZXpcXl1q1bSElJwYgRI2BmZsbI27VrhwoVKijFSBWG9n/r1q3McA5d6N69O2xtbZl92dP9vn37gs/ns+RCoZB5jXvq1CkIhUKMGzeONSB98ODBsLGxYcoui9GwYcNYT7MiIiJY+QKF16yAgABUqFCBdS1p3rw5AODs2bN6RAJo3rw5nJycWG8i0tPTcfLkSeaaARSee2/fvsXNmzf1sq+Kffv2QSqVokePHqwyuLm5oVy5ckplsLKyYp7oAUD58uVhZ2eHgIAA1psW2f/y13gZI0eOZO2PHj0awP/bsCYGDx6sdfzZyZMnkZWVhR9//JF1ngPse4T8uZyTk4OPHz+ifv36IITg7t27Wn0xJvR1pwGpXbv2F5s4EBYWhlGjRuH169fw9vbGnj17IBKJ0K9fPwCAUChUGofh7OzMNBRNs05TU1ORm5ur8rVPQEAApFIp3rx5w7yeBKDU4bO3twcApQ6Hop5MV17v+fPnePDgAfPqSpGUlBQAwMuXL+Hh4cF65asrqamp+Pz5MzZu3IiNGzdqzOf169fw8/NT6vjp+losJCQEtra22L17N1q0aAGg8FVn1apV4e/vD6BwcOyiRYswceJEuLq6om7dumjfvj3CwsLg5uamd/nkycnJQfv27ZGVlYVLly4pjQMDCl8By8Z5VaxYETNnztRoc/fu3ZgxYwYGDhyI4cOHq9RZsmQJDh48iHv37mHXrl1wcXEplo+68vr1a3C5XPj5+bHkbm5usLOzU/rhoDgjW19+/vlnhIeHw8vLCzVq1EDbtm0RFhbG+oGjiry8PCxcuBBbtmzBu3fvQAhh0jIyMvTyITU1FRKJhNm3srLSGENZDFSdwxUqVMClS5cAFP5gUxxb6uDgABMTE4P6XxQUryWyjpP8a0h5uewao67sJiYmKFu2LJMu+1uuXDmWnkAgUKrb58+f4+nTp1qvWbrC5/PRtWtX7Nq1CwUFBTA1NcW+ffsgEolYnbSpU6fi1KlTqF27Nvz8/NC6dWv07t0bDRo00Cs/WRkIIUrllSF7BSzD09NT6Zpoa2urNf7yKObl6+sLLper07hGXdrty5cvAWhfZSExMRGzZs3CoUOHlPz8EueyJmgn7Ssjf2HVh169emH8+PHYuXMnpk2bhh07dqBmzZrMhefKlSto1qwZ65j4+Phi+6sOdb9o5C/cuupJpVK0atVK7dMUWcemOMgmIPTt2xfh4eEqdSpXrlzsfIDCDlinTp2wf/9+rF27Fh8+fMDly5exYMEClt64ceMQGhqKAwcO4J9//sHMmTOxcOFCnDlzBtWqVStS3kKhEF26dMGDBw/wzz//aLxYnThxAkDhsg2fPn1S2zk8efIkwsLC0K5dO6xfv16tvbt37zI3p4cPH+KHH34oto/6oOuSIKpmcqo7VlV77dGjBxo1aoT9+/fjxIkTWLx4MRYtWoR9+/ahTZs2avMdPXo0tmzZgnHjxqFevXqwtbUFh8NBr169lCbIaKNWrVqszmdUVJTKiUr68ubNG6Wb4dmzZ9G0aVOD+l8U1F1LdL0WGRKpVIpKlSph2bJlKtMVOy660KtXL2zYsAF///03OnXqhD///BMVKlRAlSpVGJ2AgADExcXhyJEjOH78OPbu3Yu1a9di1qxZmD17tt5l4HA4+Pvvv1XGULHTb4z467OMj6FmYEskErRq1QppaWmYOnUqKlSoAEtLS7x79w4RERFf5FzWBO2kfSHs7e2VZg4JhUIkJycXyZ6DgwPatWuHnTt3ok+fPrh8+TJrQH2VKlWUXre6ubnB0tISAJgZhapwdnaGhYUF4uLilNJiY2PB5XKLdNHRFV9fX2RnZ2tdgsHX1xf//POP0gQKRVQ1fGdnZ1hbW0MikWjNx9vbG48ePQIhhGVLVXzU0bNnT8TExOD06dN4+vQpCCGsX8TyZZo4cSImTpyI58+fo2rVqli6dCl27Nihc14ypFIpwsLCcPr0aZUTF+RZv349Tp48ifnz52PhwoUYOnQoDh48qKR3/fp1dO7cGTVr1sSff/7Jeq0kT05ODvr374/AwEDUr18fP//8Mzp37szMNiyKj4qou6B7e3tDKpXi+fPnCAgIYOQfPnzA58+f4e3trdW27EmwYptVfAonw93dHSNGjMCIESOQkpKC6tWrY/78+Ro7aX/99RfCw8OxdOlSRpafn1+kxXl37tzJmokme9KjKUZA4TkseyUnIy4ujkl3c3NTuo7IOgmG9P9LIl92+SdiQqEQ8fHxzPVApvf8+XNWjEQiEeLj41mdJV9fX9y/fx8tWrQw2Ir0jRs3hru7O3bv3o2GDRvizJkzSjPwAcDS0hI9e/ZEz549mR888+fPR2RkpNIrPk34+vqCEIIyZcoY5IewLjx//pz1I+DFixeQSqUGmyglmyT26NEjpSfrMh4+fIhnz54hJiaGNcvdWMOV9IWOSftC+Pr6MmOpZGzcuLHIT9KAwoX7njx5gsmTJ4PH47HGB9jb26Nly5asTbZERePGjbF582YkJiay7Ml+6fB4PLRu3RoHDx5kPXb+8OEDdu3ahYYNG8LGxqbIfmujR48euHr1Kv755x+ltM+fP0MsFgMonA1JCFH5i1H+V5ulpaXSjYPH46Fr167Yu3evyg6r/Cuetm3bIikpiZleDhTOQFX3mlQVLVu2hIODA3bv3o3du3ejdu3arItTbm6u0tRzX19fWFtbs6bw67pMAVD4pGb37t1Yu3YtunTpolYvPj4ekydPRteuXTFt2jQsWbIEhw4dUpph9fTpU7Rr1w4+Pj44cuSIxl+yU6dORWJiImJiYrBs2TL4+PggPDxcaTkCXX1UhewHh2Ldtm3bFgBYP1oAME852rVrp9W2jY0NnJyclNrs2rVrWfsSiUTpdYiLiws8PDy0Lr3A4/GUni6sWrWqSNeEBg0asNq6rPOhLkY1a9aEi4sL1q9fz/Lz77//ZuoZAMzMzJSuI7IOrCH9B/Q7t4tDy5YtYWJigpUrV7L837RpEzIyMpiy16xZE87Ozli/fj2EQiGjt3XrVqV49ujRA+/evcOvv/6qlF9eXp7G2Yfq4HK56NatGw4fPozt27dDLBYr/bD79OkTa9/ExASBgYEghDBxzM3NRWxsrNLKA4p06dIFPB4Ps2fPVqpXQohSXoZgzZo1rP1Vq1YBgMYfN/rQunVrWFtbY+HChUrXV/n7nfy+7P8VK1YYxIfiQp+kfSEGDRqEYcOGoWvXrmjVqhXu37+Pf/75p1hLdLRr1w6Ojo7Ys2cP2rRpo3HMjzwrV65Ew4YNUb16dQwZMgRlypRBQkICjh49ynw+Zt68eTh58iQaNmyIESNGgM/nY8OGDSgoKMDPP/9cZJ91YfLkyTh06BDat2/PLM+Rk5ODhw8f4q+//kJCQgKcnJzQrFkz9OvXDytXrsTz588REhICqVSKixcvolmzZhg1ahSAwvW4Tp06hWXLlsHDwwNlypRBnTp18NNPP+Hs2bOoU6cOBg8ejMDAQKSlpeHOnTs4deoUM6Zv8ODBWL16NcLCwnD79m24u7tj+/btei0xIBAI0KVLF/zxxx/IycnBkiVLWOnPnj1DixYt0KNHDwQGBoLP52P//v348OEDq/Ot6zIFy5cvx9q1a1GvXj1YWFgoPYnr3LkzLC0tQQjBgAEDYG5ujnXr1gEAhg4dir1792Ls2LFo2bIlPDw8kJWVheDgYKSnp2Py5MlKA8t9fX1Rr149AMCZM2ewdu1aREVFMUtQbNmyBU2bNsXMmTOZ80dXH9VRo0YNAMD06dPRq1cvCAQChIaGokqVKggPD8fGjRvx+fNnNGnSBDdu3EBMTAw6deqkNAxAHYMGDcJPP/2EQYMGoWbNmrhw4QKePXvG0snKyoKnpye6deuGKlWqwMrKCqdOncLNmzdZT5hU0b59e2zfvh22trYIDAzE1atXcerUKTg6Ourkny74+vrCzs4O69evh7W1NSwtLVGnTh2UKVMGixYtQv/+/dGkSRP88MMPzBIcPj4+GD9+vFbbhvZf3yU4ioqzszMiIyMxe/ZshISEoEOHDoiLi8PatWtRq1YtZgkjgUCAefPmYejQoWjevDl69uyJ+Ph4bNmyRWlMWr9+/fDnn39i2LBhOHv2LBo0aACJRILY2Fj8+eefzDp8+tKzZ0+sWrUKUVFRqFSpEuvJMFDYCXFzc0ODBg3g6uqKp0+fYvXq1WjXrh2z3MWNGzfQrFkzra/AfX19MW/ePERGRiIhIQGdOnWCtbU14uPjsX//fgwZMoS1PpshiI+PR4cOHRASEoKrV69ix44d6N27N+spZXGwsbHBL7/8gkGDBqFWrVro3bs37O3tcf/+feTm5iImJgYVKlSAr68vJk2ahHfv3sHGxgZ79+41ygSuIvGFZpF+08im/WpaskAikZCpU6cSJycnYmFhQYKDg8mLFy+KtASHPLIpy/JToXXh0aNHpHPnzsTOzo6YmZmR8uXLKy17cOfOHRIcHEysrKyIhYUFadasGbly5YpOZVdVDm9vb9KuXTslX5o0acKazk4IIVlZWSQyMpL4+fkRExMT4uTkROrXr0+WLFlChEIhoycWi8nixYtJhQoViImJCXF2diZt2rQht2/fZnRiY2NJ48aNibm5OQHAiveHDx/IyJEjiZeXFxEIBMTNzY20aNGCbNy4keXP69evSYcOHYiFhQVxcnIiY8eOJcePH9dpCQ4ZJ0+eJAAIh8Mhb968YaXJlr6oUKECsbS0JLa2tqROnTqsZT8I0X2ZApmeuk12/IoVKwgAsnfvXtbxiYmJxMbGhrRt25YQ8v9lD9RtsphmZmYSb29vUr16ddYyCIQULsvC5XLJ1atX9fJRE3PnziWlSpUiXC6XdYxIJCKzZ88mZcqUIQKBgHh5eZHIyEhmGQYZ6s5JQgqn5Q8cOJDY2toSa2tr0qNHD5KSksJagqOgoIBMnjyZVKlShVhbWxNLS0tSpUoVsnbtWq2+p6enk/79+xMnJydiZWVFgoODSWxsbLGvCYocPHiQBAYGEj6fr7QUwu7du0m1atWIqakpcXBwIH369CFv377Vya6h/dd3CQ7F5Wpk+ezZs4clV3eNWr16NalQoQIRCATE1dWVDB8+XGkpGUIIWbt2LSlTpgwxNTUlNWvWJBcuXFB5zRIKhWTRokUkKCiImJqaEnt7e1KjRg0ye/ZskpGRwejpsgSHDKlUSry8vJjlbBTZsGEDady4MXF0dCSmpqbE19eXTJ48mZWfLC6qloBSxd69e0nDhg2JpaUlsbS0JBUqVCAjR44kcXFxjE6TJk1ULv2hrj0BYC1TI1vK4smTJ6Rbt27E2tqa2Nvbk1GjRiktlaFuCQ5V91vFJThkHDp0iNSvX5+Ym5sTGxsbUrt2bfL7778z6U+ePCEtW7YkVlZWxMnJiQwePJjcv39f7dIhXxIOIUYcTUkxOuPHj8emTZvw/v37Yi9+SaFQKBSKsZEtJpyammrwBd+/NeiYtP8w+fn52LFjB7p27Uo7aBQKhUKhfGPQMWn/QVJSUnDq1Cn89ddf+PTpE8aOHfu1XaJQKBQKhWJgaCftP8iTJ0/Qp08fuLi4YOXKlahaterXdolCoVAoFIqBoWPSKBQKhUKhUEogdEwahUKhUCgUSgmEdtIoFAqFQqFQSiDf/Zg0qVSKpKQkWFtbG+xzHhQKhUKhUCiqIIQgKysLHh4e4HI1Pyv77jtpSUlJRv0OJYVCoVAoFIoib968gaenp0ad776TJvt0xps3b2BjYwOpVIrU1FQ4Ozur7eFq01GXro9cUaaLX4bGEHnqa4PGX3s5jGlDV31NevqmFUdmbEpqHdA2YDwbNP7ay2FMG9/DNSgzMxNeXl5M/0MT330nTfaK08bGhumk5efnw8bGRmMD1aSjLl0fuaJMF78MjSHy1NcGjb/2chjThq76mvT0TSuOzNiU1DqgbcB4Nmj8tZfDmDa+p2uQLkOsvvtOmgypVMpshBBIpVKNupp01KXrI1eU6eKXoTFEnvraoPHX7q8xbeiqr0lP37TiyIxNSa0D2gaMZ4PGX7u/xrTxPVyD9Dn+u+2krVmzBmvWrIFEIgEApKamIj8/H1KpFBkZGSCEaPwVpUlHXbo+ckWZLn4ZGkPkqa8NGn/t5TCmDV31Nenpm1YcmbEpqXVA24DxbND4ay+HMW18D9egrKwsnXW/207ayJEjMXLkSGRmZsLW1hbOzs7M604Oh6N1PIImHXXp+sgVZbr4ZWgMkae+Nmj8tZfDmDZ01dekp29acWTGpqTWAW0DxrNB46+9HMa08T1cg8zMzHTW/W47aYpwuVwm6BwOh7UvlUohFAoZXalUCrFYDKFQqLaBqkrXR64o05anMTBEnvra0EWfxt94NnTV16Snb1pxZMampNYBbQPGs1GS4y978/Ol4g8o3w+/hA1d9TXp6ZtWHJm+6HMs7aRpQSgUIj4+HlLp/98hy95JZ2VlqRz4py5dH7miTFuexsAQeeprQxd9Gn/j2dBVX5OevmnFkRmbkloHtA0Yz0ZJj392djbKli0LExMTfUNB+Q9CO2kaIIQgOTkZPB4PXl5eTO+XEAKxWAw+n6+2gapK10euKNOWpzEwRJ762tBFn8bfeDZ01dekp29acWTGpqTWAW0DxrNRkuMvEonw4cMHJCcno3Tp0l+sHihfD9pJ04BYLEZubi48PDxgYWHByOkF0ng2SvIFksZfNz3aSSueDdoGtJfDmDZKcvz5fD5cXFyQlJQEsVgMgUCgbzgo/zG+3Ivt/yCy9//0sTKFQqFQSgKyjpns/kT5tqGdNB2gj5QpFAqFUhKg96PvC9pJo1AoFAqFQimB0E4apUTSv39/dOrUSWf9c+fOgcPh4PPnzxr1zpw5g8DAQPqqQAXnzp0Dl8tlYrh161bY2dnpZaNp06YYN25ckfJfv349QkNDi3QshUKhFBVJVhbE6ekq00QfPkCix+KzhoZ20v5FKlX+LJTsf8UNAOuvtnSxRIqrLz/i0L0kXHv1CRKpdnuKMm15ym8RERHgcDhK2/Pnzxmd5ORkjBo1CmXLloWpqSm8vLwQGhqKU6dO6VxOXTZ9bcjr6xpfbTGTT4+MjMT06dPB5XJBCEFSUhL69OmDwMBA8Hg8jB07VsnGli1bwOVyYWJiAi6XCw6HAzMzM5ZtqVSKmTNnwt3dHebm5mjZsiWePXumtbzJyckYM2YM/Pz8YGZmBldXVzRo0ABr165Fbm4u44OPjw9TjzweDx4eHhgwYADS0tKUbJ47dw5eXl7Fjn+PHj0QFxendx1oylPTuT5gwADcuXMHFy5c0KlNfInta7eBouro2kZ0qRdt7YrG/+vEX/6eZczNEHnpa0NXfU16uqaJMjLwZthwpPyyHAXJySw90adPeB3RH4mDh0CUkWGweEil9LNQWtHls1ASiQRSaeGCgmKxmDlWlgaoHh8gn37iSQrmHYvF+8wCJt3NxhQz2lZAcJCrWnuKMm15yiOVShEcHIxff/2VJXd2doZYLEZCQgKaNm0KW1tbLFy4EBUrVoRIJMLJkycxcuRIPHr0SKdy6oK+NuTjLou9LjZl+2KxGCKRSGWely5dwqtXr9ChQwfGbk5ODhwcHDB16lSsXr0ahBTOqJLPQyqVwsbGBvfu3QOPx2M6S/K+LV68GKtWrcKmTZvg4+OD6OhohISE4P79+2pXl3716hVTD3PmzEHFihVhamqKR48eYdOmTXB1dUXbtm0Z/aioKAwcOBASiQTPnz/HiBEjMHr0aGzdupVl98CBA2jXrp1SObTFX6YnkUggEokgEAjg4OCgcx3I5FKpVG2ems51Ho+HXr16YcWKFahbt67WNvEl+JptQJO+PtcgxfrRVV6ca5ChoPFXjr9IJIJUKsWnT5+MPrtT/n5YnMWES/JnocTp6fhsbYUcU1M8nT0bLuPGge/gAFFaGhJ3/Q6BpSUE1tYQJCWBa2v7xT8LBfKdk5GRQQCQ9PR0IpFIiEgkIklJSUQkEpGcnBzy+PFjkpubS6RSKWsrKChQkimmH3uQRHymHiHeCpvPv9uxB0ka7SnKtOUp28LDw0nHjh3Vprdp04aUKlWKZGVlKaWlpaWpzVNmd968ecTFxYXY2tqS6OhoIhQKycSJE4m9vT0pVaoU2bRpE8vG7du3SbNmzYiZmRlxcHAggwYNIpmZmUy6SCQi48aNI7a2tsTBwYFMmDCBhIWFscogFovJ/PnziY+PDzEzMyOVKlUif/75J5N+5swZAoDxX1WsRowYQbp06aK2vpo0aULGjBmjVPbNmzcTW1tbtfGXSCTEzc2N/Pzzz4wsPT2dmJqakl27dqmth+DgYOLp6amyHmR2ZXl6e3uTZcuWsdLnzJlDAgMDlY7z9fUlx44dI1KplPz5558kKCiIiX2LFi3U5ieL4YcPH1jllqXPmjWLVKlShcTExBBvb29iY2NDevbsSTIyMhgdWQxlfh8+fJjY2NiQ7du3M3nUrFmTWFhYEFtbW1K/fn0SHx/P6J87d46YmJiQnJwcndrEl9gMkae+NnTR1+UaVFx5Ua9BNP7Gi39ubi55/PgxycnJIRKJxKib/P3wS9nQVV+Tnr5puW/fktuDh5BHQRXJs1atSdat2ySudTC53rkLiWsdTPLfvTNYPCQSCUlPTycASEZGhtY+Cn3d+S+yzzzIXmfJ/y/bACBPJEGuUMz6q7jlCsXILhAj+vBjEBV5yWSzDz9BdoFYpT1FGSGE8UHVq0xFP9Xppaen4/jx4xg5ciSsrKyU0u3t7Vl2FP+eOXMGycnJuHDhApYtW4bo6GiEhobCwcEB169fx7BhwzBs2DC8e/cOHA4HOTk5aN++Pezt7XHz5k3s2bMHp0+fxujRo5l8li1bhpiYGGzevBkXL15Eeno69u/fzyrDTz/9hO3bt2P9+vV49OgRxo4di379+uHChQsa/ZUv26VLl1CjRg0luSp9RVl2djbKlSuH0qVLo1OnTnjy5Amjm5CQgPfv36NVq1aMzM7ODnXq1MG1a9dU1kNaWhpOnDihth601WdSUhKOHDmCOnXqsORPnjxBSkoKWrRogffv36N3796IiIjAkydPcO7cOXTp0kXjOaQthi9fvsTBgwdx+PBhHDhwAOfPn8eiRYtU+rpr1y707t0bO3fuRN++fSGRSNC5c2c0btwY9+/fx9WrVzFkyBDWsbVq1YJYLMb169e11smX2AyRp742dNHXpqMuXR+5unOBxr9kxF/+nmXMzRB56WtDV31Nevqkmbq7w2XcWJh4eECcmIg3ffpA/OYNBI6O8N6yGaYeHgaNB/0slJHIE0kQOOsfg9giAN5n5qNS9Amd9B/Pbg0TPbrUR44cgZWVFbPfpk0b7NmzBy9evAAhBBUqVNDT40IcHBywcuVKcLlclC9fHj///DNyc3Mxbdo0AIVjvn766SdcunQJvXr1wq5du5Cfn4+YmBjGn9WrVyM0NBSLFi2Cq6srli9fjsjISHTp0gWEEKxZswYnT55k8iwoKMCCBQtw6tQp1KtXD4QQlC5dGleuXMGGDRvQpEkTnXx//fo13N3d9S5z+fLlsWnTJgQGBiInJwdLly5F/fr18fjxY3h6euL9+/cAAFdXV9Zxrq6uTJoisnooX748S+7k5IT8/HwAwIgRIzB//nwmberUqZgxYwYkEgny8/NRp04dLFu2jHX8wYMHERwcDBMTEyQnJ0MsFqNTp07MmLZKlSrpXX55pFIptm7dCisrK4jFYvTt2xenT59m+QkA69atw6xZs3D48GGmfjIzM5GRkYG2bdvC19cXHA4HAQEBzCtmALCwsICtrS1ev36Nhg0bFstXCoVC0RW+gwPcFy7Emz59GJlDRDgEbm5f0SvaSftmadasGdatW8fsW1paAmAPyi8KQUFBrF8Brq6uqFixIrPP4/Hg6OiIlJQUAMDTp09RuXJlJn8AaNCgAaRSKeLi4mBmZobk5GTUqVOHSefz+ahZsybj64sXL5Cbm4tWrVqxfBEKhahWrZrOvufl5akdH6aJevXqoW7duszq3w0aNEBAQAA2bNiAuXPn6m1PEzdu3IBUKkWfPn1QUFDASps8eTIiIiJACMGbN28wbdo0tGvXDhcuXACPxwNQ2EkbNWoUAKBKlSpo0aIFqlevjuDgYLRu3RrdunWDvb19kf3z8fGBtbU1Uzfu7u5MXcvYu3cvUlJScOnSJdSuXZuROzg4ICIiAu3atUOrVq3QsmVL9OjRA24KF0Fzc3Pk5uYW2UcKhULRF3FaGpIjI1mytK0xcCtTBqYeHl/JK9pJ0wtzAQ9P5gQzv/w1fRLk6otUDNp+V6vNrf1roZaPvdZPgpjxuXotG2FpaQk/Pz8lebly5cDhcBAbG6uzLXkUB6pyOByVMqkes1e0kZ2dDQA4evQoSpUqxYqNPp0uJycnrUt06IJAIEC1atXw4sULAGA6GR8+fGA9qfvw4QOqVq2q0oafnx84HA7i4uJY8rJlywIo7Kio8l9Wp+XKlcPy5ctRr149nD17Fi1btkRycjLu3r2Ldu3aASjsMJ84cQIXL17E6dOnsWrVKkyfPh3Xr19HmTJlilx2eVTVdbVq1XDnzh1s3rwZtWrVYrWRzZs3Y/jw4Th16hR2796NGTNm4MSJE6hZsyajk5aWBmdn5yL5R6FQKPoiev8eqb8sh+nbtzDx8oLHokV49+OPyPn4EYkR/eETsxWCIryFMQR0TJoecDgcWJjwddoa+jnBzdYM6uYScQC425qhUTlnneyp6gwWBQcHBwQHB2PNmjXIyclRSjdEJ0aegIAAPHjwgJXX5cuXmdeltra2cHd3x/Xr15l0sViM27dvM/uBgYEwNTVFYmIi/Pz8WJuXl5fOvlSrVg1Pnz4tdpkkEgkePnzIdMjKlCkDNzc3nD59mtHJzMzE9evXUa9ePZU2HB0d0apVK6xevVplPeiC7OlZXl4eAODw4cOoX78+HBwcGB0Oh4P69etj9uzZuHv3LkxMTJjxfsaibNmyOHHiBA4dOoTRo0crpVerVg2RkZG4cuUKKlasiF27djFpL1++RH5+vl5PSCkUCqWoiN6/R2JEf4g/foTA0xPe22JgUb0aSm/dAr6TE0Rv3+J1WDhEaoauGBvaSTMSPC4HUe0DAUCpoybbjwoNBI/75T/xIVt6pHbt2ti7dy+eP3+Op0+fYuXKlWo7FUWlT58+MDMzQ0REBB49eoSzZ89i9OjR6NevHzOGa+zYsfjpp59w4MABxMbGYvTo0azOorW1NSZNmoTx48cjJiYGL1++xN27d7Fq1SrExMTo7Evr1q1x+fJlJfm9e/dw7949ZGdnIzU1Fffu3cOTJ0+Y9Dlz5uDEiRN49eoV7ty5g759++L169cYNGgQgMKO0Lhx4zBv3jwcOnQIDx8+RFhYGDw8PDQuyLt27VqIxWLUrFkTu3fvxtOnTxEXF4cdO3YgNjaW6YTJyMrKwvv375GcnIwbN25g8uTJcHZ2Rv369QEAhw4dQocOHRj969evY8GCBbh9+zYSExOxb98+pKamIiAgQOeYFRV/f3+cOXMGe/fuZRa3jY+PR2RkJK5du4bXr1/jxIkTeP78OcufixcvomzZsvD19TW6jxQKhcK1tATfwQF8JyeU3rqFeWImcHOD8/hxEHh6gu/gAK7ckJ0vCX3daURCKrphXd/qmH34CZIz8hm5m60ZokIDEVLx6zw+LVu2LO7cuYP58+dj4sSJSE5OhrOzM2rUqMEax2YILCwscOTIEUyaNAm1atWChYUFunbtyhrwLvMhPDwcXC4XERER6Ny5MzIyMhiduXPnwtnZGQsXLsSrV69gZ2eH6tWrMxMWdKFPnz6YOnUq4uLiWBMnqlevzvx/+/Zt7Nq1C97e3nj+/DkAID09HUOGDMH79+9hb2+PGjVq4MqVKwgMDGSOmzJlCnJycjBkyBB8/vwZDRs2xPHjxzW+jvX19cXdu3exYMECREZG4u3btzA1NUVgYCAmTZqE4cOHs/RnzZqFWbNmAShc865WrVo4ceIEHB0dkZOTg9OnT2P58uWMvo2NDS5evIgVK1YgMzMT3t7eWLp0Kdq0aaNzzIpD+fLlcebMGTRt2hQ8Hg9TpkxBXFwctm3bhk+fPsHd3R0jR47E0KFDmVemv//+OwYPHvxF/KNQKBSetTU8N6yHIClJaZIA38EBpWO2gm9lBZ61tUGH8egKhxR3JPl/nMzMTNj+u0CdjY0NpFIpUlJS4OLiAqFQiPj4eJQpU4Z1s9VlTJp8ukRKcCM+DSmZ+XC05KOurzP4PK5Ge4oybXkaA0Pkqa8NXfT1jb+8fNKkScjKysLGjRs16v/X4r9v3z7MmDGD9QRQXxv66GvS0zdNJouLi0OLFi3w7Nkz2NjYaK2TL8G32AZ0lf/X2oChbJT0+MsWJFe8LxkD+fthcRaz1ceGrvqa9PRNK46sKCj2OzRBn6R9AXhcDur5OjKN7Gu84qQU8uOPP2Ljxo2QSqXFamQlDSsrKyxatOhru1EskpOTsW3bNtja2hZ7FjKFQqF8C9BOGuW7ws7ODtOmTftiTwK+FK1bt/7aLhSbli1bfnP1QqFQKMXh23mUQKFQKBQKhfINQTtpFAqFQqFQKCUQ2kmjUCgUCoVCKYHQMWn/IpVKmY0Qwvpftskj21c3wFlduj5yRZm2PI2BIfLU14Yu+jT+xrOhq74mPX3TiiMzNiW1DmgbMJ6N/0L8ZfcoYyJ/P/xSNnTV16Snb1pxZEVBn+O/207amjVrmEVdASA1NRX5+fmQSqXIyMgAIQQSiQRSqZSZ9ixDlgZA7fRrVen6yBVl2vI0BobIU18buujT+BvPhq76mvT0TSuOzNiU1DqgbcB4Nkp6/EUiEaRSKT59+qT0mTZDI38/LM4SHPrY0FVfk56+acWRFYWsrCyddb/bTtrIkSMxcuRIZr0SZ2dnZp00DocDZ2dnCIVCZGVlgc/ng89XDpW2BqIuXR+5oszYjVIVhshTXxu66NP4G8+Grvqa9PRNK47M2JTUOqBtwHg2Smr8AYDL5cLR0fGLrJMmux8Wp5Omjw1d9TXp6ZtWHFlR0KfevttOmiJcLpcJOofDYfY5HA6zySCEMPvqfkWpStdHrijTlqcxMESe+trQRZ/G33g2dNXXpKdvWnFkxqak1gFtA8az8V+Jv/w9y5gYIi99beiqr0lP37TiyPRFn2PpxAHKN03Tpk2Zb0fqwtatW2FnZ2c0fwwNh8PBgQMHvrYbKgkLC8OCBQuYfR8fH9Znq0oa8rFMSEgAh8PBvXv3jJ6vvudoSUeXeo6OjkbVqlW/iD/q6NWrF5YuXfpVfaBQtEE7acYk4w2QdO//W/I9IPl+4d+ke8DnN0bJ9lu76Jd0Lly4gNDQUHh4eKjtNEVERLCeynI4HISEhDDpCQkJGDhwIMqUKQNzc3P4+voiKioKQqHwC5bEcNy/fx/Hjh3DmDFjvrYrRcLLywvJycmoWLHi13blP4+qNjFp0iScPn36i+R/7tw5mJiY4PPnzyz5jBkzMH/+fNY3gimUkgZ93WksMt4C6+sA4gJGxAHAGl3ANwVG3QZsPb+0dzohFAq/qU8nGYucnBxUqVIFAwYMQJcuXdTqhYSEYMuWLcy+qakp839sbCykUik2bNgAPz8/PHr0CIMHD0Z2dvZ/8tf+2rVr0a1bN1hZWX1tV4oEj8eDm5sbCCGsSUMUw2BlZfXVz42KFSvC19cXO3bswMiRI7+qLxSKOugd2FjkfgJHroOmEnEBkPvJoNlGRETg/PnzWLFiBfPEJiEhAQBw/vx51K5dG6ampnB3d8ePP/7IugE1bdoUo0aNwrhx4+Dk5ISQkBAkJCSAy+WyXvt8/vwZHA4H586dAwCkp6ejT58+cHZ2hrm5OcqVK8fqjCjStGlTjB49GuPGjYO9vT1cXV3x66+/IicnBwMGDICDgwPKlSuHv//+m3WczH8zMzOULl1ayf+cnByEhYXB2toapUuXVtm5KSgowKRJk1CqVClYWlqiTp06TDmKSps2bTBv3jx07txZo56pqSnc3NyYzd7enkmTdeBat26NsmXLokOHDpg4caJOrzI/fvyIzp07w8LCAuXKlcOhQ4dY6RcuXECdOnW01vuoUaNgZ2cHd3d3zJw5k7UUwNq1a1GuXDmYmZnB1dUV3bt3V+uPRCLBvn37EBoaqpSWlZWFvn37wsrKCqVKlcKaNWtY6YmJiejYsSOsra3h6OiInj174sOHD0z6wIEDleI8btw4NG3alFWeMWPGYMqUKXBwcICbmxuio6NZxzx//hyNGzeGmZkZAgMDcfLkSVa64uvOc+fOgcPh4PTp06hZsyYsLCxQv359xMXFsY6bN28eXFxcYG1tjUGDBuHHH39EtWrV1MZKhlgsxqhRo5hJTFFRUaz4p6enIywsDPb29rCwsEDbtm3x/PlzAIUz093c3Fivlq9cuQJLS0u1T6tk5fvzzz/RqFEjWFhYoF69enj27Blu3ryJmjVrwtraGqGhoUhNTWWOa9mypdJT+k6dOiEiIkJlPj4+PgCAzp07g8PhMPuqXndu3rwZQUFBzHk6atQoJm3ZsmWoVKkSLC0t4eXlhREjRiA7O5tJf/36NUJDQ2Fvbw9LS0sEBQXh2LFjSEhIQPPmzQEADg4O4HA4LF9DQ0Pxxx9/qPSdQikJ0E6aPhACCHN028R5utkU5+lmT8f1fVasWIF69eph8ODBSE5ORnJyMry8vPDu3Tu0bdsWtWrVwv3797Fu3Tps2rQJ8+bNYx0fExMDExMTXL58GevWrdMpz5kzZ+LJkyf4+++/8fTpU6xbtw5OTk4aj4mJiYGTkxNu3LiB0aNHY/jw4ejevTvq1auH69evo1WrVujXrx9yc3MBgOX/vXv3sGrVKmzevJnl/+TJk3H+/HkcOHAAR48exfnz53Hnzh1WvmPHjsW1a9fwxx9/4MGDB+jevTvatGnD3PCMyblz5+Di4oLy5ctj+PDh+PRJcwc9IyOD1ZFTx+zZs9GjRw88ePAAbdu2RZ8+fZCWlgagMG4dOnRAzZo1tdY7n8/H9evXsXTpUvzyyy/47bffAAC3bt3CmDFjMGfOHMTFxeH48eNo1KiRWn8ePHiAjIwM1KxZUyltyZIlqFy5Mu7cuYMff/wRY8eOZTpIUqkUHTt2RFpaGs6dO4e///4br169Qs+ePbXGQJGYmBhYWlri+vXr+PnnnzFnzhxWPl26dIGJiQmuX7+O9evXY+rUqTrZnT59OpYuXYpbt26Bz+djwIABTNrOnTsxf/58LFq0CLdv30bp0qV1bkOy+N+4cQPLly/HihUrmPgDhT++bt26hUOHDuHq1asghKBjx44QiURwdnbG5s2bER0djVu3biErKwthYWEYMWIEWrRooTHfqKgozJgxA7dv3wafz0efPn0wZcoUrFixAhcuXMDLly8xa9Ysncqgips3bwIAtmzZguTkZGZfkXXr1mHkyJEYMmQIHj58iEOHDsHPz49J53K5WLlyJR4/foyYmBicOXMGU6ZMYdJHjhyJgoICXLhwAQ8fPsSiRYtgZWUFLy8v/PXXXwAKn1YnJydjxYoVzHG1a9fGjRs3UFCg5Qc15YshycqCOD1dZZro/XtI9Fi+4puAfOdkZGQQACQjI4MQQohEIiHJyclEIpGQvLw88uTJE5KXl1eoXJBNSJTNV9mk+VlEKBQSqVSqtUxNmjQhY8eOZcmmTZtGypcvzzp+zZo1xMrKikgkEua4atWqMelSqZQ8e/aMACB3795l5Onp6QQAOXv2LCGEkNDQUNK/f3+VvkilUiW/mzRpQho2bMjsi8ViYmlpSfr168foJyUlEQDk6tWrSv7LdFavXs34n5WVRUxMTMiff/7JpH/8+JGYm5szsUhISCA8Ho+8ffuW5WOLFi3IlClTiFQqJVu2bCG2trZKfqsqhyoAkP379yvJf//9d3Lw4EHy4MEDsn//fhIQEEBq1apFxGKxSjvPnz8nNjY2ZN26dRrzBEBmzJjB7GdnZxMA5O+//yaEEBIZGUn8/f2ZOiZEdb0HBASwYjtlyhQSEBBACCFk7969xMbGhmRmZjI2NMVj3759hMfjsfIkhBBvb28SEhLCOq5nz56kTZs2RCgUkn/++YfweDySmJjI2H/06BEBQG7cuEGkUinp168f6dixI8vu2LFjSZMmTZh9xfOLEEJq1apFpk6dSggh5J9//iF8Pp+8e/eOSf/7779ZdRcfH08AkDt37hChUEjOnDlDAJBTp04xxxw9epQAYK4PderUISNHjmTl26BBA1KlShWN5458/GWxnTRpEhN/WRu8fPkyc0xqaioxNzcnu3fvZmQjRowg/v7+pHfv3qRSpUokKytLKU9ZXF+9ekUAkN9++42Rb9++nQAgp0+fZmTz5s0j5cuXZ/YbN25MxowZw7LZsWNHEh4ezux7e3uTX375hdlX1SaioqJIlSpVmH0PDw8yffp0pdioO8/27NlDHB0dmf1KlSqR6OhopeMJIUzdpaWlKaXdv3+fACAJCQla89TFL23p+sjVXYNyc3PZ9yUjIn8//BI2xJmZ5FXPXuTOoMEkX659EkKIMCmJPG/ZisT36EnEmZka7eqbVhxZUVDsd2iCPkn7Tnj69Cnq1avHmvbdoEEDZGdn4+3bt4ysRo0aetsePnw4/vjjD1StWhVTpkzBlStXtB5TuXJl5n8ejwdHR0dUqlSJkbm6ugIAUlJSdPL/5cuXEAqFqFOnDpPu4OCA8uXLM/sPHz6ERCJB+fLlmTExVlZWOH/+PF69eqV3ufWhV69e6NChAypVqoROnTrhyJEjuHnzpspXre/evUNISAi6deuGgQMHarUtH0tLS0vY2NgwcYuNjUXdunW11ruiTr169fD8+XNIJBK0atUK3t7eKFu2LPr164edO3cyTzhVkZeXB1NTU5VLE9StW5e1X69ePTx9+hRAYR17eXnBy8uLSQ8MDISdnR2joyvyMQEAd3d31rnk5eUFDw8Plh/62nV3dwfw/3M0Li4OtWvXZunL71+8eJF13u3cuZNJU4x/3bp1mfg/ffoUfD6fdW47OjrC39+fFZclS5ZALBZjz5492LFjB2vMoy7lkbU5xXYoK5+xSElJQVJSksanfqdOnUKLFi1QqlQpWFtbo1+/fvj06RNzHo4ZMwbz5s1DgwYNEBUVhQcPHuiUt7m5OQBoPJ8pXw5pTg7EaWkQf/yIxIj+ECUnAwBEycl4HRYO0Zs3EKelQZqT85U9/XLQiQP6ILAApiUxg4n5fL7aNXLE7+5CsK29dpsDjoO4VlKyp5QH3xz4d/VpY2Jpacnal00cIHKvW0UiEUunTZs2eP36NY4dO4aTJ0+iRYsWGDlyJJYsWaI2H8UFGjkcDksmi4MhP3uSnZ0NHo/HvKqSQQgx+qKQipQtWxZOTk548eIF6+aUlJSEZs2aoX79+ti4caNO5VcVS0PGzdraGnfu3MG5c+dw4sQJzJo1C9HR0bhy5YrK19pOTk7Izc2FUCjUqaOgD1wuV+lzOornI2C8mBTnHK1ZsyZrbKesU2QoXr58iaSkJEilUiQkJCAgIEDrMarKoyiTL5+u8dcHWUdJHQkJCQgNDcXw4cMxf/58ODg44NKlSxg4cCCEQiEsLCwwaNAgBAcH4+jRozhx4gQWLlyIpUuXYvTo0Rpty4YFODs7F6sMFMMgcHND6a1bkB0VDVFcHF6HhcNj0SIkTZ0K0Zs3EHh5wXtbDARubkb/JFZJgT5J0wcOBzCx1G3ja77wMPDNdbOnx0KOJiYmzOdEZAQEBDBjWWRcvnwZ1tbW8PRUP7tUdvFK/vcXDQCVa0c5OzsjPDwcO3bswPLly7Fx40ad/dUFbf77+vpCIBDg+vXrTHp6ejqePXvG7FerVg0SiQQpKSnw8/NjbW5ubgb1Vxtv377Fp0+fmKcxQOETtKZNm6JGjRrYsmWLQWbWVqhQAdeuXdNa7/JxA4Br166hXLly4PF4AAA+n4+WLVvi559/xoMHD5CQkICzZ8+qzFM2IPzJkydKaarykXUmAgIC8ObNG7x58/+laZ48eYLPnz8jMDAQQGEHUP5cBFSfj5qQ5SNv59q1a3rZUEX58uWVxlzJ75ubm7POOWtrayZNMS7Xr19n4h8QEACxWMzS+fTpE549e8bERSgUom/fvujZsyfmzp2LwYMHG+UJmJOTE96/f8/sSyQSPHr0SOMxAoFA6Xokj7W1NXx8fNROcrhz5w6kUimWLl2KunXrwt/fH0lJSUp6Xl5eGDZsGPbt24eJEyfi119/BVB4PZT5qsijR4/g6empdQwt5cshcHOD8/hxEHh6QvTmDV737s3uoMldM78HaCftG8THxwfXr19HQkICPn78CKlUihEjRuDNmzcYPXo0YmNjcfDgQURFRWHChAkaOwPm5uaoW7cufvrpJzx9+hTnz5/HjBkzWDqzZs3CwYMH8eLFCzx+/BhHjhzR6Ve8Pij6f+jQIURHRzP+W1lZYeDAgZg8eTLOnDmDR48eoX///qyy+fv744cffkB4eDj27duH+Ph43LhxAwsXLsSxY8eK7Ft2djbu3bvHdBbi4+Nx7949JCYmMumTJ0/GtWvXkJCQgNOnT6Njx47w8/NDcHAwgP930EqXLo0lS5YgNTUV79+/Z90Qi8KIESPw9u1brfWemJiICRMmIC4uDn/88QdWr16NsWPHAgCOHDmClStX4t69e3j9+jW2bdsGqVQKf39/lXk6OzujWrVquHTpklLa5cuXsWTJEjx79gxr1qzBnj17mLXUWrZsiUqVKqFPnz64c+cObt68ifDwcDRp0oSZhNCsWTPcunUL27Ztw/PnzxEVFaW1k6BIy5Yt4e/vj/DwcNy/fx8XL17E9OnT9bKhitGjR2PTpk2IiYnB8+fPMW/ePDx48ECnVe7l4//7779j7dq1TFzKlSuHjh07YvDgwbh06RLu37+Pfv36oVSpUujYsSOAwgkNGRkZWLlyJaZOnQp/f38MGTKk2GVSpFmzZjh69CiOHj2K2NhYDB8+XGn9MUVkHbD3798jXc2A8OjoaCxduhQrV67E8+fPcefOHaxatQoA4OvrC5FIhFWrVuHVq1fYvn071q9fzzp+3Lhx+OeffxAfH487d+7g7NmzzDXI29sbHA4HR44cQWpqKmtW6MWLF9G6detiRIRiDPgODnBfuJAl81i06LvroAG0k2Y8LBxB+Fpe9fBNAQtHg2c9adIk8Hg8BAYGwtnZGYmJiShVqhSOHTuGGzduoEqVKhg2bBgGDhyo1OFSxaZNmyAWi1GjRg2MGzdOaWagiYkJIiMjUblyZTRu3Bg8Hs/g09rl/a9atSpGjRqFAQMGsPxfvHgxGjVqhA4dOqBNmzZo0KCB0hi73377Df369cPEiRNRvnx5dOrUCbdu3WKNg5JHtlSBpmU6bt26hWrVqjFLLUyYMAHVqlVjZsXxeDw8ePAAHTp0gL+/PwYOHIgaNWrg4sWLzOvAkydP4sWLFzh9+jQ8PT3h7u4ODw8PlC5dujhhQ6lSpXDo0CHcvHlTY72HhYUhLy8PderUwdixYzFmzBjmJm9nZ4d9+/ahefPmCAgIwPr167Fr1y4EBQWpzXfAgAHYtWuXknzChAm4ffs2qlevjnnz5mHZsmVMR5XD4eDgwYOwt7dHkyZNEBISgrJly2L37t3M8a1bt8aMGTMwZcoU1KpVi5nJqA9cLhf79+9HXl4eateujUGDBmH+/Pl62VBFnz59EBkZiUmTJqF69eqIj49HRESETq/SZfGvXbs2sxyKfCdry5YtqFGjBtq3b4969eqBEIKDBw9CIBDg3LlzWL58ObZv3w4bGxtwuVxs27YNly5d0nl2qa5EREQgLCwMYWFhaNKkCcqWLYtmzZppPGbp0qU4efIkvLy81C5HEh4ejuXLl2Pt2rUICgpC+/btmRnXVapUwdKlS7Fo0SJUrFgRO3fuxEKFG7hEIsHIkSMREBCAkJAQ+Pv7Y+3atQAK28CsWbMQGRkJV1dXZmmP/Px8HDhwAIMHDy5uWCgGRpyWhuTISJYsaepUZoza9wSHKA4w+M6QfWA9IyOD+cB6SkoKXFxcIBQKER8fjzJlyrAutDqNSROLwc9JBic37f9yEIjFEvD5PHDAKeyg2XmptKco05anMTBEnvra0EVf5/grpOsjl8kuXryIrl274uXLl7C2tv4m49+0aVNUrVoVy5cv1zlPTXqEEGRlZaFixYrYvXs3a1C+Lue6PjJjU9w8W7VqBTc3N2zevPk/2wa+1WvQunXrsH//fpw4cULvPL9m/MViMRISEpTuS8ZA/n5YnA+s62OjICkJT6KiYXr5MkxKlVI9Js3dXaNdfdOKIysKiv0OTdCJA8bE1guwk3sSQgggFgN8vl5jzChfj2PHjmHatGmwt7enK8/rgbm5OWJiYvDx48ev7coXIzc3F+vXr0dwcDB4PB5+//13nDp1SqkTQCkZCAQC5pUqpWQgev8eiRH9Iba0hJWnJ7xjtkLg7g7vbTHM7M7XYeHw3r4NPBeXr+3uF4F20igUDSxevJj5FUvRj6ZNm36xJy4lAQ6Hg2PHjmH+/PnIz89H+fLlsXfvXrRs2ZJ28EsggwYN+touUBTgWlqC7+AAvrU1Sq9ZzYxBk++o8R0cwFVYheBbhnbSKJTvnOJ+FotSiLm5OU6dOqUkpx18CkU3eNbW8NywHoKkJAgUZtwL3N3hvX0buJaW4FlbfzdLcNBOGoVCoVAolBIBz9oafDWfw1PsuH0P0NmdFAqFQqFQKCUQ2kmjUCgUCoVCKYHQThqFQqFQKBRKCeQ/30n7/PkzatasiapVq6JixYrMp0AoFAqFQqFQ/sv85ycOWFtb48KFC7CwsEBOTg4qVqyILl26wNHR8Cv5UygUCoVCoXwp/vNP0ng8HiwsLAAABQUFIITQKe/fIOfOnQOHw9H6nUCK7sg+eaXvB8q/FM2bN1f5aSl5OBwODhw4AKDkl0fG1q1bYWdn97Xd0IuEhARwudyvFttmzZph3LhxRs9H/nz6Gjx58gSenp7Iycn5aj5QShZfvZN24cIFhIaGwsPDQ20DWbNmDXx8fGBmZoY6dergxo0brPTPnz+jSpUq8PT0xOTJk+Hk5PSFvNdMcnYynnx6wtqepj1l/k/ONs53yJo2bfpFLmhfkvr16yM5ORm2trZf2xW1GPMCf/r0adSvXx/W1tZwc3PD1KlTlRZIJYRgyZIl8Pf3h6mpKUqVKmWQb1J+DQ4dOoQPHz6gV69eOh/j5eWF5ORkVKxY0YieFZ+ePXvi2bNnX9sNigqSk5PRpk2bL5KXqut0YGAg6tati2XLln0RHygln6/+ujMnJwdVqlTBgAED0KVLF6X03bt3Y8KECVi/fj3q1KmD5cuXIzg4GHFxcXD597MQdnZ2uH//Pj58+IAuXbqgW7ducHV1/dJFYZGck4zOhztDKBWq1THhmeBIpyNwsyyZa78IhcJifZ/MkJiYmMDt3zVyDPmkVCgUQiAQGMyeMbh//z7atm2L6dOnY9u2bXj37h2GDRsGiUSCJUuWMHpjx47FiRMnsGTJElSqVAlpaWn49OnTV/S86KxatQrh4eF6nX88Ho85R0oy5ubmMDc3/9puqEQoFMLU1NRotk1MTIxi21C4ubkx38j8WvTv3x+DBw/Gjz/++NV8oJQcvvoduE2bNpg3bx46d+6sMn3ZsmUYPHgw+vfvj8DAQKxfvx4WFhbYvHmzkq6rqyuqVKmCixcvqs2voKAAmZmZrA0o/HCqbCOEsP5X3ACw/qpKT89P19hBAwChRIi0/DSV9hRl2vKUbRERETh//jxWrFgBDocDDoeD+Ph4EEJw7tw51K5dG6ampnB3d8fUqVMhEomYY5s2bYqRI0di7NixcHJyQkhICOLj48HlcnH37l1GLz09HRwOB2fPngUhBGlpaejTpw+cnZ1hbm6OcuXKYfPmzWr9btq0KUaNGoWxY8fC3t4erq6u2LhxI7KzszFgwAA4ODigXLlyOHbsGHPM2bNnweFwkJ6eDgCIiYmBvb09jh8/joCAAFhZWSEkJARJSUlqYwUALVu2ZPJ2cnJCcHAwAOD8+fOoU6cOE5sff/wRIpGIsVGmTBmsWLGCZbNq1aqIiooCIQQ+Pj4AgM6dO4PD4cDHx4fRO3DgAKpXrw4zMzOULVsW0dHRrLir22R57d69G5UrV8bMmTPh6+uLxo0bY9GiRVizZg0yMzNBCMGTJ0+wbt06HDhwAKGhofDx8UH16tXRqlUrrefqy5cv0axZM1hYWKBq1aq4evUqS/+vv/5CUFAQTE1N4ePjw3QMZcf7+Phgzpw5+OGHH2BpaQlPT0+sXbuW0ZFKpYiKikLp0qVhZmYGb29vjBkzRu25npKSgjNnzqBdu3YsP54/f47mzZvD3NwcgYGBzDcxZenx8fHgcDjMuSo7Z44fP45q1arB3NwczZs3x4cPH3Ds2DEEBATAxsYGvXv3Rk5ODmNHIpFgwYIFKFOmDCwsLFCjRg389ddfSufiqVOnULNmTVhYWKB+/fqIjY1ldO7du4dmzZrB2toatra2qFOnDm7dugVCCLZs2QI7OztWPaxduxa+vr4wMTFBhQoVsGPHDlbZOBwOfv31V3Tu3BkWFhbw9/fHoUOHNNYrIQQPHz5kYubo6IghQ4YgKyuLSY+IiECnTp0wf/58eHt7o0KFCiCE4Pr166hevTqsra1Rq1Yt3Llzh3Xtktlu06YNrKys4Orqin79+iE1NVXt9SQ4OFin47KzsxEWFgYrKyt4eHgwT5W0Xfc6derEko0dOxZNmzZljm3WrBlGjx6NyZMnw8HBAW5ubkz7lY/zgQMHmBi+efMGP/zwAxwcHGBpaYmaNWvi2rVrAIAXL16gU6dOcHV1hZWVFWrVqoWTJ0+y4r927VqUK1cOZmZmcHV1Rffu3UEIQf/+/VnXaS6Xi/j4eOYalZaWhvPnz2u8B8jfs4y5GSIvfW3oqq9JT9+04siKsunKV3+SpgmhUIjbt28jMjKSkXG5XLRs2ZK5kXz48AEWFhawtrZGRkYGLly4gOHDh6u1uXDhQsyePVtJnpqaivz8fEilUmRkZDAXa6lUCrFYzPpllSvKhUQiAU/MU/o2IZfDhQnXROdKkEgkEIlEkEgkyBPnMfZk+fPEPFgILJh9ABq/h7hkyRLExcUhKCgIUVFRAABnZ2e8fv0a7dq1Q1hYGDZt2oS4uDgMHz4cJiYmmDVrFpPntm3bMGTIEJw7d445GQGwYiD7K5FIIBaLMWPGDDx+/BiHDx+Go6MjXr58iby8PIjFYpV+y/KZOHEiLl++jD179mDEiBHYv38/OnTogAkTJmD16tUICwvDy5cvYWFhwdgQi8UQiUSQSqXIzc3FkiVLsGXLFnC5XISHh2PixImIiYlRGSvZhVi+jEDheJuOHTuiX79+rNjw+XxERUWx/BaJRKx92flx5coVlCpVCr/99htat24NHo8HsViMS5cuITw8HMuWLUPDhg3x6tUrjBgxAlKpFDNnztR4bshil5eXB1NTU9Y5aGJigvz8fNy4cQNNmjTBwYMHUaZMGRw6dAjr1q0DIQTNmzfHggULmFfEiueNzN706dOxaNEi+Pn5YdasWejXrx+ePn0KgUCAO3fuoGfPnpg5cya6d++Oa9euYfTo0bCzs0OfPn1Y593UqVMxY8YMnDhxAhMnTkS5cuXQqlUr7N27F8uXL8eOHTsQEBCApKQkPH78mMlf8Rw5f/48LCwsUK5cOSbeUqkUXbp0gYuLCy5duoTMzExMnDiRdR7Kn59isZixGR0djeXLl8Pc3Bx9+vRBjx49YGpqipiYGOTk5KB79+5YsWIFJk+eDKDwGrFr1y6sXr0avr6+uHDhAvr16wcHBwc0btyYsSuLm5OTE0aNGoUBAwbg/PnzAIA+ffqgatWquHLlCjOWi8PhQCwWs9oUABw4cADjxo3D0qVL0bx5cxw9ehSDBw9GqVKl0KxZMybGc+bMwYIFC7BgwQKsWbMGERERaNCggcpJUoQQZGZmIiQkBHXq1MGVK1eQmpqKoUOHYuzYsdi0aRMT19OnT8PKygqHDx8Gj8fD58+fERoaihYtWuC3335DYmIiJk2axMRaJBIhIyMDLVq0QP/+/fHzzz8jLy8P06dPR48ePVidZ8W29vHjR63HTZo0CefPn8fevXvh7OyMGTNm4M6dO6hcubLaJ1yyG598uqy9y66vMn/Gjh2LS5cu4dq1axg0aBDq1q2Lli1bstqFRCJBdnY2mjZtCg8PD+zbtw+urq64e/cuRCIRRCIRMjMz0bp1a0RHR8PU1BQ7duxAhw4d8OjRI3h5eeHGjRsYO3YstmzZgnr16iEtLQ2XLl2CRCLB4sWLWddpQggcHBwgEonA5XJRpUoVnD9/Hg0aNGDahaydyK5/nz59MvpbAPn7YVHfquhrQ1d9TXr6phVHVhRkP5R0gpQgAJD9+/cz++/evSMAyJUrV1h6kydPJrVr1yaEEHL9+nVSpUoVUrlyZVKpUiWyfv16jXnk5+eTjIwMZnvz5g0BQNLT04lEIiEikYgkJSURkUhEcnJyyOPHj0lubi6RSqXMVnFrRbXb8JPDiVQqJfeS72nUk22PUh8RqVRKCgoKSKPfG6nUkeVbUFDA8kPd1qRJEzJmzBiWLDIykpQvX55IJBJGtnr1amJlZUXEYjFzXLVq1VjHxcXFEQDkzp07jCwtLY0AIGfOnCFSqZSEhoaSiIgItf4o+t2kSRPSsGFDZl8kEhFLS0vSr18/Rj8pKYmpe6lUSs6cOUMAkLS0NCKVSsmvv/5KAJDnz5+zyuPq6qoxVo0bN1YqY2RkJPH399cYG29vb7J48WLWcVWqVCGzZs1i9gGQffv2sXRatGhB5s+fz5Jt27aNuLu761SXBQUF5Pjx44TL5ZKdO3cSkUhE3rx5Qxo1akQAkJ07dxKpVEqGDBlCTE1NSZ06dcj58+fJmTNnSNWqVUmzZs3UxuLVq1cEAPn1118Z2aNHjwgA8uTJEyKVSknv3r1Jq1atWMdNmjSJBAYGMna9vb1JSEgIS6d79+6kTZs2RCqVkiVLlhB/f39GX5U/8rJly5aRsmXLsmTHjx8nfD6fxMfHM7Jjx46xYi4rj+xclZ0zJ0+eZI5ZsGABAUBevHjByIYMGUKCg4OJVColeXl5xMLCgly+fJnl24ABA8gPP/yg1u6RI0cIAOZaYW1tTbZs2aKyfJs3bya2trbMfv369cmgQYNY8ejatStp27Yt69yaPn06s5+VlUUAkGPHjqk9d9auXUvs7e1JVlYWy08ul0uSk5OJVCol4eHhxNXVleTn5zM+rl+/njg6OpLc3FxGtnbtWgKA3Lhxg0ilUjJnzhzSunVrVn6JiYkEAImNjVV7PdF2XGZmJjExMSG7d+9m0pOTk4m5ubnSNU1+Cw8PJx07dmTJxowZQ5o0acLEX/G6I5VKSa1atciUKVOU2nBBQQFZv349sba2Jh8/flTbNhVlQUFBZOXKlUQqlZLdu3cTGxsbkpGRofI4xeu0vL3OnTuTiIgIpTwKCgpIbm4uefz4McnJySESicSom/z98EvZ0FVfk56+acWRFWVLT08nAEhGRobWftFXf91ZXGrXro179+7h/v37ePDgAYYOHapR39TUFDY2NqwNKHxCJ9tkj59l/ytuuqCvniZ9RR1VPin6pyiLjY1FvXr1WGVq2LAhsrOz8e7dO+a4GjVqKNlRZU9eNnz4cOzevRvVqlXD1KlTcfXqVY02AKBy5crMPp/Ph6OjIypVqsToycYUpqamqrVlYWEBPz8/Jt3DwwMpKSkAgMuXL8Pa2prZ5GcJVq9eXSk2devWZckaNGiA7OxsvH37VmM5tMX9/v37mDt3LsuXIUOGIDk5GXl5eTrVY+vWrbF48WIMHz4cZmZmKF++PNq2bQugcByW7Fd2QUEBtm3bhsaNG6NZs2bYtGkTzp49ywxSV5dHlSpVmH13d3cAQEpKCjgcDp4+fYoGDRqwjmnYsCGeP3/Oelpcr149ls26devi6dOn4HA46NGjB/Ly8uDr64vBgwfj4MGDEIvFaus1Pz8fZmZmLFlsbCy8vLxQqlQpRla/fn2tdaBYPjc3N1hYWMDX15clk5X35cuXyM3NRevWrZn6cnBwwPbt2/Hy5Uu1dj08PFjn64QJEzB48GC0atUKP/30E169eqX23Hn69CkaNmzISqtfvz4TP1X5WVpawsbGhvG7YsWKjL+ycyMuLg5VqlSBlZUV67yWSqWIi4tj7FaqVAkmJibMfmxsLCpXrsyqA1msZTx48ABnz55lndcBAQEAgFevXqm8nnA4HK3HvXr1CkKhkGmPAODo6Ijy5ctrbSuqznF5OcC+7sjOd8VrjEz/3r17qFatGhwdHVXazcnJwaRJkxAYGAh7e3tYW1vj6dOnePPmDYDC15be3t7w9fVFWFgYdu3ahdzcXJX+KrYBc3Nzlq6qv/L3LGNuhshLXxu66mvS0zetOLKibLpSol93Ojk5gcfj4cOHDyz5hw8fDD5AWKrmnTNRGOcBANd+uAaxWAw+Xzl8PC4PgO6D22V6hBD83eVvVposD3kdXW0r+qyYn6I92f8WFhYsHdlFQRYLoPA1tPxxISEhSEhIwLFjx3Dq1Cm0aNECI0aMUBq7JG9XvlyyfGTxJP+ODQHAvKZQFQOBQKBURtl+9erVcffuXUYuP5HE0tJSbQzVxYzL5TKdEpmO/LgyeV35/ezsbERHR6ucFGNqaqq1LmXp48ePx7hx45CcnAx7e3skJCQgMjISZcqUASEEbm5u4PP5KFeuHHNMhQoVAACJiYkICgpSGyv5ulBV36rKqBgrTTqenp6IjY3FqVOncPLkSYwZMwa//PILzp07x7yukdd3dHREenq6yvzVyTTtK55rqs4bWXllryGOHDnCdAhFIhEEAgFTX+rsAv8/X6OiovDDDz/g6NGjOH78OKKjo/H777+jc+fOesdPhqo2I/P76NGjzBhK2aQEbdcLmdzS0lKtvjob2dnZCA0NxU8//aRk193dXe31RNtxL168UIqHuvNQHtkPFfl02XVK3oamGMpDCFGKoyJTpkzBmTNnsHjxYvj5+cHc3Bzdu3dHQUEBAMDKygq3b9/GuXPncOLECcyaNQvR0dG4fPkynJ2dNZYzLS0NZcuWVXvtl79HGRP5++GXsqGrviY9fdOKIysK+hxfojtpJiYmqFGjBk6fPo1OnToBADOGYtSoUcWyvWbNGqxZs4YZX6LPmDQBBOByuOBBeUwaCJgxA7ogPyZNwBMw9gghTB7qxnapQyAQKPns7++P/fv3s8ZUXbx4kVnSQZYHIYQ1VsjBwQEA8PbtW+ZJ1+3btxnfZbr29vbo06cP+vTpg/r16+PHH3/ETz/9pNJvxXxkSKVSJhby8ZEfXyQ/Jk22L68LFMbfxMQE3t7erFjJOlWyOlWMjfxs1gsXLsDa2hqurq4Qi8VwcnJCcnIyE7/MzEzEx8ezbAkEAgiFQpbtatWqITY2lplYoFheTeeJqtjJZjTv3LkTXl5ezDidunXrQiwWIy4uDr6+vgAK11wCAE9PT1a9y1AcwyWLkbysfPnyuHz5MqtMFy9eRLly5Vj6165dY503169fR/ny5VmxadOmDUJCQjBo0CBUrVqVeVKhWM7KlSvj/fv3+PjxI5ycnMDhcODv7483b97g7du38PT0BIfDweXLlwFoH5Mmn6bqvJFdeMViMbN8SXx8PBo0aMD4Jntiqc6uqliWLVsWo0ePxqhRo9C3b19s3rwZoaGhSj5UqFABly5dYsb4EUJw5coVVKhQQen8lo+xvEzWoZQhEong7++P7du3IyMjg+mInT9/HlwuF2XLlmXGxym2u/Lly2PHjh3IyspiOtFXrlxh8hOJRKhSpQr2798PT09PlT9WVV1PAGg9ztvbGwKBAFeuXIGHhwcIIfj48SOePXuGRo0aqR2T5ujoiEePHrHS7927B4FAwBqTpuiPrPzyMlkdBwUFYdOmTUhJSWGugzIIIbh69Sr69euH0NBQAIUd0ISEBDRu3JjJk8fjoWnTpmjatCmmT58OZ2dnnD59Gl27dmVdpxXbwKNHj9CpUyeWTKZDx6Rp19M3rTiyoqDPmLSv3knLzs5mfj0BQHx8PO7duwcHBweULl0aEyZMQHh4OGrWrInatWtj+fLlyMnJQf/+/YuV78iRIzFy5EhkZmbC1tYWzs7OsLGxgVQqBYfDgbOzM4RCIbKyssDn81VeUDQ1EF0rkMfjMXZU2VOU6dIofXx8cPPmTbx9+xZWVlZwcHDAqFGjsGrVKowfPx6jRo1CXFwc5syZg/HjxzPT4mWP3uXLamVlhbp162LJkiXw8/NDSkoKoqOjGd/5fD5mzZqFGjVqICgoCAUFBfj7778REBDAsiPvt6p8gMKYKcZClgePV/iEks/nQyAQMPGVtyHT0RRP2aNq+eNksZkwYQITm7lz52Ls2LHMcgTNmzdHTEwMOnXqBDs7O0RFRYHH47Fs+fj44Ny5c2jcuDFMTU1hb2+PWbNmITQ0FN7e3ujWrRu4XC7u37+PR48eYd68eVrrUlaOxYsXIyQkBFwuF/v27cPixYuxe/duxr/g4GBUr14dQ4cOxS+//AKpVIpRo0ahVatWqFChgspYyPyWP79lejLZpEmTULt2bSxcuBA9e/bE1atXsW7dOqxZs4Z17l65cgXLli1Dp06dcPLkSezbtw9HjhwBn8/H1q1bIZFIUKdOHVhYWGD37t0wNzdH2bJlVZ4jNWvWhJOTE65fv878OAsODoa/vz+GDBmCJUuWIDMzk5kYIztHFMsjf87I0lSdN7JXGHw+H/b29pg4cSImT57MvNr99OkTrl+/DhsbG4SHh6u0K/9XJBJh8uTJ6NatG8qUKYO3b9/i9u3b6Nq1K/h8vpIPkydPRs+ePVG9enW0bNkShw8fxsGDB3Hy5Eml81uxzaiSyejTpw/mz5+PQYMGISoqCqmpqZgwYQL69OkDT09PpuyK7a5v376YNWsWRo4ciUmTJuHt27f45ZdfmPwEAgFGjx6NzZs3IywsjJkt+eLFC+zevRu//vor06lVbOfajrOzs8OAAQMQGRkJFxcXuLi4YNq0aaw6UkXLli2xbNky7Nq1C/Xq1cOOHTvw+PFjVKtWjSmbKn9k5ZeXyc6dvn374ueff0b37t2xYMECuLu74+7du/Dw8EC9evXg5+eHgwcPomPHjuBwOJg1axZz/xAIBDh69CgSExPRuHFj2Nvb49ixY5BKpQgICIBAIFC6TltbW0MgECAhIQHv3r1DcHAwq43JIIRALAXup0rgYGOGWj4O4HF1G16jL6KMDEg+f4aLi4vSPU304QO4FhbgWVtrtCF/T9W1k6aLviY9fdOKIysKZmZmOut+9TFpt27dQrVq1VCtWjUAwIQJE1CtWjVmxmHPnj2xZMkSzJo1i/n1ffz4cYOvg6bunbO68Q+qxr/Ip9ub2cOEq3lNIBOeCRzMHFTaU5Rpy1N+mzx5Mng8HoKCguDi4oI3b97A09MTx44dw82bN1G1alUMHz4cAwcOxMyZM5XyVNzftGkTxGIxatasifHjxzOdC5meqakppk2bhipVqqBJkybg8Xj4448/1PqtqgyqyqnOH02xkD9O1fGq7JYqVQqHDh1ixWbAgAGYNm0aoxsZGYlGjRohNDQU7du3R6dOnVjjmjgcDpYuXYpTp06hdOnSzLi3kJAQHDlyBCdPnkTt2rVRr149LF++HD4+Psxx/fv3R7NmzTTG5Pjx42jcuDFq1aqFY8eO4eDBg8xyHxwOBzweD4cPH4aTkxOaNGmC9u3bIyAgAL///rvWc1VTjGrUqIE///wTu3fvRqVKlRAVFYU5c+YgIiKCpT9x4kTcvn0b1atXx/z587F48WIEBweDw+HA3t4ev/32Gxo2bIgqVargzJkzOHToEPOUTLG++Hw+IiIi8McffzAyHo+Hffv2IT8/H3Xq1MHgwYOZhXq1lUdbmRVl8+bNw8yZM/HTTz8hMDAQoaGhOHbsGMqWLauTDT6fj7S0NISHh6N8+fLo2bMnQkJCMHv2bJXHd+7cGStWrMDSpUtRsWJFbNy4Eb/++iuaNm2qcz2p8sXS0hLHjx9HWloaateuje7du6N58+bMUjKq7ACFn9o7fPgwHj58iNq1a2PGjBlYtGgRS7dUqVK4fPkyJBIJgoODUblyZYwfPx52dnZMB02Vb7oct2TJEjRq1AgdOnRAq1at0KBBA6WxbYpbSEgIZs6cialTp6J27drMMh6KZdN0vVDcNzExwYkTJ+Di4oJ27dqhcuXKWLRoEdOhW7x4Mezt7dGgQQN06NCB+aEks2Nvb4/9+/ejRYsWCAwMxIYNG7Br1y4EBQUBgMrrNAD88ccfaN26NetNgOxvVoEE8R9zkZotxMK/Y9H7txtovPgcTjz5YJBxUvIbyclB0vAR+PjLckhSUlhpkg8f8CYsHO+GDAXJyTHYGDN99TXp6ZtWHFlRNl3hEG2DYr5xZE/SMjIymCdpKSkpcHFxgVAoRHx8PMqUKcPq+coemfP5fKULnXx6an4qPgs/s+Tyr07sTe3hbuWu0p6iTFuexsAQeeprQxd9XeOvmK6P/EvGv0mTJmjWrBnzhFLXcuqCMeKvqFemTBmMGzeOWT1dkw1dYg2A+XLA7du3mVfFtA3orvNfawPq+N7iL5VK4e/vj127dqF+/fos3YxcIV6n5YKIhUhJeovosyl4lyWBzMq6vtURUtFdv+BoQPT+PRLCwvHZ0hJ2OTnwidkKgbs7RMnJeB0WDtGbNxB4ecF7+zYINIwRl7+n6vokTRd9TXr6phVHVhQU+x2a+OqvO0sK+kwcAHQblOtu5Q53sBuNbBCyOjuaBu1qy9MYGCJPfW3ooq9L/FWl6yP/EvHPyMjAy5cvceTIEZV2S2r8FdMV24gmG7rE2s3NDevXr0diYiK8vb2LXBZDUFLr4FtpA9r4nuKfmJiIyMhIZjatTEdKCJIy8lXnA4ADYPbhJ2hRwcVgrz55Li7w3LIZWbPnQPj8ORLCI+C+cCGSIyMhevcOgtKl4bV1C3guLhrH1tKJA6p915XvtpNWnIkDsjRA9SB+den6yBVl2vI0BobIU18buuh/S/G3tLRkVhpX9S3Okhh/RT0ArMHXmmzoEmuZrH379syiwEUpiyEoqXXwLbUBTXxv8ff29sbAgQOVJhPkiqSARAhzSCGBCCYQw4/zFnac/3fc0jOsceLuK9Tw0jxGTB+kHA5MBg1EAQhyUz/i87x5gKUl+A0awGb8OKRzucC/yx6ptUEnDijxn5o48LUw9sQBTen6yIsyccDQGCJPfW3ook/jbzwbuuoLBAKmk6mPDV1irY/M2JTUOqBtwHg2SlL8OcJclOe8BZdDkM8lIJx0bDBZDjPTN4xePhHgvPAYXFx8tfqtK7L7oeWQoXgrN8av9IwZsPh3iR9dbdCJA/9Hn4kD320nTRH5wXzyAwNVDTIl5P/reKn7FaUqXR+5okxbnsbAEHnqa0MXfRp/49nQVV+Tnr5pxZEZm5JaB7QNGM9GSYy/gCMFl6P5Va0ZRwR3QV6xOg+qkKSn48O0aeDIvaJ7/+OP8N4WA4G7bmPg5O+phtTXpKdvWnFk+qLPsV99dieFQqFQKBT1mPB0u1UHldI8CF1fRO/fI/WX5RC9fVs4SWDXLgi8vCB686Zw8kByskHzoyhDn6T9izEmDqhK10eubrBucQbQ6osh8tTXhi76NP7Gs6GrviY9fdOKIzM2JbUOaBswno2SFH+RRIrkz7nwhnY40G9QuiZEHz7gdf8BEFlawtLLC15bNkPg5gavmK1IjOgP4du3SAiPQOmYrRBoWBKLThxQ7buufLedNDpxQDuGyFNfG7ro0/gbz4au+pr09E0rjszYlNQ6oG3AeDZKUvwLRGK8zcwHX0J0eu+VlpYGMU/zQH5dkebnIy8oCCJzc5j36f3/SQJcLsyX/4LsX5ZDZG2FT/n54GqYPEAnDihDJw7oAJ04oDvf+6Bdff0yNCU1/tr06MSB4tmgbcCwef7X4l8gkuBthggiKYE5jwPo8BDQwcEB+PfTcYbAceYMpCQnw93fn90pcXGB85zZ9IsDRYROHCgCdOKAMobIU18buujT+BvPhq76mvT0TSuOzNiU1DqgbcB4NkpC/POEhV8WEEsJTPk8eNpwgM9aXQeXwwEMOHFAYGsLQUGByoHypjpOGgDoxAFF6MQByjdLQkICuFwu7t2797VdMQoRERHM9yqBwm8SylbyNybnzp0Dh8PB58+fAQBbt26FnZ2d1uM2bdqE1q1bs2TR0dHw9PQEl8vFgQMHNB4vFApRpkwZ3L59u4ieUyjfFrlCMV59zIZYKoUZnwtfBxPws95+bbcoXwnaSfsGkX1XUXELCQlh6d29exfdu3eHq6srzMzMUK5cOQwePBjPnj0DANy/fx99+/ZF6dKlYW5ujoCAAOa7fyWZrVu3wtnZ2aA2z507BxMTE6YT87Xw8fHB8uXLv6oPMvLz8zFr1izmQ+cA8PTpU8yZMwdr1qxBUlIS2rRpo9GGiYkJJk6cyHwnlUL5nskpECM+NQcSKYGFCR/eNlzw0l8AEqH2t518U8DC8Uu4SfmC0Ned/2KM2Z2SrCxIc3JY3zWT6QuTk8G1tGTe5xt6ZlVISAg2b97MkpmamjLHHjlyBN26dUNwcDB27NgBX19fpKSkYM+ePZg5cyb++OMP3L59G87Ozti+fTu8vLxw5coVDB06FFwuF6NGjdLqg7pyFEdfXT1os6XNti7x1zVPQ6ApT1Xnoy62NB2jqtzajtm7dy9sbGxQv359Ru/FixcAgNDQUJiYmKi0oehPnz59MGnSJDx+/Jj5+LQ6v/U9lwyBIfI0ZBvQVUefc12dvDjXIEPxvcQ/u0CCdxkFkBICK9PCDho3/SU4UjEIzwSw84ZIIgURi0EyeJB23wYpX+5Vq4UDYFMKMNDsTsAwsxnp7E7VvuvKd9tJM/bsTtHnz/gwajQkaWko9e/UZdlx4vfv8W7AQPAcHOCxfh24VlYGnVkllUohEAjg5OSklCYWi5Gbm4sBAwYgJCQEf/31F5Pm5eWFGjVq4PPnzxCLxejXrx969+7NfBC+dOnSuHLlCvbt24dhw4apzDshIQH+/v7YvXs31q5dixs3bsDPzw+rV69GvXr1GL0rV65g1qxZuHnzJkxNTVGrVi3s2LEDdnZ2EIlEWLJkCTZv3ow3b97A1dUVgwYNQmRkJFMPEokEIpFIKRbnz5/HgAEDAPz/vf+MGTMwa9Ys5OfnY+bMmdizZw8+f/6MoKAgLFiwAI0bN4ZEIsHr168xbtw4XLlyBUKhEN7e3li4cCECAwPRvHlzAP8OzAXQr18/bNq0CXv37sW8efPw8uVLWFhYoGrVqti7dy8sLS2VYiORSDB8+HCcO3cO79+/h5eXF4YNG4bRo0ez6k52zsk6Y7L9li1b4vXr15gwYQImTJgAoPB1IQDs27cPs2fPxsuXL+Hu7o4RI0Zg/PjxzHmzc+dOrF69Gs+ePYOlpSWaNm2KpUuXwuXfQcayc0skEkEikTD7ip+qkkEIwe7du9G2bVtGZ86cOZg3bx6A/w+Mlfm3detW/PLLL3j58iXs7e3RuXNnrFy5EgBgZWWFevXqYdeuXZg9ezZjn87u/PqzC+nszi8T/8x8MZIyCkAAWJnw4GkpBTftFThEAinPFBJbHxAOHxKOBCIAUg4Pn7gOEPDkJiAUQOsnmvTFELMZ6exOZejsTh0w9uxOcX4BpOnpEL99i6QBAwvXknF3h/j9eyQNHATx27eFAxALCiCwt1drrygzq2SDGlX5DACnT5/Gx48fMXXqVJU6ip07+TyzsrLg6Oio1rZMHhUVhcWLF6NcuXKYNm0awsLC8Pz5c/D5fNy7dw/BwcHo378/VqxYAT6fj7Nnz4LD4UAgEGDatGnYvHkzli1bhoYNGyI5ORmxsbGseuDxeCpj0ahRI/zyyy+IiopCbGwsgMJOAJ/Px/jx4/HkyRP8/vvv8PDwwP79+9G+fXs8ePAAPj4+GD9+PEQiEc6fPw9LS0s8ePAAdnZ2KFOmDP766y9069YNsbGxsLGxgbm5OVJTU9GvXz8sWrQInTt3RlZWFi5evAgej6cyPoQQeHl54c8//4SjoyPzZLJUqVLo0aOHyrqTDVLl8/nYt28fqlatisGDB2Pw4MFMvG/fvo3evXsjKioKPXv2xJUrVzBy5Eg4OzsjIiICQOHFZe7cuShfvjxSUlIwceJEDB48GEePHmXiKatrHo/H7KurZwC4evUqwsPDGZ0pU6agbNmyGDBgAF6/fg2BQAA+n49169Zh4sSJWLhwIdq0aYNPnz7h+vXrLNu1atXClStXlPKjszuLp0NndxbdxpeK/+dcEZIyCl9n2poL4GVFwEmLL+yg8c3AcfQDn8tuF1wuF46OjnrNEiwKhpjNSGd3KqNXvZHvnIyMDAKAZGRkEEIIkUgkJDk5mUgkEpKXl0eePHlC8vLyWMdIpVIiFAqJVCpVaVOWXvDuHXneshV5Ur4Ced6yFcm+fZs8a9GS2RcmJam1pyjTlqc84eHhhMfjEUtLS9Y2f/58QgghixYtIgBIWlqaRjuKeV6+fJnw+Xzyzz//qD0mPj6eACC//fYbY+PevXsEAHn69CkhhJAffviBNGjQQOXxGRkZxNTUlGzcuFGj/Rs3bqiNxebNm4mtrS0r/fXr14TH45GEhASWvEWLFuTHH38kQqGQVKpUiURHR6ss+5kzZ5Ridvv2bQKAJCQkqI2HNkaOHEm6du3K7IeHh5OOHTsyPjRu3JiMGTOGSff29ia//PILy0bv3r1Jq1atWLLJkyeTwMBAtefNzZs3CQCSlZVFCCHk7NmzTPmEQiETQ3WkpaURAOT8+fMs+f79+wkAVp4eHh5k+vTpTJlUnevLli0jPj4+LJm2NvElMESe+trQRV/Xa5Biuj7y4lyDDMW3Hv9P2fnk/pt0cv9NOklIzSKSvExCku4R8u4OkabEEWF+nlL8c3NzVd6XjIH8/fBL2dBVX5OevmnFkRUFxX6HJujEASMicHcv/L7Zv5/RSOzdB2LZ5zX0+O5ZUWjWrBnu3bvH2mSvKEkRxnY8evQIHTt2RFRUlNJsPlVUrlyZ+d/933Km/Pso/t69e2jRooXK454+fYqCggK16YoEBQXBysoKVlZWGgepP3z4EBKJBEFBQbC2tmaOOX/+PF69egUAGD16NObNm4cGDRogKioKDx480Jh3lSpV0KJFC1SqVAndu3fHr7/+ivT0dI3HrFmzBjVq1ICzszOsrKywceNGJCYm6lRWdTx9+hQNGjRgyRo0aIDnz58zr1Zu376N0NBQlC5dGtbW1mjSpAkAFDnvvLw8ANp/EaakpCApKUlrfZqbmyM3N7dIvlAo/0VSswrwNr2wHTlamqCUmRCctJcAkQImVoCjL8DlfWUvKV+b7/Z155dC4O4Oj0WL8Lp3b0bmvugno3bQAMDS0hJ+fn4q0/z9/QEAsbGxrHFi6njy5AlatGiBIUOGYMaMGTrlL/84Xzb2QjZY0tzcXO1xmtJUcezYMYhEIq3HZmdng8fj4dq1azA1NWWNB5GNHxs0aBBCQkJw9OhRnDhxAj/99BOWLFmCMWPGqLTJ4/Fw8uRJXLlyBSdOnMCqVaswffp0XL9+HWXKlFHS/+OPPzBp0iQsXboU9erVg7W1NRYvXozr16/rVWZ9ycnJQUhICIKDg7Fz5044OzsjMTERwcHBzJgxfXF0dASHw9HaKdW1PtPS0gw+I5dCKYkQQpCSVYAPmfkAAGdrU7iZ5APpieCAAKY2gH0ZgMMBYLhJAJT/JvRJmpERJScjaepUlix56o9f9cO0rVu3hpOTE37++WeV6fLLTDx+/BjNmzdHeHg45s+fb5D8K1eujNOnT6tMK1euHMzNzdWmK+Lt7Q0/Pz/4+fmhVKlSAAqXdZA9QZJRrVo1SCQSpKamMvqyzU1u9q1sMP/evXsxbtw4/Pbbb4xNAEp2ORwOGjRogNmzZ+Pu3bswMTHB/v37Vfp6+fJl1K9fHyNGjEC1atXg5+eHly9f6lROGarKFhAQgMuXLyvl5e/vDx6Ph7i4OHz69Ak//fQTGjVqhAoVKjBPNYuKiYkJAgIC8OTJE4161tbW8PHx0Vqfjx8/RrVq1YrlE4XyXyA1W8R00FxtzOAmyAPSE8ABATGzAxzKGHRBWsp/G3omGBFRcjJeh4VD9OYNBF5eKL1rJ/ienhC9eVMoN2JHraCgAO/fv2dtHz9+BFD45Oi3337D0aNH0aFDB5w6dQoJCQm4desWpkyZwrwWffToEVq3bo3WrVtjwoQJjJ3U1NRi+RYZGYmbN29ixIgRePDgAWJjY7Fu3Tp8/PgRZmZmmDRpEqZOnYpt27bh5cuXuHbtGjZt2qSzfR8fH2RnZzMTJHJzc+Hv748+ffpgwIAB2LdvH+Lj43Hjxg0sXLiQGTw/btw4/PPPP4iPj8edO3dw/vx5BAQEACjsDHI4HBw5cgSpqanIzs7G9evXsWDBAty6dQuJiYnYt28fUlNTmWMUKVeuHG7duoV//vkHz549w8yZM3Hz5k29Yufj44MLFy7g3bt3TH1OnDgRp0+fxty5c/Hs2TPExMRg9erVmDRpEoDCjqeJiQlWrVqFV69e4dChQ5g7d65e+aqidevWSp1DVURHR2Pp0qVYuXIlnj9/jrt372LVqlUsncuXL6NVq1bF9olCKakQQpD0OR+fcguf/LvbmsOVlw3O59fgAJCY2gH23gCH3pYpchRr9Ns3gGwAX3p6OpFIJEQkEpGkpCQiEolITk4Oefz4McnNzSVSqZS1FRQUKMnkt5zXiaxJAwXv3v0rf82WJyWptaco05anbAsPDycAlLby5cuz9G7cuEG6dOlCnJ2diampKfHz8yODBw8mz549I1KplMyaNUulHW9vb7V5v3r1igAgd+7cYWQfPnwgAMiZM2cY2dmzZ0n9+vWJqakpsbOzI8HBwSQtLY1IpVKSl5dH5s6dS7y9vYlAICClS5cm8+fPZ9mXTRxQtw0ePJg4OjoSAGTWrFlM/KZPn058fHyIQCAg7u7upHPnzuT+/fukoKCAjBw5kvj6+hJTU1Pi7OxM+vTpQ1JTUxmbUVFRxM3NjXA4HBIeHk4eP35MgoODmfj5+/uTlStXqvUpLy+PREREEFtbW2JnZ0eGDRtGpk6dSqpUqcKqu44dOzL7sokDsv0rV66QypUrE1NTUwKAke/Zs4cEBgYy8fr5559Z583OnTuJj48PMTU1JfXq1SMHDx5k1ZP8xIiCggLW5At1271794i5uTlJT09nZPv27SMAlM7VdevWkfLlyzNxHzVqFJN2+fJlYmdnR3JycrSe77q2AUNuhshTXxu66GvTUZeuj7yo1yAaf3a6RColiZ9ymEkCH7PyiTTrPSHv7hROEkhP1Br/AlEByczLJGmZaeT+w/vkwbsH5FHKI2Z7m/mWSCQSg27y98MvZUNXfU16+qYVR1aULT09XeeJAxxCvuCqhCUI+XXSnj17hmfPnsHa2ppZB8XW1hYSiQQZGRnw9vZmDZAm/651I1s/TBGiZZ00kpqqcp00eXuKeWjL0xgYIk99beiir0v8VaXrI6fx16wv0+vbty+qVauGqXKv9DXZUJXWu3dvVKxYEZGRkWrjX5SyGIKSWge0DRjPhqHjDwBJGUJkFojBAeBqxYcjMsDLLRxyIDF3gsTSVWP8xUSMhOyEwnUTRVKkvE3BoheLkCz8/9sYAVeArQ23wsXccB9Yl78fFnedNF1t6KqvSU/ftOLIikJWVhb8/f2RkZEBGxsbjbrf7cQBY6+TBjs7lP7tV6UvDgAonN25fRvriwPq7NE1ioquQ9eIKroNXfWXLFmCw4cP690+ZGlCoRCVK1fGmDFjdF4TjdaB7jq0DRTdhiHiz+Pz8SYtr7CDxuHAy94c1gUfwMsrHKpArN3BtXJlxh2pi79ELNE6K18kFYFnxYOLo2E7aXSdtK+7Ttp320lTRP6r9vJfuZf/9qUMQgizr+5XFIfDAc/aGny5XrL8cSZysztV2VOUacvTGBgiT31t6KKva/wV0/WR0/hr1pfp+fj4KM1+1WRDMc3U1BQzZsxgvlqgLv5FKYshKKl1QNuA8WwYKv6EAK8/5SK7QAwuh4PSDhawLngPzr8dNNiUAsfKRa09eRl0LDqHyylW50GlTbn74Zeyoau+Jj1904oj0xd9jqWdNAqFQqFQDIhESpD4uQB5Igm4HA58HC1glZcE5KUVfijd1gscS+XP9lEoitBpJBQKhUKhGAixRIr4jznIE0nA43JQ1skCVrlvmQ6axNoTsHDUz6hEx4+T59AFob816JM0HfhO51ZQKBQKRQMEQE6+GGKpFHwuFyZ8LhI+5SBfJAGfy0EZRwuYZycCBZkAOIC9DwjfUv98iFwn7d/bkVTFQrfS/LyiFYRSYqGdNA3IZuYIhUK9V8KnUCgUyrdLZr4YKdl5EMk95eIULkkLAY+L0rYCmGUlAMJsANzCRWpNrYF/x1/qA0duYg6REIilYuRKlJ+a8R31fEJHKfHQTpoG+Hw+LCwskJqaCoFAwAz2I4RALBaDz+erHTSqKl0fuaJMW57GwBB56mtDF30af+PZ0FVfk56+acWRGZuSWge0DRjPhi76WXlCJGXkKx/77187cwE46e9QIMkDwAXsPAGYgOTnFyn+BZICSEVSgAC5n3PxJOsJsiXZ+oaC8h+EdtL+RSqVMhshBFKpFFwuF66urkhISEBCQoKSvrbpwarS9ZEryrTlaQwMkae+NnTRp/E3ng1d9TXp6ZtWHJmxKal1QNuA8Wxo03+fWQCJVPUwGB6k4HAy8RkigMMFsXQGsj9ota1KLpFIkCXOAp/LR7YwG1JIkSHMwL4P+0DAzt+EawJbE1vmG8mGQP5++KVs6KqvSU/ftOLIioI+x3+3nTT5xWwBIDU1Ffn5+cxidYQQpsHIFraVjU0jhCArKwtWVlZqf8WqStdHrijTlqcxMESe+trQRZ/G33g2dNXXpKdvWnFkxqak1gFtA8azoU3/8fscbDgdCxtOjlKaHXIwlr8XHtw0FPBtkd18AaR2/38FqU/8hRIh1j5Zi4cZD8HlcDE5aDKE6VngHNyPsZ+sARSuscmzs4N9n95wcCoNXg4PKTnF+y6vPKruh8a2oau+Jj1904ojKwpZWVk66363nTRdFrPV9CQgNTVVrY66dH3kijJteRoDQ+Sprw1d9Gn8jWdDV31NevqmFUdmbEpqHdA2YDwb2vSfJN7C78LRMOOIVBsQAoQA56pOQcvABkWKf544DxPOTcCV1CsQcAVY3Hgx6vMr4OmqOTC9ch8mHh5wX7gQyZGREN24D8GTTyi9dQsELoZbyFbml7b7oaFt0MVs2Xy3nTRF1C1mqw5tOurS9ZErygyxiJ6+lNSFDGn8jWfjW19IUl9Kah3QNmA8G5r03QV56jtozPGApbV9keKfJ8nDqDOjcOvDLZjxzLC86XLU5vsiISwcEktLmHh4wCdmKwTu7vCJ2YrXYeEQJSbiTXgEvLdvU/rCTXEpafHXVa8kX4P0OfbLDi6gUCgUCuU/TE6BbrMzy7vovyJAZkEmhpwYglsfbsFKYIWFNRainkc9cC0twXdwAN/JqfCJ2b9frBG4u8N7WwwEXl7gOziAa6n/8h6Ukg19kkahUCgUig4cvp+E347H4qAOnwHlFmEM3dm3Z/Hg4wPYmtpiXYt1cJY4AwB41tbw3LAegqQk5W9Bu7ur/BY05duAdtIoFAqFQtHCnzffYOq+BwiE8RY37+jbEZnCTNT3qA9fW1+kpPx/EgDP2hp8e3uVxxn6FSel5EA7aRQKhUKhaGDr5XhEH34CAGhTyQ14bjjb77LfwYpvxeyHB4UD0G+ZBsq3Cx2TRqFQKBSKGtacfcF00AY1LIORPkkGs/0q4xXCjoVh1JlRyBPTTzpRlKFP0igUCoVCUYAQgsX/xGLN2ZcAgLEtymGc6z1w9kUZxH5sWiyGnhyKtPw0WJtYI1+i/AUDCoV20igUCoVCkYMQgrlHnmLr1dcAgMg2FTDU6iKwbxwAAsLhgkPUv44kfFNIzVSPHwOAB6kPMOLMCGQJsxDgEIB1LdZBlKl5WQ/K9wntpP2Lqs9CadLVpKMuXR+5osxQn6PQh5L6SRAaf+PZ+B4+yaIPJbUOaBswng2RWIIFp17j8ONPAIDZHQLRD8eAw9MAAKRGf0jqjUF60ivY29urXkzYzB4SoanKON//dB+z7s1CrjgXVZ2rYnXz1bDkWyKVpNL466H/X74G6XP8d9tJ0+ezUIpo01GXro9cUaaLX4bGEHnqa4PGX3s5jGlDV31NevqmFUdmbEpqHdA2YBwbYglB9D+vcOrZZ3A5wPSW3uiSvhXcm8sBADlVBiCr5hRIhQQZglIQ8WxVx79AioyMz0p53ky9iai7URAREao5VMPsyrOR9zkPOdIcGn899f/L1yD6WSgdKO5noTTpqEvXR64o08UvQ2OIPPW1QeOvvRzGtKGrviY9fdOKIzM2JbUOaBswvI0CkQSj/7iHU88+g8cFlnevjPapv4Jzc0WhnSaRMG88GeYcTpHj7y/wh8VDC1RxqYIlTZbAlGeqVv97i7+++v/laxD9LFQRkP/MA/0ki3Z/jWmDxl+7v8a08a1/kkVfSmod0DZgOBu5QjGG7riDi88/woTPxcK2PmiftAKcm78WKrSeB2790XrZVJVezr4cltdZjkrelWDKN9Wq/73Ev6j6/9VrkD7HfrmaplAoFAqlhJGZL0L45hu4+PwjLEx42NKvGtq8/vnfDhoHaLcMUOig6cOeZ3twLfkas+9p6QkBV4dPFlAooE/SKBQKhfKdkp4jRNjmG3j4LgPWZnzEhFVFtVtTwYnbXziDs9M6oEqvItvf9mQblt5eCnO+OfaG7kUpq1IG9J7yPUA7aRQKhUL57kjJyke/324g7kMWHCxNsD2sMoIujwGeHQfhCkC6bgInqGORbBNCsP3Fdmx7uQ0A0LtCb3hae4IQ431SivJtQjtpFAqFQvmuePc5D31/u474jzlwsTbFrvCK8Ds9GIg/D8I3Q3rrlbALCFV7vCQrC+L0dMDFRSlNmJyMFa+2YtvLXQCA0dVGY0jlIQBAO2kUvaGdNAqFQqF8NyR8zEGf367j3ec8lLIzx+9hFVD67zDgzXXAxAqk1+8QWvirPV6SlYW7YwfgnZ0pyg4bCoGzM5MmSk3Bmt2TcdWn8OsBk2tORlhQmNHLRPl2oZ00CoVCoXwXPPuQhT6/XUdqVgHKOlliVx8/uB3qBSTfB8xsgb77AI/qQEqKWhvvPsZjWONnEPEAXB+lrOADgAD9vXqjb0BfYxWF8p1AO2kUCoVC+eZ59C4D4VtuIj1XhApu1tjZ0weO+7oCqbGAhRMQdgBwqwRoWQ0+24Zf2EHTBAeo5dXQYL5Tvl9oJ41CoVAo3zQPkrIx4eB9ZBeIUcXTFtu6usP2zw5Aejxg7Q6EHQKc1b/ipFC+FrSTRqFQKJRvlssvPmLMvufIF0tR28cBmzvYw+r3DkDmW8DOGwg7CDiU+dpuUigqoZ00CoVCofznkUgJbsSnISUrHy7WZqhdxgHn4lIwfOcdCMVSNCrnhF+DLWC2MxTISQEcyxV20Gzp2mWUkgvtpFEoFArlP83xR8mYffgJkjPyGZmdhQCZeSJICdDE1w4bWnILO2h56YBrRaDfAcDKWb1RCqUEQDtpFAqFQvnPcvxRMobvuAPFFcg+54oAADW97bC4ejpMdw0DCrKAUjWAPn8BFg5Fyi8l6aVOepLMzCLZp1DkoZ20f5FKpcxGCIFUwwwfbTrq0vWRK8p08cvQGCJPfW3Q+Gv315g2dNXXpKdvWnFkxqak1gFtA4VIpATRh54oddDkcf90DU5/LwZHnAdSuj7ID38AptZqZ3FqyvNd9jvMfbJMN+dNTL75+BvDxvdwDdLn+CJ10kQiEd6/f4/c3Fw4OzvDwaFov0i+JmvWrMGaNWsgkUgAAKmpqcjPz4dUKkVGRgYIIWq/VK9NR126PnJFmS5+GRpD5KmvDRp/7eUwpg1d9TXp6ZtWHJmxKal1QNtAIbffZIGb+RZBnCyVx9fmPsWPot/B5UiQ79UIn1utAjLyAOQVKc87qXeQmv9Jq98CjgA8qRlSUlK+6fgbw8b3cA3KylJ9vqpC505aVlYWduzYgT/++AM3btyAUCgEIQQcDgeenp5o3bo1hgwZglq1ahXJ6S/NyJEjMXLkSGRmZsLW1hbOzs6wsbGBVCoFh8OBs7OzxgukJh116frIFWW6+GVoDJGnvjZo/LWXw5g2dNXXpKdvWnFkxqak1gFtA4XwXybijOlEmHFEGu28ta8Dtz674WJiXqw8Q11CYW1jDQdTB2RmZMLe3h5cjrJfNiY24Ofyv/n4G8PG93ANMjMz01lXp07asmXLMH/+fPj6+iI0NBTTpk2Dh4cHzM3NkZaWhkePHuHixYto3bo16tSpg1WrVqFcuXJFLsDXgMvlMkHncDisfVVo01GXro9cUaaLX4bGEHnqa4PGX7u/xrShq74mPX3TiiMzNiW1DmgbANwFeVo7aACQEDAcHibmRYr/o4+PYGdqB09rTwBAc+/mkEqlSEEKXJxc1HaSU/JSvvn4G8vGt34N0udYnTppN2/exIULFxAUFKQyvXbt2hgwYADWr1+PLVu24OLFi/+5ThqFQqFQ/lsEuNvopOfvZl0k+7fe38LI0yNhb2aPbW22wcVC+YPqFIox0amT9vvvv+tkzNTUFMOGDSuWQxQKhUKhaEMskWL56WeYrIMul8PR2/7lpMuYcG4C8iX5qGhVEVYCK/2dpFCKSbGfmWZmZuLAgQN4+vSpIfyhUCgUCkUjBWIJRuy8g3NxqUaxf/nDZYw9Oxb5knw09myMNS3WwEJgYZS8KBRN6N1J69GjB1avXg0AyMvLQ82aNdGjRw9UrlwZe/fuNbiDFAqFQqHIyBNKMCjmFk48+QABz/Bjs46+Ooo59+dAJBWhlXcrLG+6HGZ83Qd6UyiGRO8z/MKFC2jUqBEAYP/+/SCE4PPnz1i5ciXmzZtncAcpFAqFQgGArHwRwjffwMXnH2Eu4CG6fQWD2j+RcALTL0+HlEjRoWwH/Nz4Zwh4AoPmQaHog96dtIyMDGZdtOPHj6Nr166wsLBAu/+1d57hUVVbA37PTJJJD+kQCAlNpEgRRAVFEBRRQewFNGC9Cjb088LVS7GAFfAiFy4WQEXFgg0VRRBRQUAQFBCkhJ4GCel1zv5+REYCycw5k5lkSNbrk+cxa6+z1trrzN5ZnNl7nyuuYOfOnR4PUBAEQRCOFZUx4rW1rNubTZjNj7dHdafbzpke9dGzaU9aRbRiSOIQJveejJ9FznsX6hfTn8DExETWrFlDVFQUS5cu5b333gMgJyfH1NkfgiAIgmCErPxSUuatZ3t6PpHB/ryd0pVOqx+C3StcXqv8bOiBkYb8RAVGseCyBRTnFFd7/pkg1DWmi7SHHnqI4cOHExoaSlJSEv369QMqvwY966yzPB2fIAiC0IjJzC/jwbfXknqkkNgwG++knEW7FffAnu/AaoMrp0N89cdDAaigSPRSW/VtSvHyxpdpEdaC6864Dqg8iLZEK6lWXxDqGtNF2n333UevXr04cOAAl1xyieNQttatW/PMM894PEBBEAShcbLvaCH3fLCDtLwymjcJ4p3bOpK0NAX2rwH/ELj5XWh9kXMjug6ZmaeKlc7UtVN5b8d7WDQLPeJ70CqilZd6Igju4dbz3J49e3L11VcTGvr3uTHdu3fnxx9/9FhggiAIQuNlV2Y+N726lrS8MpKig/kwpT1JS26qLNBsEXDbJ64LtBqw63Ym/DSB93a8h4bGE+c9IQWa4JOYfpJ2++23Vyvft28f69at47HHHqt1UIIgCELjZevhXG59fR3ZhWW0jg7kvVtaEffxdZC5DYKj4daPoVlXAOz5+eiFhfg3bXqKnfL0dAiq+r7Ocns5438cz9d7v8aqWXmqz1MMaTOkTvolCGYxXaTl5ORU+d1ut7Nnzx62bdvG7NmzPRaYIAiC0PjYuD+HkW+sI6+kgs7Nw3m5fyCxH10D2bshtCnc9inEVR69Yc/P58Cdd1GRnU3Smwvwb9bMYac8LY19t6VgjY7GNnUKAKX2Uh5Z+QjfH/weP4sfL/Z9kQFJA+qln4JgBNNF2scff1yt/JlnnuGTTz7hnnvuqXVQgiAIQuNjze6j3LFgPUVldnomRfLG0EhC3r0WreAQRLSElE8hqrVDXy8spCI7m/IDB9h3W4qjUDteoJUfOIDSNPxLKjcCfL33a74/+D02q40Z/WdwQfML6qurgmAIj+0xvvnmm1m5cqWnzAmCIAiNiO+2ZzJy3jqKyuxc0DaGt4aGE/buMKwFh1DRbeH2r6oUaAD+TZtWFmaJiY5CrWjjr44CzT8xkZbz5+EXWXkEx5DWQ7inyz3MHjhbCjThtMBjJ/Vt3ryZ7t27e8qcIAiC0Ej46vc0HnjvV8rtioEd4pnV34Lt7SFQnE151BlYUz5HCz91zRmAf7NmJL25wFGY7bvllkp5YiJJby4gLyKA4uw8ADRNY0z3MXXWL0GoLaaLtLFjx54iy8jI4NNPP+WKK66o0j5t2rTaRScIgiA0aD7acJD/+3AzuoIruzRjRu9S/BbeCKW5qITuZF86m9jQuBqvTytIIycgh5JJ95L2r3855M0m3cs6fQ9Tlk4hJiCGOfFzCLIE1WhHEHwR00Xar7/+Wq38nHPOITMzk8y/zqPRNK12kQmCIAgNmrd/3scTn2wB4IaeLZjaPQfrwlugvBBano+6+T1Ubs0Hy6YVpHHlJ1dSZi+rFNx+wp+0XRNgV+X/5gfkk1WURcuIlt7qiiB4BdNF2nfffeeNOARBEIRGxKur9vDMl38AMLJ3MhPOOIDlndvAXgqt+8NNC8EvCKi5SMspzfm7QHPCw50epkVYC0+FLgh1hrw9VhAEQagzlFK8vHwnM77dCcB9/drwfy3+QHv/TtAroP0VcP088LNVvi3ACRVHjxryGVkWUOu4BaE+MLS787LLLuPnn392qZefn89zzz3HrFmzah2YUQ4cOEC/fv3o2LEjXbp04YMPPqgz34IgCIJxlFJM+fIPR4H2f4Pa81jTjWgf3V5ZoHW+Dm5YUFmgGcASaGyNmRZgzJ4g+BqGnqRdf/31XHvttURERDBkyBB69uxJQkICgYGB5OTksG3bNn788Ue+/PJLrrjiCl544QVvx+3Az8+PGTNm0K1bN9LT0+nRoweXX345ISEhdRaDIAiC4BxdV/z70y0sXLsfgAlXduT2gG/hk0crFc6+Da6cARarYZuWkGBjeoFSpAmnJ4aKtDvuuIMRI0bwwQcfsGjRIubOnUtubi5QuUGgY8eODBo0iPXr19OhQwevBnwyzZo1o9lfp0w3bdqUmJgYsrOzpUgTBEGoJ+y6Yl1qNpn5JcSFBXJ2yyaMX/w7i389hKbBs9ecxY2li+HLiZUXnHsvXDYVZMOZIFTB8Jo0m83GiBEjGDFiBAC5ubkUFxcTHR2Nv7+/2wGsWrWKF154gQ0bNpCWlsbHH3/MsGHDqujMmjWLF154gfT0dLp27crMmTPp1avXKbY2bNiA3W4nMTHR7XgEQRAE91m6JY3Jn28j7YRdmYF+FkoqdKwWjWnXd+GqnAWw6vnKxgsfhYufkAJNEKrB7TcORERE0LRp01oVaACFhYV07dq1xnVsixYtYuzYsUycOJGNGzfStWtXBg0a5Djq4zjZ2dncdtttzJ07t1bxCIIgNAT04mLKMzKqbSvPyMCen+9xn0u3pHHv2xurFGgAJRWVGwDuuiCZqzL++3eBNmAiDPi32wXaT4d+qlW8guDr1PvuzsGDBzN48OAa26dNm8Zdd93FqFGjAJgzZw5ffPEFb7zxBuPGjQOgtLSUYcOGMW7cOHr37u3UX2lpKaWlpY7f8/IqT6LWdd3xo5RCd7KryJVOTe1m5CfLjMTlaTzh06wNyb/reL1pw6i+Mz2zbbWReRtfvQeudMpzc8mcOZOCrdtImvcG/k2bOq4rP3qUfWMfwT8ykhb/m4M1LMwjY8CuKyZ9ugVVQ8waOmf+MgH0ZZX2LnsOet3tdAenM58f7/qY//z6nxqvPU6AJYBwv3CZg9zAVz//rvR8fQ4yc329F2nOKCsrY8OGDYwfP94hs1gsDBw4kDVr1gCVu4VGjhzJxRdfzK233urS5tSpU5k8efIp8qysLEpKStB1ndzcXJRSWCzVP2h0pVNTuxn5yTIjcXkaT/g0a0Py77of3rRhVN+Zntm22sjcQS8uRi8pcbzP8UQqcnKwBAZiCQrymE9d1zmWlUVFTg7+UVEe8elKpzw7m8KgYMpCQiicOInYhx/CLyqK8uxs9r/zLv4hIfiHheF/+DB+kZEeGQO/7D6CJf8wnbRTn9BZ0XnI7yMu1jehsJDX7ymKk4fBSd+IGO3nx/s+5r/b/wvAxU0v5prka7Bo1ecqzBpGQGkAmZmZMgeZROYg78xB+SaeYvt0kXbkyBHsdjvx8fFV5PHx8Wzfvh2An376iUWLFtGlSxc++eQTAN566y3OOuusam2OHz++yqur8vLySExMJDY2lvDwyn9taZpGbGys0wHqTKc8Nxf7sWPExcWdMhDtx44RHRSEf0SEU3sny4zE5Wk84dOsDU/kv6Z2M3LJv3N9Z3pm22ojM4s9P5+DDz1MRXY2LefPczxhAihPT2f/g5XFzIlPmGrrszw3l8wFCyjeuo24E55q1canyzEQEwO33Ezx2Eeo2L6d4n37aDZ1Kof/9S/8Q0JoUlhI0iszqzxhq+0YsGzaygrbIwRq5TXmQinYdMZDdO17H2EGcledz9d+f81RoN3W8TbGnj3W6RtudF0nKytL5iA3kDnI83MQQGBgoGFdny7SjHDBBReYenRos9mw2U7djm2xWBxJ1zStyu/VUZOOPT+fw/fex7GwMOImT8IvIcHRVp6ezpHpM6jIz6flq3OxhoU5tXeyzEhcnsYTPs3aqE3+XbWbkUv+nes70zPaZs/Px15QUK2+/dgxVHAwlhP+QVPbfNiLi7EfPUrFgQMcSBlJ0psL8G/WjPK0NA6kjKTiwAE0paC42OG31vegpASVX+Bxn650/KOjiZv3BgdSRlK+fz8Hhg9HWSz49+lD0iszsZ0wN9Vkz8wYaBHl57RAq7werO36mcrliT6X7FnCzE0zAbi3673c2/VeQ68glDnIfRr6HOQJmVnMXGu6SEtJSeGOO+6gb9++Zi81TUxMDFarlYyTFr9mZGTQ9IR/jXoCXffMmrSKggLKc3IoLytj36jbHetBytPT2TfqdspDQigvLKSioADtr2NCjHz37Soue34+elER/ic9dYTKRcKW4OAqRaHRnNTWpxYS4tH1CPb8fCoKC1GadoqOK5812fZE/r2BJ3yasWEkt8c/Q87sGm2z5+dz8J5/UJ6TQ9C0lyqf/vxFaVoamdNnUJafT+Kc2TWumzKLNS6OxAXz2T9yFGUHD7I3ZSTNpk4lbfx4yg8dwr9lSxLnz8MaF2d4PnDpMzaWmIcepHj/fsoOHPCIT1c6x9utcXE0ffZZ9v+1FERZLESm3Obw5cye2TEQFWJsE1mHZmFuzwcDEgdwfrPzOb/Z+aR0SkEphVI1rYKr3oZZn+7omJlrapI3xjnIjL4n5iBPyNzBzPWmi7Tc3FwGDhxIUlISo0aNIiUlhebNm5s1Y4iAgAB69OjB8uXLHcdy6LrO8uXLGTNmTK1sz5o1i1mzZmG32wEPrknTNIKmTyNr4Tsc++MPCidOImpkCtnzF1QWaB06EDT8FnI0zbEeozp7J8uc+dSLi8maORM9v8Cx9uQ4FdnZZE2fgSUslNj773esezGCJ3xGjx5NflmZR9YjHPdpLygg4M47q+gY8VmTbVf5p7QUe3ExBZp2yrUnryfyFHpxsUd8Gvk8H/dnJLfHP0NO75PBNj03l2NhYZSXlZH1zrsw/Bb8o6KoyM4mc8YMCm2BgCLAybopt7BYCJoxnYLpMyg8coRjTz8NISH49elD+MMPkWOxOB2bZtF1nQKLhdDp0zjiIZ+udI63V+TkcGTOHCratwdAaRrHvlqKNSqqyvq42s5B+7JLmLn4N942kI/cYzlU+Dtfi3ZiXDnHctB1Hau18pDbiWdNxKpZT9nh78yGrIt1D099/mVNWlW8uibtk08+ISsri7feeosFCxYwceJEBg4cyB133MFVV11l+kiOgoICdu3a5fg9NTWVTZs2ERUVRcuWLRk7diwpKSn07NmTXr16MWPGDAoLCx27Pd1l9OjRjB49mry8PCIiIjy6Ju3k9SCFP/2EDQhJTCTolptpdsYZpwzQ2qxHKM/IoHDLVsoPHqR43z7HWpvy9HT2P/QwtoMH8W/RgujAQPzj4gznyCM+g4KwBgd7ZD3CcZ9lhw9TikbkxAnY4uIM+6zJtrP8RwUGcnjsI5Tn5BA67aUq6wyrW0/kCY6vm/KETyOfZzCR278+Q87sGm5r2pTYyZPYN+p2jv3xB8UPjyVyyhTSxo/Hdvgw9O5Nh4kTsf11WLXRvhgiLo4m//iH4wkTQMsnniD4zDPdyp8zTrQR6SGfLucgXceek0Px2EewHThAaIsWjjVpx3Mdd9KuT3fnoD/S8rjvo99pWlQBBg72j4qKAoPzUGlFKVN/n0piQSKP9HjE0Feb1eVC1sW6h6c//57Kvys9s221kbmDmTVpbnmJjY1l7NixbN68mbVr19K2bVtuvfVWEhISePjhh9m5c6dhW7/88gvdu3ene/fuAIwdO5bu3bszYcIEAG688UZefPFFJkyYQLdu3di0aRNLly49ZTNBbTn+HbPFYqnynXNNP650/KOjSZgyBU3XHT8JU6bgHx1t2N7Jspp82po1I3nBfAKaN6di/34OpIykZNPmyvUu+/cT0Lw5yQvmY2vWzGW/jPbTsM+mTQ3l05TPhATsWVkcHHW7aZ9m5JqmQUmJYw3TkRkvY/9rh5g9I8Ph1370aOV6IpO5remHE9ZNecKnkfwbzu0JnyFndo222RISKpcEREdXrtkaPrzSX0ICcQ89eMpn1uxnqaYfe0YG6ePGVRmf6ePGYc/IcCt/Ru6BPTPToz6d6dizsjgy42UqDhxw3LvQHmdXzXXKSMdny9kYcPb7b4fyuOW1dRwtLKNNrLG3vFgM5rNclfPYD4+xMn0l7+54l735e2uVf0/ru9Lx1Bzk6c+it3NXV/l3pWe2rTYyd34M1ybuFjUAaWlpLFu2jGXLlmG1Wrn88sv5/fff6dixI9OnTzdko1+/fo51BSf+zJ8/36EzZswY9u3bR2lpKWvXruXcc8+tTdh1QkV2NmknHB0CkDZ+PBXZ2V7x59+sWeWC5MREyg8cYN8tt1B+4AD+iYmOhcoNxWfL+fPwi4mh/OBBr/v0j4+vtNuiBRVHjrB/5CiKNv7KvttSqvr14BpJ/6ZN69wn1H1uHX6bNiVqZEoVWbOpU6t8he5JytPSqubynXf+/gzflkJ5WprHfVZkZ7N/5Kg682kJDsYSFop/ixZV7p1/06bEPvwQ/i1a4BcVhaUWr89bu+cow1/9mdzics5u2YSnexp752b50aMudYrKixizfAwrD64kwBLAjH4zaNOkjduxCsLpiukirby8nI8++ogrr7ySpKQkPvjgAx566CEOHz7MggUL+Pbbb3n//fd58sknvRGv19D1UzcOOPtxplOalkbmjJcpO3wYv5YtSVy4EL+WLSk7fJjMGS9TmpZmyN7JMldxWePjafrssyiLxfHT9NlnscbHu+yPO/006tNIPk35jIsjMuU2t3yakR+XWePjaTHvDayxsZQdPszeESMoO3So8t4umF+r/DrLq6d8msm/kdwasWumrTQtjaML3qzi8/C//kX50aO16kt1P6VpaexNGVkll4HdupK4YH7lGD10iL0pI6uM0Vr7TE+vOh94yKczHS0khJgxY2gxf94p984aGUmL+fNoPvd/aCEhhsbAyb+v3J5Byrx1FJbZOb91NG8NbULY6onGJlubzWm/cktyuWfZPfyc9jNBfkE8ffbT9GnWp1b3wNNzkBEdT85Bnvos1kXu6ir/rvTMttVG5s6PUUyvSWvWrBm6rnPzzTezbt06unXrdopO//79adKkiVnTdYq3Ng5U5OSQOX165aLn3ucT8dBDFERFETRjOvkzZlBos/HH5MnEPfyw4zDN6uydLDMSV0V2NlknLBIG2DlnzikL+43iCZ9GbJj1WZ6dzaGvluJ/xhmVRxcY9GlGfkr+NQ2/m26k8MgRh8+4Rx+psuDb03jCp9n8u8qtEbtm2qpsEuh9PtEplZtsCo4erbKZwJ2+VJuPkhKKOndCT0qqumDf8vdmgvKwUI6WlGDJzPSIz4riYkqTWgKKiIc849OVjq7r5JWWogUGYjnhs+K4LiICS3ExFBfXaK+mOWjlrhz+/dVeKnRFn1YRvHBuIYFv34xWko1CQ6vxnQOgrAHkaMHoNXx+88ryGLdhHDvzdhLiF8LT3Z+mudbc8EG07uTKHX0j+ZeNA+7ZMKrvqTmotjJ38OrGgenTp3P99dc7XfjWpEkTUlNTzZquU7y1ccAeFERZfgGgVVn0TFwc0RMn8sfkJ2mSX0B8QkKV4wxqu2j0xAXexxcJp40fT/mOHVUW9pvBEz6tcXEeXTRanp7OvofHOg7kTPhrobkRnzXZNpL/0rQ00t9bRMiff6L99a+gsslP0tSNvBrFEz6NfJ6PYyS3zhaaG/F5Yps9K6vy83PSJoGmrVpV2UwQN38e/n89DfLEot3YiROrPzomLo7YJyefctSIRxZO33wzUYGB2E6+b276dKVj5rNek7y6OejrHdk8vWw/dl1xeeemTO9TRsCiUWgluahmXVFDZqKUk6cEwVHERCTW2LztwDZ25e0i0hbJ7IGzad+kvamDaKvD7D305fzLxgFjembbaiNzB68eZjt06FCKiopOcZKdnY2fnx/h4eFmTfoEJy7mM3JYXU06logIEufMJuDwYcei5+PYmjUj7uGHiE9IqPLGgZrsnSyryWd5errjUMyAxESSFszH/6+F4PtuS6k8yDJlJElvvWm6oKitz8Q3FxjKpymfhw5VOZDTjE8z8uMye0YGB0fdjj0khICEBJo/+yyH//nPv/PqhfVa5WlpHvNpJP9Gc3viZ8iZXSNtWmgo/n89TY44YZPA8c0EhRMn4R8QgF9oqKmx6QpLRAScNP6OY6smp57waQ0OxnbSG0hq69OVjifHgMVi4d11+3ny630o4NqzW/B8j1ys79wE5YWQeC7a8A/QAqvPq1EuTrqYZy54ho7RHWnTpI3jj2Jt8++pOciMjqfz704/PIGv5t+Vntm22sjMYuZa015uuukm3nvvvVPk77//PjfddJNZcw0Sa1hYte8FBPCLjPTYUQ3HsYSE4BcVdcoC7xMX9td2kbDbPoODPe+zRYvKxc9/FQve9FmekVFZpBw8iF9MDC3nzyP47O5VN0zclkJ5errnfKan17lPw7n14GfIGhZG4muv0nLB/FO+jvdv2pTYsQ979GgTwX1e+2EPj3+yFQWMOLclL3TNwPru9ZUFWquL4NaPwc0CbX/efjIK/z6wfEibIbJJQBD+wvSTtLVr1zJt2rRT5P369ePxxx/3SFD1QXULA53pOtOpqd2M/GSZM59aSAjN5/4PvajIsUj4ONb4eBLfXOA4id/MgkVP+VRFRYb9GvFZUVjIMU0z7dOd/BMYiDU6Gj+LhbCHHnSc1G6Nj3ecXG+NioK/Dnj1CEFBHvNp5PMM5nLrapyYadNCQrAGBaGysk7Jv7VJkyqfWaN98SSe8GnWhi/NQbqu85/lO5mxvPIsyxE94pnUdifaortBL0edcRnqunngFwhu5GjXsV3c8+09hPmH8cagN4gKrFqsN/b8G/0b4C18Nf+u9My21UbmDmauN12klZaWUlFRcYq8vLyc4r8WoZ4OeO2NA07azchPlhmJC02rfkG5plUuEDZ5fzzhUy8s9PyiXSD32LFTdVz4dCv/ERHYpk7B76/T/6ssYLZYCHp5BpbAQI66kV9neMqnoXt4oj7Oc+tsobkRn0Y+62Zk3sYTPk3fAx+Zg+x2O9O/S+XDLbkA3HVeM0YGfId18dNoSqe4zeXkXvQ8ZOcBeWZSAsDOvJ2M+2UceeV5hFnDyMrKosJW9W9LY86/W38DPIyv5t+Vnq/PQV7dONCrVy/mzp3LzJkzq8jnzJlDjx49zJqrN7z6xoEa2s3IT5YZicvTeMKnWRu+nP9qFzCbeIODWTzh0xv5d6Vntq02Mm/TWMeArismfr7VUaA9fvmZ3G5bgfWrJwFQ3YZju/Jl4izGzkY7mU2Zm3jsl8coKC+gU3QnZg+YTYTt1K9LG2v+q5PJ59+4nq/PQV7dOPD0008zcOBANm/ezIABAwBYvnw569ev55tvvjFrzmcwuzhTFo16z4bk33W83rTR0BftmsVX74G3xoCu4J+Lt/DRxoNowNPDOjFc/xy+egIA1etutMueQ3MzHz+n/cwDKx6guKKYHvE9eOXiVwgNCHW7n0Y4nfIvc1DDn4PMXGvaS58+fVizZg2JiYm8//77fP7557Rt25bffvuNCy+80Kw5QRAEwUcoq9B58L1NfLTxIFaLxsRBSdxc/C58U1mgFXS/BzXoWXDzD9Tqw6sZ/e1oiiuK6Z3Qm9kDZzst0AShsWP6SRpAt27dWLhwoadjEQRBEOqJkgqdexdu5LsdWfhbNf5zY1f67n4Ry+Y3AND7P0FB+1sJ1mp+yXlaQRo5pTk1tof6hxIXHMcZkWfwwkUvEGAN8Hg/BKEh4VaRpus6u3btIvOv07FPpG/fvh4JrK7R9dN3d6e38IRPszYk/67j9aYNo/rO9My21UbmbXz1Hnh6DOQXl/HIJzvZcLAAm5+FOcO7cdHOZ7Fsnl953aCp6Ofcfcou3BNJK0xj6CdDKdPLaow7wBLAvEHzaB/VHj/Nr1afM6OcDvmXOci8/uk8B5m53nSR9vPPP3PLLbewb9++yiMKTkDTNMduSV+nQe7u9DCe8GnWhuTfdT+8acOovjM9s221kXkbX70HnhwD+SUVPPzJLrakFxLsb+GlIcmc9+s/sfz5KQqN3L5PUtLqGvTMTKc+9+TtcVqgAZTpZeQeyyVHr/lpm9lceNqGzEGu++FNG41hDvLq7s5//OMf9OzZky+++IJmzZqhOXn07cvI7k7XeMKnWRuSf9f98KYNo/rO9My21UbmbXz1HnhqDBwtKOXBRevZll5ImM3Kgtu60P2Xf6L9+RlKs3Ls4ucIO38U4QbGwFHrUUP5iIqMIi7a2A7lhp5/Z3KZgxruHOTV3Z07d+7kww8/pG3btmYv9WnM7qCRnT3esyH5dx2vN2009J1VZvHVe1DbMZBVUMbw19axK7OA6JAA/jOkOd3X3I+261uwBqCum0dpZE8iDI4BzWLsH+yaRTOVy4aaf5mDaq9/us5BZq41XaSde+657Nq1q8EVaYIgCA0Ru65Yl5pNZn4JsaEBJAUrDuYUMeL19ezPLqJpeCALb+1Ai69S0A6vA78guPkdaNWv+oOqBUGoM0wXaffffz+PPPII6enpnHXWWfj7+1dp79Kli8eCEwRBENxn6ZZ0nvriD9JySxyy6GA/7GgcKyonMSqId0ecSfMvbkU7/AsqIAxt+PuQ1BvqcIG6IAjVY7pIu/baawG4/fbbHTJN01BKnVYbBwRBEBoy3+3K4V9L9qBOkh8tqnz1Uny4jY9ubUfcJzdAxu/otiYw4iO0xJ51HqsgCNVjukhLTU31RhyCIAiCh7DriukrD5xSoJ1IrDpK7EfXwJE/USFxZF/+GlHNz66zGAVBcI3pIi0pKckbcdQ7ui7npJ2MJ3yatSH5dx2vN20Y1XemZ7atNjJv46v3wJXO2j1H8StIo5NW/Vb/eC2bp0rnoR3JRoU3xz58MeV6hFtjoLiiGItmISIgggBLgMtz0iICIjz+efSkDZmDXMfrTRuNYQ4yc71bh9m+9dZbzJkzh9TUVNasWUNSUhIzZsygVatWXHXVVe6YrHPknDTXeMKnWRuSf9f98KYNo/rO9My21UbmbXz1HrjSSdu9hRW2RwjUyp36yrfFUzzkLSrs4eTmHjM9BgorCvn3xn8T6hfKhG4TmHfBPHLLc2v0F+EfgbXQSmahsQ0Jp2v+ZQ5y30ZjmIO8ek7a7NmzmTBhAg899BDPPPOMo8hp0qQJM2bMOG2KNDknzTWe8GnWhuTfdT+8acOovjM9s221kXkbX70HrnTaRlpdFmgA+3tPpUOb7m6NgdzSXB5a/hBbc7YS6h9KcWAxnZt2dunTDKdr/mUOct9GY5iDvHpO2syZM3n11VcZNmwYzz77rEPes2dPHn30UbPmfAazZ9HIGTnesyH5dx2vN2009DOKzOKr98CZTufmEYb8nNm2tdPPd01jILs0m7uX3c3OnJ00sTXhf5f8j3ZR7Qz5NMvpmH9n7TIHeU7/dJ2DzFxr2ktqairdu3c/RW6z2SgsLDRrThAEQfAwy//IMKRn1cy/MSa9MJ1RS0exM2cnsUGxzBs0j47RHU3bEQTBNaaLtFatWrFp06ZT5EuXLqVDhw6eiEkQBEFwk9d/TGXmd7u9Yvtw0WFGfT2KvXl7aRbSjPmXzadtpBxsLgjewvTXnWPHjmX06NGUlJSglGLdunW8++67TJ06lddee80bMQqCIAgGeGXFTl785k86eemVysfKjpFdkk1SeBKvXvIqzUKbeceRIAiAG0XanXfeSVBQEE888QRFRUXccsstJCQk8PLLL3PTTTd5I0ZBEATBCUopnv96B7NXVj5Bu6VXImz2vJ+OTToye+BskiOSiQmK8bwDQRCq4NYRHMOHD2f48OEUFRVRUFBAXFycp+Oqc3Rdzkk7GU/4NGtD8u86Xm/aMKrvTM9sW21k3sZX78GJOrqueHLJH7z58z4A/jX4TG4qe9uYL6WghnnvuGxT5iZsfjbaN2mPUopusd0cuw29zemQfzPtMgd5Tv90noPMXG+6SLv44otZvHgxTZo0ITg4mODgYADy8vIYNmwYK1asMGuyXpBz0lzjCZ9mbUj+XffDmzaM6jvTM9tWG5m38dV7cFynwq7z3IoDLNl2FA34v4tbclPJB1jXTHPpR1kDOFKk0DMza8z36gOreW7nc9gsNqadM42wijDJvwEdmYPct9EY5iCvnpO2cuVKyspOPVG6pKSEH374way5ekPOSXONJ3yatSH5d90Pb9owqu9Mz2xbbWTexlfvga7r2HV49vs0vth2FIsGL1x7FlcXvItl5VQACjvchO38u7BYrVWuy8nJITIyEktoDDERiTX6/P7A90z9cyplehndYrvRIbEDBTkFkn8DOjIHuW+jMcxBXjkn7bfffnP8/7Zt20hPT3f8brfbWbp0Kc2bNzfs2NcwexaNnJHjPRuSf9fxetNGQz+jyCy+eA9Ky+2M/3IPP+7Jxd+q8Z8buzE4Yw789DIAer/HyW9/K0Hx8VVt6Dp2/0wscXFOx8Cyfct4bNVjVOgV9GvRjxf7vYi/5k+hVij5N6gjc5D7Nhr6HGTmWsNFWrdu3dA0DU3TuPjii09pDwoKYubMmYYdC4IgCOYpKqvgzjc3sHpPLjY/C3NGdKf/7hdg/V+76wdNgXPvhUxjr146mc92f8a/f/o3utLp17QfL170IjarrU7XQgmCUInhIi01NRWlFK1bt2bdunXExsY62gICAoiLi8N6wmN1QRAEwbPklZRz+7z1/LIvhyB/C6/f2o3eWyfD5ncADa6cDj1HgZsF1coDK3n8x8cBGNZmGP9o8w/8Lf6e64AgCKYwXKQlJSUB5nYlCIIgCJ4hu7CM295Yy5ZDeYQH+jF9SEvO3zQOtn0CmhWungNdbqiVj15Ne9E9rjsdozvyaI9HOZJ1xDPBC4LgFm4dwQGV69L2799/yiaCoUOH1jooQRAE4W8y80oY8fpa/swoIDokgDdvPYs2K+5B278SrAFw3RvQYYhbtpVSKKUACPYPZu4lc7FZbQ6ZIAj1h+kibc+ePVx99dX8/vvvaJrmGMjaX++AO36khSAIglB7DuYUMeK1tew9WkR8uI13UjrTetldaPtXofwC0W5aCG0HumVbKcWLv7xIiF8I1zS7BoBAv0BHmyAI9Yvp7QkPPvggrVq1IjMzk+DgYLZu3cqqVavo2bMnK1eu9EKIgiAIjZPUI4XcMGcNe48W0SIyiA9HdqLNV7ei7V2F7h+MuuVDtws0Xek89fNTvLntTWb/Nptdebs8HL0gCLXF9JO0NWvWsGLFCmJiYhzbUC+44AKmTp3KAw88wK+//uqNOAVBEBoVO9LzufWN9RwpKKV1bAjv3tKG+E9vgPTfUIFNyBn8KpHJfdyyXaFXMHHNRJbsWYKGxsTzJ9I2XF6ULgi+hukizW63ExYWBkBMTAyHDx+mffv2JCUlsWPHDo8HKAiC0NjYnlHIQ5/8xrHics5sGsbCG5OIXnwdZG2HkFjUiMWUa85fx5dZnMnRo0fRLH+/bV3pisxjmby1+S3Wp6/HqlmZeuFUBiUNItPNIzsEQfAepou0zp07s3nzZlq1asW5557L888/T0BAAHPnzqV169beiLFO0HV5d+fJeMKnWRuSf9fxetOGUX1nembbaiPzNvVxD9btOcroj/6ksEyna4sIFlwdT8T7V0FOKiosAXXrJ+hRbVBZWTXaPJR/iJE/jqRcL3fq64lzn2BQ0iAZAyb1ZQ7yno3GMAeZud50kfbEE09QWFgIwJNPPsmVV17JhRdeSHR0NIsWLTJrrt6Qd3e6xhM+zdqQ/LvuhzdtGNV3pme2rTYyb1PX92Dd/jwe+2w3JRU63RJC+c9FGmHvXIlWmE5FeCI5V87DrkegZ2Y6tbnn2B6XBRpAvBZPppN3d8oYkDnIF/PvSs/X5yCvvrtz0KBBjv9v27Yt27dvJzs7m8jISMcOz9MBeXenazzh06wNyb/rfnjThlF9Z3pm22oj8zZ1eQ+W/5HBo5/uosyuOC8pnNcvDyX4/RvQCjNRMWdgGfEx0eEJhmxmWbIMxRYVGUVcdJyMAZP6Mgd5z0ZjmIO88u5OZ0RFRXnCTL1i9v1o8t4279mQ/LuO15s2Gvp788xSF/fg882HeXjRJip0xaUd43nyrGyC37sVrTgHmp6FNuJjtNBYwzYtmsF7bdGcfr5lDMgc5Kv5d6Xny3OQmWsNFWnXXHONYYOLFy82rCsIgtCYsOuKdanZZOaXEBcWSK9WUXy04SD/XPwbSsGwbgm8cE4B1vdGopUXQotzYPgHEBRZ36ELglAPGCrSIiIivB2HIAhCg2bplnSe+uIP0nJLHLLwQD/ySioAuLlXS545Kwvt3VvQKopRSReg3fIe2MLqK2RBEOoZQ0XavHnzvB2HIAhCg+W7XTn8a8keTj7D/3iBNuDMOKZ02If23iiwl1Ga2Bf/W95Ds4W45e9QwaFaRiwIgi/gkTVpgiAIQvXYdcX0lQdOKdBOpPnBL2Dff0DZUR2GknPBM8T5B7nlb0f2DiatmeTWtYIg+BZSpAmCIHiR9Xuz8StIo5NW/bb7Sy2/cH/5x2ga0OUm1NCZcCTbLV9bjmzh7mV3k1+ej4aGclIaBlgDiLTJWjdB8GWkSBMEQfAi+RmprLA9QqDm/Nyy/S2upOWw2bXy1cTWhCC/INpEtOG+dvcR3iT8lDcOZOdkExUZRVRQFM1Cm9XKnyAI3kWKNEEQBC/S1K/QZYEGcOysu2hpsUAtTjNvEdaC+YPmE2mLpCCngLjouFPO48q0Z54iFwTBN5FRKgiC4EXaxYUa0uvUwr1d9N/u+5YV+1c4fk8MTyTYP9gtW4Ig+BaGnqT95z//MWzwgQcecDsYQRCEhkRRWQVTvtrO0wZ0rW68sWXJniVMWD0Bi2Zh4eUL6RDdwXyQgiD4LIaKtOnTpxsypmmaFGmCIAhAfkk5t89fT9GBXLB53v6SA0v4z7b/oFBc2fpKzog8w/NOBEGoVwwVaampqd6OQxAEocGQU1hGyrx1/HYwl16BVo/bf3Pbm7y87WUAbj7zZsb1Gmf4VVCCIJw+yKgWBEHwIJn5Jdw092d+O5hLVEgAU4a085htpRSzN8/mpQ0vATCq0yjG9xovBZogNFAMPUkbO3asYYPTpk1zO5j6RNd1x49SCt3JDitXOjW1m5GfLDMSl6fxhE+zNiT/ruP1pg2j+s70zLbVRuZtzPo8fKyYEa+vY+/RIuLCbLx9aydaf3WzMV9KgYs5aPn+5fx3038BGNl2JPd3vR+lFEpVPQ9NxoD7NmQOch2vN200hjnIzPWGirRff/3VkDHNjYWv9cWsWbOYNWsWdrsdgKysLEpKStB1ndzcXJRSNW5Rd6VTU7sZ+ckyI3F5Gk/4NGtD8u+6H960YVTfmZ7ZttrIvI0ZnweOlXD/RztJzy+jaVgAs4c0JfnLW7Ck/4oCnM2OyhrAkSKFnpnp1GcnWycGJgykTWgbBjQZQFZWlowBD9uQOch1P7xpozHMQfn51R9sXR2GirTvvvvO7WB8ldGjRzN69Gjy8vKIiIggNjaW8PBwdF1H0zRiY2OdDlBnOjW1m5GfLDMSl6fxhE+zNiT/rvvhTRtG9Z3pmW2rjczbGPW5MyOf0R9tITO/jOToYN65pQ3NloxAS9+MCowgp+8zhCd2rNlGcBQxEYnV+qzQK1Ao/C3+ALwY9yJKKbKysmQMeMGGzEGu++FNG41hDgoMDDSsK4fZ/oXFYnEkXdO0Kr9XhyudmtrNyE+WGYnL03jCp1kbkn/X8XrThlF9Z3pm22oj8zaufG45lMutr68lp6ic9vFhLLwpiZjFN0DWHxAcgxqxmDJLPJY44wfIHvdpV3bG/TgOP82PqRdOxWqp3IRw/I+FjAHv2JA5yHW83rTR0OcgM9e6VaT98ssvvP/+++zfv5+ysrIqbYsXL3bHpCAIwmnHhn3ZjJy3nvySCrq0iOCta5oR8cEwyN4DYc3gtk8huh1kZpq2XVJRwqOrHuWHQz/gb/EnJSeFTtGdPN8JQRB8FtOl4HvvvUfv3r35448/+PjjjykvL2fr1q2sWLGCiAj3TswWBEE43Vi96wi3vr6O/JIKzkmO5J1rYoh4b2hlgdakJYz6CmLbu2W7uKKYMSvG8MOhHwi0BjLz4plSoAlCI8R0kTZlyhSmT5/O559/TkBAAC+//DLbt2/nhhtuoGXLlt6IURAEwadYsT2DkfPXU1Rm58J2Mbx5ZRih7wyFvIOVT85GLYWoVm7ZzivLY9yGcazPWE+IfwizB86mT/M+Hu6BIAinA6aLtN27d3PFFVcAEBAQQGFhIZqm8fDDDzN37lyPBygIguBLfPFbGne/uYGyCp1LOsbz+qV+BC0cAgUZEN8ZRn0JEc3dsp1dks2d39zJtmPbCA8I57VLX6Nn054e7oEgCKcLpou0yMhIx/bR5s2bs2XLFgCOHTtGUVGRZ6MTBEHwIT7ccJD7391Iha4Y2jWB2X3LCHh7GBTnQPMekPI5hMa5bT81N5XU3FSaBDTh9Utfp3NMZ88FLwjCaYfpjQN9+/Zl2bJlnHXWWVx//fU8+OCDrFixgmXLljFgwABvxCgIglDvvLVmL//+dCsAN52TyDNdj2JdeAuUF0FSH7hlEdjCauWjR3wPZvSbQWBZoLyLUxAE80XaK6+8QklJCQCPP/44/v7+rF69mmuvvZYnnnjC4wEKgiDUN/9btYfnlu4AYFSfZCa024f2bgrYy6DNALjxbQgIrvH6zOJMjh49imY59UjbwwWHCfUP5byE8wDo07wPmW7sBhUEoeFhukiLiopy/L/FYmHcuHEeDUgQBMFXUEoxd81h3libBsCY/m15JGEL2vt3g14BZ14J170BfrYabaQVpjHyx5GU6+VOfc0fNJ8eTXt4NH5BEE5vanUi3hVXXEFaWpqnYhEEQfAZlFI88+V2R4H22GXteTR2HdpHd1QWaGfdANcvcFqgARwrOeayQAMM6QiC0Lio1RsHVq1aRXFxsadiEQRB8AnsuuKJT7bw7rr9AEwa0pGRft/AZ49VKpydAldOh7/eAOAJwm3hHrMlCELDQF4LJQiCcAIVdp1HP9jMJ5sOY9HgXwOTuM3+ESx7qlLhvNEw6BnQnL0yXRAEofbUqkhLSkrC39/fU7EIgiDUK6UVdh5491e+3pqBn0Vj2vVduHj/TCy/zqlU6PsY9P+XFGiCINQJtSrSjp+RJgiCcLph1xXrUrPJzC8hLiyQs5pHcN87G1n1ZxYBfhb+e3N3Buyfjna8QBs4GS54qF5jFgShceF2kbZhwwb++OMPADp27MjZZ5/tsaAEQRC8ydItaUz+fBtpuSUOWYBVo8yuCPK38uqI7lyw/Sn49S0A9MEvYDn37voKVxCERorpIi0zM5ObbrqJlStX0qRJE6DybQP9+/fnvffeIzY21tMxCoIgeIylW9K49+2NqJPkZfZKyf0XJXHB7/+CLR+iNAu5/aYQfs6dbvv7MvXLWkQrCEJjxvQRHPfffz/5+fls3bqV7OxssrOz2bJlC3l5eTzwwAPeiFEQBMEj2HXF5M+3nVKgHcdGGZ1WPwBbPgSLH+ra1ylpf7VbvpRSzP1tLm/+8aZL3QBrAJG2SLf8CILQcDH9JG3p0qV8++23dOjQwSHr2LEjs2bN4tJLL/VocIIgCJ5kXWo2Wu5BOmn5p7QFUsbjfgs5W9+FbgnActPb0PYScOP0f6UUMzbO4I0tbwBwbdK1XNfxOizW6v9dHGmLpFloM9N+BEFo2Jgu0nRdr3ZHp7+/P7queyQoQRAEb5CfsYcVtkcI1Go+OFYp2HD2M5xzxiBwY07Tlc6z657l3e3vAvBIj0e4LOYy4mLisFhqdX64IAiNDNMzxsUXX8yDDz7I4cOHHbJDhw7x8MMPywvWBUHwaZr6FTkt0KDydA1bXHu37Nt1OxN+msC7299FQ+Pf5/2b2zre5pYtQRAE00XaK6+8Ql5eHsnJybRp04Y2bdrQqlUr8vLymDlzpjdidMnVV19NZGQk1113Xb34FwTh9CAowNiU16m5e6f//3T4Jz7d/SkWzcIzFzzDDe1vcMuOIAgCuPF1Z2JiIhs3buTbb79l+/btAHTo0IGBAwd6PDijPPjgg9x+++0sWLCg3mIQBMG32bg/h+c+2cIiA7pWNw+r7duiLw+e/SBJ4UlcknSJWzYEQRCOY7pIe/PNN7nxxhu55JJLuOSSvyehsrIy3nvvPW67re4f7ffr14+VK1fWuV9BEE4P1uw+yh0L1tOq3A7O34dumqLyIipUBeEBlU/f7jzL/eM6BEEQTsT0152jRo0iNzf3FHl+fj6jRo0yHcCqVasYMmQICQkJaJrGJ598corOrFmzSE5OJjAwkHPPPZd169aZ9iMIQuPkux2ZjJy3jqIyO90Tm3jUdn5ZPvcsu4d7v72XovIij9oWBEEwXaQppdCq+Srg4MGDREREmA6gsLCQrl27MmvWrGrbFy1axNixY5k4cSIbN26ka9euDBo0iEw3tsULgtC4+Or3NO5+8xdKK3QGdohjwnm1ehNeFXJKcrjj6zvYlLWJ1NxUDuQf8JhtQRAEMPF1Z/fu3dE0DU3TGDBgAH5+f19qt9tJTU3lsssuMx3A4MGDGTx4cI3t06ZN46677nI8pZszZw5ffPEFb7zxBuPGjTPtr7S0lNLSUsfveXl5QOXRIsd/lFJOjxNxpVNTuxn5yTIjcXkaT/g0a0Py7zpeb9owqu9Mz2xbbWTO+PjXQ/zfh7+hK7jyrGZM612C/ztjDV2rKwVO5oOsoizu/vZu9uTuIdIWyf8G/o92TdqZzke1vmUMuIzXmzYk/67j9aaNhjQHOYvdKIaLtGHDhgGwadMmBg0aRGhoqKMtICCA5ORkrr32WuNRGqCsrIwNGzYwfvx4h8xisTBw4EDWrFnjls2pU6cyefLkU+RZWVmUlJSg6zq5ubkopWo808iVTk3tZuQny4zE5Wk84dOsDcm/635404ZRfWd6ZttqI6uJxb9l8fyK/QBc2TGaiWfsw2/hfWgVRSg0tBrfOQDKGsCRIoWemVmtz4ziDB5b/xiHiw8TbYvm+Z7PE1kRWePTfRkD7uOrY0Dy7z0bDWUOckZ+/qmHadeE4SJt4sSJACQnJ3PjjTcSGBhoPjKTHDlyBLvdTnx8fBV5fHy8Y2cpwMCBA9m8eTOFhYW0aNGCDz74gPPPP79am+PHj2fs2L//NZ2Xl0diYiKxsbGEh4ej6zqaphEbG+t0gDrTqandjPxkmZG4PI0nfJq1Ifl33Q9v2jCq70zPbFttZNXx2g+pjgIt5fwkJpxxAMuH96DZS1Gt+6MGPYuqKK45CcFRxEQkVutzX94+Hv3hUdKL02ke2py5A+fSIqxFzbZMxG1GX8aA92xI/l33w5s2GsIc5Aoz9ZPpBRopKSmO/7/vvvt48skniYmJMWvGo3z77beGdW02Gzbbqdu7LBaLI+maplX5vTpc6dTUbkZ+ssxIXJ7GEz7N2pD8u47XmzaM6jvTM9tWG9lxlFK8vHwnM77dCcB9/drwfy3+QPvgTtAroP3laNfNQ/M39w/ME30qTVGml5Ecnsyrl75K05Cmpm14Sl/GgPdsSP5dx+tNG6frHGQUM9fWahXt22+/zaOPPuq1Ii0mJgar1UpGRkYVeUZGBk2bGpscjaLrsibtZDzh06wNyb/reL1pw6i+Mz2zbbWRHUcpxbNLd/DqD6kAPHLpGYxushY+uh+Ujup8Leqq2WD1BxP5PNlnclgycwfOJSowiuigaEN5lTHgPr46BiT/3rNxus5BZjBzfa2KNKVqXtfhCQICAujRowfLly93rInTdZ3ly5czZsyYWtmeNWsWs2bNwm63A7ImrTo84dOsDcm/635404ZRfWd6ZttqIwPQleKFFfv5+PcjADx8UQtu0z/D8tmTABSdeR15vZ+Eozku+19dP9ceXIv/EX96xvYEIIII7OV2MvON7TCXMeA+vjoGJP/es3E6zkFm8cqaNG9RUFDArl27HL+npqayadMmoqKiaNmyJWPHjiUlJYWePXvSq1cvZsyYQWFhoVtnsp3I6NGjGT16NHl5eURERMiatGrwhE+zNiT/rvvhTRtG9Z3pmW2rjazCrvPPxb/z8e9H0DSYcnVnbixdjGV5ZYGmet1D4KApBGru5W/1odU8s+MZNE1j/qD5dIjuYNqGjAH38dUxIPn3no3TbQ5yB6+uSTsRM9VgTfzyyy/079/f8fvxRf0pKSnMnz+fG2+8kaysLCZMmEB6ejrdunVj6dKlp2wmqC1mv/eX9QjesyH5dx2vN22cLutByip0Hly0ma+2pGO1aEy7vgtX5SyAVc9XKl/4KNrFT6Bp7r3i6bv93/HI949QrpfTJ6EPrSNbu30fZAy4j6+OAcm/92ycLnOQu5i51lCRdvwsMSOEh5t7MXG/fv1cfm06ZsyYWn+9KQhCw6Gk3M4/3t7Ayh1ZBFgtvHJzNy49OBN+/utQ7AET4MJH3Lb/VepXjP9hPHZl54K4C5jebzqBft7f0S4IgnAihoq0Jk2auPzXqFKVbyI4vsbrdEPXZePAyXjCp1kbkn/X8XrThlF9Z3pm28zK8orL+Mfbv/JzajaB/hb+d0s3+u6cChsXVOpd9hz0uhvczNvHuz5m8prJKBRXJF/BmLZj8NP83L4PMgbcx1fHgOTfezZOhzmotvfAzPWGirTvvvvO7WB8Fdk44BpP+DRrQ/Lvuh/etGFU35me2TYzsoOZ2UxetJ2t6UUEB1iYPiSZXhv/D23n5yjNQt5FT1GcPAycvDYusziT3PJT3z8MsPXYVmb9Ufk07ooWlQVafl4+Fs39rzdkDLiPr44Byb/3bPj6HOSTGwcuuugit4PxVWTjgGs84dOsDcm/635404ZRfWd6ZtuMyrLyS3h8+XZ2ZhXTJMif+bedRde1j6LtXIKy+KGu/h9hna4hzEn/0grTGLVsFGV6WY06FiwMazuMCedNQCmF1WL1uXsgY8B7NiT/rvvhTRu+PAd56h54fOPAb7/9Zthgly5dDOv6EmYXZ8qiUe/ZkPy7jtebNnxx0W56bgnDX1vH7qxiYkIDWJjShfYr74Xdy8EagHb9ArQzL3fZt9yyXKcFGoCOzg3tb8BqtTomZV+8BzIGvGdD8u86Xm/a8MU5yN2+VIeZaw0Vad26dUPTNJcL/E/nNWmCIPgmB7KLuOW1nzmQXUxcqD/vjuxMm2WjYN9P4B8MN70Dbfq7NmQCV2twBUEQ6gJDRVpqaqq346h3dF02DpyMJ3yatSH5dx2vN20Y1XemZ7bNmWxXRj63zVtPel4pLaOCmHlZNK2/vAUOb0TZwlA3L4KW54PB/ind2AHcSleG5wNXyBhwH8m/5N8dPU/OQa5k7mDmekNFWlJSktvB+CqyccA1nvBp1obk33U/vGnDqL4zPbNtuq6TnXOMDQfyyC6yEx3iT5dmwfy+L4vHv80gp7iCVlGBvHJZE9osvx0tdxe6rQnZV75ORWAbp5sETiY7L9uYXk42mfZMn70HMga8Z0Py77of3rRRX3PQab1xoDq2bdvG/v37KSurur5j6NCh7pqsU2TjgGs84dOsDcm/635404ZRfWd6Ztu++j2NyZ8fJLOg3KEXHRJAYWkFJRU6nRLCefPaBKIW34CWuwsVGg8jFhMV19FMKgA4aj1qSC8qMoq46DifvQcyBrxnQ/Lvuh/etFEfc1BtZO7g1TcO7Nmzh6uvvprff/+9yjq142s4Ttc1aWYXZ8qiUe/ZkPy7jtebNupy0e7SLWmMeXcTJ38JebSw8h9/rWKCeff6poQvGgbH9mMPbYaW8jmW2HaG+nIypXqpIT3Nonn0vssYcB/Jv+TfHb2GsnHAtJcHH3yQVq1akZmZSXBwMFu3bmXVqlX07NmTlStXmjUnCEIjxa4rJn++7ZQC7UTiSvcR9s4QOLYfFdWao1cthOg2bvvck7vH7WsFQRDqGtNP0tasWcOKFSuIiYlxVJMXXHABU6dO5YEHHuDXX3/1RpyCIDQw1qVmo+UepJNW/fqM1tphJpctQCsvgNgOqBGL0Ytr9wShT0If/Cx+VOgVNeoEWAOItEXWyo8gCIInMF2k2e12wsIqj4uMiYnh8OHDtG/fnqSkJHbs2OHxAOsKXZfdnSfjCZ9mbUj+XcfrTRtG9Z3pGW3LS9/NCtsjBGrlp+idSG5YO8JSPkcPjEQVZZnOx5HiI5RUlNAirAXxwfEsGbaEYyXHatRvEtiE+OB4w/OBK2QMuI/kX/Lvjp7ZttrI3MHM9aaLtM6dO7N582ZatWrFueeey/PPP09AQABz586ldevWZs3VG7K70zWe8GnWhuTfdT+8acOovjM9o23BxRkuCzSArZ3+j9YFdvS8TNP5yCzO5LFfHsOu7EzrNY3YwFisWIkmuuaLCiGzMNNlX4wiY8B9JP+Sf3f0zLbVRuYOXt3d+cQTT1BYWAjAk08+yZVXXsmFF15IdHQ0ixYtMmuu3pDdna7xhE+zNiT/rvvhTRtG9Z3pGW2LtTeHH1334ZxO7bDGmd9puT9vP4/++ChpRWkkhCQQHhlOXFica4cG++ItGzIGXPfDmzYk/6774U0bdTkH1ZRrMzJ38OruzkGDBjn+v23btmzfvp3s7GwiIyMdOzxPR8zuoJGdPd6zIfl3Ha83bdTFzipN0/hw4yFuMhCPv9UCJu/Brpxd3LXsLo4UHyE5PJlXL32VpiFNDXg7FV+9BzIGvGdD8u86Xm/akN2dJ+i67eUEoqKiTusCTRCEukMpxXNf7+Ctn/d5xf62o9sY9fUojhQfoV1kO+ZdNs/tAk0QBKE+cfswW0EQBLPouuKllQf4cHMWnbzw77qtR7Zy5zd3UlBeQOfozsy5ZA4RtgjPOxIEQagDpEgTBKFOqLDr/HPx73y0OQtNgzH928Jqz/pICE2gaUhTwgPCmTVgFqEBoZ51IAiCUIdIkfYXui5HcJyMJ3yatSH5dx2vN20Y1XemV11bWYXO2Pc38+WWdKwaPHdNZwalTTcWk1JgcGxGBEQwd+Bcgv2DCfILqvX98tV7IGPAezYk/67j9aYNb81BztpqI3MHM9c32iJNjuBwjSd8mrUh+XfdD2/aMKrvTO/kttIKnX99sYefUnPxs2iMuzCWyw7MwPLbPJfxKGsAR4oUembNLzv/Pv17CsoLuCLxCocs/6//aouv3gMZA96zIfl33Q9v2vDGHOSqrTYyd6iTF6yf7sgRHK7xhE+zNiT/rvvhTRtG9Z3pndhWXK7z8FsbWZOai83Pwn9v7sJ5W58iZNu7lboXPoZ+xmXk5OQQGRlZJf85OTlENm9LTGTLGn1+uvtTpvw2BaUUXVp0oUd8D7fyVNt8eNKGjAHX/fCmDcm/635404an5yAjbbWRuYNXj+BoqJjd5izbr71nQ/LvOl5v2vDU9veCUju3L/iFjfuPERJg5fXbunPubxPQtr2HQkMb8jKWHimg69j9M7HExf1t67gsMq7Ge/Lu9neZsnYKANe2u5az48/2yr3x1XsgY8B7NiT/ruP1pg05guNvpEgTBMHjHCuu4JH317H1cB7hgX68mdKNbusehT8+Q2lW1NVz0Lrc4Lb9139/nRkbZwAwosMIHjvnMTkGSBCEBocUaYIgeJSMvBLu/WAHqdklRIcE8HZKFzr8MAZ2foOyBnBs4HQiOl/nlm2lFK9seoVXf38VgLu73M2YbmOkQBMEoUEiRZogCB7jYE4Rw19dy77sEpqG21h4W2fafHsH7P0B/IJQN75NaVhnt+1vOLrBUaA9dPZD3HHWHZ4KXRAEweeQIk0QBI+wJ6uAEa+t5XBuCQnhASy67UwSl46Ag+shIAyGvw+J50Fmpts+ekT3YFSnUTQLbcbNZ97swegFQRB8DynSBEGoNdvT8xjx2jqOFJTSJjaE/wxqQoslN0H67xDYBG5dDM17gBvnC1XoFZTZywi0BqJpGg+d/VCdLpwWBEGoL6RI+wtdl8NsT8YTPs3akPy7jtebNozqn6j328FcRs5bz7Hicjo0C+PNaxNo8uENaMd2o0JiUSMWQ3xnqGF8OZOVlJcw/qfx5Jfl859+/5F7YFBHxoD7NiT/ruP1pg135qDattVG5g5mrm+0RZocZusaT/g0a0Py77of3rRhVP+43qZD+Tzy2W6KynQ6NQ1h5sU2IhcNwy//APaQpmQPmY9di3N8xWkk18dlmdmZjN8wng1HN+Bv8WfNnjW00FrIPTCgI2PAfRuSf9f98KYNs3OQHGbbQJHDbF3jCZ9mbUj+XffDmzaM6uu6zvr9+fzzi92UlOuc1yqK166IIOT9a9HyD1MRngi3fUZ0VLJT+2mFaWQXZ5NjyaHCrwKLVumzQC/g2T+fZVf+LgKtgfyn/384J/4csrKy5B4Y0JEx4L4Nyb/rfnjThpk5SA6zbUSYPTBQDjL0ng3Jv+t4vWnDiP6ybRk8+vluyu2Kfu1j+d8lgdjeHQqFWaiY9mQPfpWYqORqbRy3n1GUwdBPh1JmL3Maz5QLp3B+8/MdE6TcA2M6MgbctyH5dx2vN23IYbZ/I0WaIAim+HTTIca+vxm7rhjUKZ6ZfXUC3h4CJcegaRfU8I/QC12vucgpzXFZoAE0D23ugagFQRBOP+quHBcE4bRn0fr9PLRoE3ZdMbhDFK/0LiJg4TWVBVqLXpDyOYTE1HeYgiAIDQIp0gRBMMS8n1L550e/oxTc0iuRpzscwO/dG6AsH5IvhFs/hqAm9R2mIAhCg0G+7hQEoQp2XfHznqNkFZQRFxZIr1ZRzPl+Ny98vQOAuy5sxbjkXVg+ug9NL4d2g+CGBeAfVM+RC4IgNCykSBMEwcHSLelM+mwLmQXlDlmIzUphaeVRNQ8NbMeDcZvgw3+gKTuqw1Vo174GfgH1FLEgCELDRYo0QRAAWLoljdHv/Io6SX68QLvm7OY8FLkaFj8EKIrPuBrbtXPR3CzQ/sz5s1bxCoIgNHSkSBMEAbuumPz5tlMKtBNpuWM+bJsHgOp5B7k9HiXO4t4Usj59PU///LRb1wqCIDQWpEgTBIF1qdlouQfppFV3ErbiRstKbrN/W/lr7wdQAyZBVpZbvtZmreWpzU9Rai9FQ0M5KQ0DLAFE2iLd8iMIgnC6I0XaX+i6vLvzZDzh06wNyb/reL1hIy99NytsjxColTvV29kmhTYDJqEr5dZ78zakb2DSr5OoUBVc1OIiHunxCAVlBeTk5BAZGel444CudHJycmjVtBXxwfGGx6ankTHQeMaAGX3Jv/dsGNV3pme2rTYydzBzfaMt0uTdna7xhE+zNiT/rvvhDRvBxRkuCzSA9GaXEJaV5dSus7ZYPZZO4Z1oEtiEcR3G4Vfih0234af7EVERUSX/frof1kIrmcU1v/fT28gYaDxjwIy+5N97NozquzsHGcm1GZk7yLs7DSDv7nSNJ3yatSH5d90Pb9gIbBUPP7q2d1775ljj4pzaddX2dM+naRbXDH8//xr1jcq8jYyBxjMGzOhL/r1nw6h+beYgeXfnaYjZ96PJe9u8Z0Py7zpeT9rILizjyc+2McOALX+rBQzk48S2BVsXkF6YzmPnPIbFYiHQLxB/P3+nuTYj8zYyBhr+GHBHX/LvPRvy7s6/kSJNEBoxmfkljHhtLf6ZBWDzrG2lFLM3zea/m/8LwIUtLuS8pud51okgCEIDpu7KcUEQfIpDx4q5Yc4a/swoIDrEs4fRKqWYtnGao0B7oPsD9E7o7VEfgiAIDR15kiYIjZDUI4WMeG0th44V0yIyiOcHt4bFnrGtK53//PEflhxYAsA/z/knIzqO8IxxQRCERoQUaYLQyPgzI5/hr60lK7+U1jEhvDOiLU0/uMojtiv0Cv69+t8sObAEDY1JvSdxTbtrPGJbEAShsSFFmiA0In4/mMttb6wlp6icM5uGsfDGZKI/uhaO7nJ9sZ8NgqOdqmzO2syXqV9i0SxM6TOFK9pc4aHIBUEQGh9SpAlCI+GXvdmMmree/NIKuiY24c1rmhLx/lDISYWwBLj6f+i2MLKzs4mKisKiaVUNBEdDk0SnPnrE92DS+ZNQxYrBrQZ7sTeCIAgNHynSBKER8OOuI9zz1kaKy+30ahXFG0OaEPruUMg7CE2SIOUziEwGXafCmglxcY6jNlxRWF5IYXkhccFxAFzV5ioyMzO92BtBEITGgRRpgtDA+WHPMR7/MpWyCp2+Z8Qy99IgAhcOhcJMiDkDbvsUwhPcsp1XmsfoFaPJK8tj/mXziQ5y/nWoIAiCYBw5gkMQGjBLfktj3JLdlFXoDOoUz2sDLQQuHFJZoMWfBSO/dLtAyynN4c5ld/Lbkd/IKc0hs0iengmCIHgSeZImCA2U9385wLiPfkNXcFW3BF46twi/t2+Esnxo3hNGfAhBkW7ZzijK4JH1j3Cg8ADRgdHMvXQuZ0Se4eEeCIIgNG6kSBOEBsiC1XuZ+NlWAK7qHMNLZx/Fb+EIqCiGpAvglvfAFuaW7YP5B7nzmzs5VHiIpsFNefXSV0mOSPZg9IIgCAJIkSYIDY7/rtzF80t3AHB7n2TGxGzCuughsJdB24Fww1sQEOyW7b25e7njmzvILMokISiB1we9TovwFh6MXhAEQTiOFGl/oeu640cpha7rTnWd6dTUbkZ+ssxIXJ7GEz7N2pD8u463JpRSTFu2k1krdwNw/8VteTBuE9ZP70dTdtSZQ1DXvFp53pnJ3B0nyBqEzWqjdURrnun6DE2DmxrOc01ttZF5GxkDp9cY8IQNyb/reL1pw6i+2XnGWVtdz0Fmrm+0RdqsWbOYNWsWdrsdgKysLEpKStB1ndzcXJRSNb6p3pVOTe1m5CfLjMTlaTzh06wNyb/rflSHUooZ3x9k0abKxfujL2jOnf7fYP3k32goitoNJa/vVMjOrdFGZnEmOaU5FBQUEJobeorPCP8I4oLieLb7s/hpfmjFGpmZmafoOYvbSK7NyLyNjIHTZwx4yobk33U/vGnDqL7ZecZZW13PQfn5+YZ1G22RNnr0aEaPHk1eXh4RERHExsYSHh6OrutomkZsbKzTAepMp6Z2M/KTZUbi8jSe8GnWhuTfdT9Oxq4rnvhki6NAmzy0I7dqS7F8/QQAhR1vwnb1TOKsNQ/3tMI0Ri0bRZleVqNOgCWAz4Z9Rue4zui6TlZWVrWxOYvbSK7NyLyNjIHTYwx40obk33U/vGnDqL7ZecZZW13PQYGBgYZ1G22RdjIWi8WRdE3TqvxeHa50amo3Iz9ZZiQuT+MJn2ZtSP5dx3uccrvOIx/8xmebD2PR4PnrunJd4Xuw4ikA1HljyO86hiCrn9O4c8tynRZoAGV6GblluTQPa+4yNrNttZF5GxkDvj0GvGFD8u86Xm/aMKp/us5BZq6VIk0QTlNKK+yMeedXlm3LwM+i8fKN3bgi61X4cVqlQr/xqAv/D7Ky6jdQQRAEwS2kSBOE05DiMjt3v/ULP+w8QoCfhTnDu3Fx6nRY979KhUuegj4P1LhBQBAEQfB9pEgThNOM/JJy7pj/C+v2ZhMcYOW1Ed3p/ceT8OvblQpXvATn3Fm/QQqCIAi1Roo0QfBh7Lri5z1HySooIy4skHZxody+YD2/HcwlLNCP+Snd6PHLONi6GDQLXPVf6HZzfYctCIIgeAAp0gTBR1m6JZ1Jn20hs6DcIfOzaFToishgf95O6Uqnnx6EP78Ciz9c9zp0vKoeIxYEQRA8iRRpguCDLN2Sxuh3fkWdJK/QKyUPX9SCTt/fDXtWgl9g5VsEzrjULV9KKSrsFbULWBAEQfA4UqQJgo9h1xWTP992SoF2nDCK6LpyFKjt4B9S+R7OVn3d8qUrnSlrp5Cam0qAJcD5OWnWACJt7r2QXRAEQTCPFGmC4GOsS81Gyz1IJ+3UU6nDKeRJ//m0U4epCAjH79bFkHiOW34q9Aomrp7IZ7s/Q0NjygVTaBXeiuycbKIio9AsWhX9SFskzUKbueVLEARBMI8UaYLgY+Rn7GGF7RECtfIadZSCdWc/T283C7RyeznjfhjHN/u+wapZeeaCZ7ii9RXouk6mPZO46Lg6PTBTEARBOBUp0gTBx2jqV+S0QAPQNAiNSnDLfklFCY98/wirDq7C3+LPCxe9wICWA9yyJQiCIHgPKdIEwcewGzyAtlPzcNO2i8qLeGDFA6xNX0ugNZAZ/WfQp3kf03YEQRAE7yPfZwiCD7FiewaTl/xhSNeqaa6VTmJ//n5+P/I7wX7BzB44Wwo0QRAEH0aepAmCj/DFb2k8+N6vtFe610bmmVFn8sqAVwi0BnJW7FnecSIIgiB4BCnSBMEH+HDDQR77cDO6govax8I+z9k+UnKE7OxsOsZ0BOCcpu5tNhAEQRDqFvm6UxDqmbfW7OXRDyoLtJvOSeSRjgUes32o4BBj143l7mV3szNnp8fsCoIgCN5HijRBqEfmfL+bf3+6FYBRfZKZ2ukg1q//6RHbe3P3MurrUaQVpxEWEEawf7BH7AqCIAh1g3zdKQj1gFKK6cv+5D8rdgEwpn9bHknYgvb+3aBXVL4sXTnZ5elng+DoGpv/zPmTu7+5m6MlR0kMSeT1Qa/LQbSCIAinGVKkCUIdo5Ti6S/+4PUfUwF47LL23BfxMyy+v7IwO+sG6P84enEO2dnZREVFYTl5J2dwNDRJrNb+1iNbuefbe8gtzaV9ZHue7vY08cHx3u6WIAiC4GGkSBOEOsSuK574ZAvvrtsPwKQhHRnptww+/b9KhbNT4MrpYLGC3pIKaybExYHB0/+3Z2/njm/uoLC8kC6xXZh18SxKjpV4qzuCIAiCF2kQa9KWLFlC+/btadeuHa+99lp9hyMI1VJh13nk/U28u24/Fg2ev7YLI/WP4au/CrTzRsOQlysLNDdJCk+ifWR7zml6DnMvmUt4gPkDbwVBEATf4LR/klZRUcHYsWP57rvviIiIoEePHlx99dVER9e8XkcQ6prSCjsPvPsrX2/NwM+iMf2Grgw5+gb88GKlQt/HoP+/Kt/3VAuC/IKYNWAWfhY/Av0C0Q2+vUAQBEHwPU77J2nr1q2jU6dONG/enNDQUAYPHsw333xT32EJgoPiMjt3vbmBr7dmEGC1MGf42QxJf+XvAm3gZLj4cbcLtC/3fMl/N/3X8XtoQCiBfoGeCF0QBEGoR+q9SFu1ahVDhgwhISEBTdP45JNPTtGZNWsWycnJBAYGcu6557Ju3TpH2+HDh2nevLnj9+bNm3Po0KG6CF0QXFJQWkHKvHWs+jOLIH8rb9x2NgN3PwM//1VUXf4iXPCQ2/Y/+vMjxv0wjtmbZ/P9ge89E7QgCILgE9R7kVZYWEjXrl2ZNWtWte2LFi1i7NixTJw4kY0bN9K1a1cGDRpEZmZmHUcqCOY4VlTG8NfWsi41mzCbH2+P6s4FWx6HjW9WHrExbDb0ustt+29ve5tJayahUNxwxg1c2OJCD0YvCIIg1Df1viZt8ODBDB48uMb2adOmcddddzFq1CgA5syZwxdffMEbb7zBuHHjSEhIqPLk7NChQ/Tq1atGe6WlpZSWljp+z8vLA0DXdcePUsrpWh5XOjW1m5GfLDMSl6fxhE+zNhpK/rPyS0mZt57t6fk0CfLnzZQunLXmIdjxJcrih7rmVeg4DNzs52u/v8bMTTMBSOmYwsNnPwwKdGWsz+74NKpntq02Mm8jY0DmIHd0JP/u22gMc5CZ6+u9SHNGWVkZGzZsYPz48Q6ZxWJh4MCBrFmzBoBevXqxZcsWDh06REREBF999RX//ve/a7Q5depUJk+efIo8KyuLkpISdF0nNzcXpRSWGo49cKVTU7sZ+ckyI3F5Gk/4NGujIeQ/M7+M+xf/yb6cUqKD/XhlaAvaf3sH2sEfUdYAjl06k9KY3uDkaXBmcSY5pTkUFBQQmhvq8KmU4tP9n/LN4cp1l7e1uY3hLYaTlZXldj7d0XemZ7atNjJvI2NA5iB3dCT/7ttoDHNQfn6+YV2fLtKOHDmC3W4nPr7qQZzx8fFs374dAD8/P1566SX69++Prus89thjTnd2jh8/nrFjxzp+z8vLIzExkdjYWMLDw9F1HU3TiI2NdTpAnenU1G5GfrLMSFyexhM+zdo43fO/P7uI+xav42BOKc0iAll4awdafXM72sE1KP8Q1E3vENGqr9McpBWmMWrZKMr0Mqd6d551J/d3u9+pjjfy70rPbFttZN5GxoDMQe7oSP7dt9EY5qDAQOMbu3y6SDPK0KFDGTp0qCFdm82GzWY7RW6xWBxJ1zStyu/V4UqnpnYz8pNlRuLyNJ7wadbG6Zr/XZkFDH/tZzLySkmODmbhiDNo/vlwOLwRbBFowz9Aa3muy/7nluW6LNAALkm6xFBOvZF/V3pm22oj8zYyBmQOckdH8u++jYY+B5m51qeLtJiYGKxWKxkZGVXkGRkZNG3a1KO+dF3WpJ2MJ3yatXG65n/b4TxS5q3naGEZZ8SF8vaNycR+fB1kbkMFRaFGLIZmXcFAHpSuXOoc13OVV2/k35We2bbayLyNjAGZg9zRkfy7b6MxzEFmrvfpIi0gIIAePXqwfPlyhg0bBlR2bvny5YwZM6ZWtmfNmsWsWbOw2+2ArEmrDk/4NGvjdMz/lrQCHv5kF/mldtrHBTPr0mCi3h+KlrsXe3As2VfOx25t5nQN2olk52Ub08vJJtPu3KY38u9Kz2xbbWTeRsaAzEHu6Ej+3bfRGOag02pNWkFBAbt27XL8npqayqZNm4iKiqJly5aMHTuWlJQUevbsSa9evZgxYwaFhYWO3Z7uMnr0aEaPHk1eXh4RERGyJq0aPOHTrA1fzr+uYE+BH0cKy4gLs3FOchTr92bz4Me7KCyz06NlE+ZdFU3Y+9eh5R5ARbRAu/VToqNam8rZUetRQ3pRkVHERcc51fFG/l3pmW2rjczbyBiQOcgdHcm/+zYawxx0Wq1J++WXX+jfv7/j9+OL+lNSUpg/fz433ngjWVlZTJgwgfT0dLp168bSpUtP2UxQW8x+7y/rEbxnwxfzv3RLOpM+20JmQbmjPTLYn/ySCip0RZ+20bw2OJSgd6+CgnSIaoOW8hlaRAtDfa4Si8XYmwc0i2Yop7IepHbIGJA5yB0dyb/7Nhr6HGTm2nov0vr164dSztfgjBkzptZfb7pC12VN2sl4wqdZG76Y/6Vb0hn9zq+c/CnNKaos2M5qHs7rl/hjWzgUio6i4jqgRnwMofHgRu5yS3IN6cmaNO8jY0DmIHd0JP/u22gMc5CZ6+u9SKsvZE2aazzh06wNX8t/hV1n0mdbTynQTiTu2CYCFj6HVpZPeWxnsi9/DVWkQZH5t2IcLT3KxLUTDenKmjTvI2NA5iB3dCT/7ttoDHPQabUmrb6QNWmu8YRPszZ8Lf+78634FaTRSat+UHWx7ObfFW9jsZehEs/DevN7xAZGuOxnTVQUVFChKlzqBVgCaN2sNXEhsibNm8gYkDnIHR3Jv/s2GsMcdFqtSfMVzH7vL+sRvGfDl/JfmLWXFbZHCNTKccaRqB7E3LoYLSDEVfec0iK8BW9c9gZHi48SaA0kOyebqMioU9apRdoiaRbazJBNWQ9SOxr7GJA5yD0dyb/7Nhr6HGTmWinSBMEJTf0KXRZoAId7jSfGzQJtZ85ODhUcol9iPwCSI5JJjkhG13Uy7ZnERcfV6aQsCIIg+AYy8wuCEzomGPvqslNijFv2tx7dyqivR/HwyodZn77eLRuCIAhCw0SepP2FrsvuzpPxhE+zNnwt/5rTLQN/o2Fuxw7AxsyN3L/ifgrKC+gc3Zm2EW2dfgbcwRv5d6Vntq02Mm/jq/dA5iDv2ZD8u47XmzYawxxk5vpGW6TJ7k7XeMKnWRu+ln+/8mxiDfQzOzubCqvx3Zwbj25k4q8TKbGXcFbkWTzd7WlKc0vJ5G8bvpp/V3pm22oj8za+eg9kDvKeDcm/635400ZjmINkd6cBZHenazzh06wNX8t/ZEWRoX5GRUVBnPOdlsdZeWAlT2x8gnK9nD4JfXjpopcI8gsy3U8jeCP/rvTMttVG5m189R7IHOQ9G5J/1/3wpo3GMAfJ7k43MLuDRnb2eM+Gr+XfCBZNAwO6W45s4ZHvH6FCVTCg5QCe7/s8AdaAGvV9Nf+u9Hx5Z5VZfPUeyBzkPRuSf9fxetNGQ5+DzFwrRZog1CEdoztyeevL0ZXOU32ews8iQ1AQBEGoHvkLIQjOCI4CPxtUlNas42eD4GinZnSlY9EsWDQLk3tPdvy/IAiCINSEFGl/oeuyu/NkPOHTrA2fy39Ycxi9Hr3gCDk5OURGRp76qDo4CsKbQw3xvL7ldbZnb+fZC57FarFiwQKqsnBzhq/m35We2bbayLyNr94DmYO8Z0Py7zpeb9poDHOQmesbbZEmuztd4wmfZm34Zv5t6NZm5PoHU26NONVnKZB56s5OpRTzd83nnT3vANAnqg+943q7zIGZXHjahlF9Z3pm22oj8za+eg9kDvKeDcm/635400ZjmINkd6cBZHenazzh06yNhpJ/pRQv/PKCo0B7+OyHGdZpmMvrzPTTGzaM6jvTM9tWG5m38dV7cDqMAU8g+Zf8u6Pn63OQ7O50A7M7aGRnj/dsnO75t+t2nlz7JIt3Lgbg8XMf56Yzb3J6TU34av5d6fnyziqz+Oo98OUx4Ekk/5J/d/R8eQ4yc60UaYLgQcr1ch7/8XG+Sv0Ki2bhyd5PclXbq+o7LEEQBOE0RIo0QfAgu4/tZsX+Ffhpfjzb91kGJQ+q75AEQRCE0xQp0gTBg5wZdSbT+01Hoejbom99hyMIgiCcxkiR9he6LkdwnIwnfJq1cTrmv6CsgCMlR0gOTwagT0IfxzW1wVfz70rPbFttZN7GV++Br40BbyH5l/y7o+frc5CZ6xttkSZHcLjGEz7N2jjd8p9Xlsf4DeM5UnKEab2m0TykueHcuMJX8+9Kz2xbbWTexlfvgS+NAW8i+Zf8u6Pn63OQHMFhADmCwzWe8GnWhi/mP6M4g+zibHIsOVT4VTjeFJBTmsMz65/hYMFBIm2RBEYEEhdl7CXrRvDV/LvSM9tWG5m38dV7IHOQ92xI/l33w5s2GsMcJEdwuIHZbc6y/dp7Nnwp/xnFGQz9dChl9jKnMT/f93k6xXRy1TXT+Gr+Xen58vZ3s/jqPZA5yHs2JP+u4/WmjYY+B5m5tu7utCCchhwrOeayQAMIt4XXQTSCIAhCY0KKNEEQBEEQBB9EijRBEARBEAQfRIo0QRAEQRAEH0SKNEEQBEEQBB9Ednf+ha7LYbYn4wmfZm34XP6VsbiV7vl746v5d6Vntq02Mm/jq/dA5iDv2ZD8u47XmzYawxxk5vpGW6TJYbau8YRPszZ8Lf/ZWrahfmbnZJNpzzSkaxRfzb8rPbNttZF5G1+9BzIHec+G5N91P7xpozHMQXKYrQHkMFvXeMKnWRu+lv/Q4FACLAGU6TUfwxFgCaB1s9bEhXjuIFtn8XrThlF9Z3pm22oj8za+eg9kDvKeDcm/635400ZjmIPkMFs3MHtgoBxk6D0bvpT/5mHNWXL1ErKLs8nOySYqMgrNolW5NtIWSbPQZob6ZhZfzb8rPV8+SNIsvnoPZA7yng3Jv+t4vWmjoc9BZq6VIk0QXNAstBnxwfFk2jOJi46r0wlSEARBaLzIXxtBEARBEAQfRIo0QRAEQRAEH0SKNEEQBEEQBB9EijRBEARBEAQfRIo0QRAEQRAEH0SKNEEQBEEQBB9EijRBEARBEAQfRM5J+wtdl3d3nownfJq1Ifl3Ha83bRjVd6Zntq02Mm/jq/dAxoD3bEj+XcfrTRuNYQ4yc32jLdLk3Z2u8YRPszYk/6774U0bRvWd6Zltq43M2/jqPZAx4D0bkn/X/fCmjcYwB8m7Ow1w/N2dubm5NGnSBJvNRmBgILquU1BQQGBgoNMB6kynpnYz8pNlRuLyNJ7wadaG5N91P7xpw6i+Mz2zbbWReRtfvQcyBrxnQ/Lvuh/etNEY5qCyssp3QSulXOo22iLtOMcr2qSkpHqORBAEQRCExkJ+fj4RERFOdTRlpJRrwOi6zuHDhwkLC0PTKl+cfc4557B+/Xqn17nSqandjPxEWV5eHomJiRw4cIDw8HCX/fIURnLhaRuSf+dxeduGUX1nembbjMjkHpjTkTHgvg3Jv/O4vG2joc9BSiny8/NJSEhw+USu0T9Js1gstGjRoorMarW6vAGudGpqNyOvThYeHl6nA9RILjxtQ/LvPAZv2zCq70zPbJtRGcg9MKojY8B9G5J/5zF420ZjmINcPUE7jhzBUQ2jR4+utU5N7WbkRuLwNp6IwawNyb9nY/BG/l3pmW3z1fyD794DGQPesyH592wMMge5T6P/uvN0IS8vj4iICHJzc+v0X1FCJZL/+kfuQf0i+a9fJP/1T33cA3mSdppgs9mYOHEiNputvkNplEj+6x+5B/WL5L9+kfzXP/VxD+RJmiAIgiAIgg8iT9IEQRAEQRB8ECnSBEEQBEEQfBAp0gRBEARBEHwQKdIEQRAEQRB8ECnSBEEQBEEQfBAp0hoAycnJdOnShW7dutG/f//6DqdRkpqaSv/+/enYsSNnnXUWhYWF9R1So2HHjh1069bN8RMUFMQnn3xS32E1KqZPn06nTp3o2LEjDzzwgKEXRwue5cUXX6RTp0507tyZt99+u77DaRRcffXVREZGct1111WRL1myhPbt29OuXTtee+21WvmQIzgaAMnJyWzZsoXQ0ND6DqXRctFFF/H0009z4YUXkp2dTXh4OH5+jf6ta3VOQUEBycnJ7Nu3j5CQkPoOp1GQlZXFeeedx9atW/H396dv3768+OKLnH/++fUdWqPh999/JyUlhdWrV6OUon///ixdupQmTZrUd2gNmpUrV5Kfn8+CBQv48MMPAaioqKBjx4589913RERE0KNHD1avXk10dLRbPuRJmiDUkuN/nC688EIAoqKipECrJz777DMGDBggBVodU1FRQUlJCeXl5ZSXlxMXF1ffITUq/vjjD84//3wCAwMJCgqia9euLF26tL7DavD069ePsLCwKrJ169bRqVMnmjdvTmhoKIMHD+abb75x24cUafXMqlWrGDJkCAkJCWiaVu3XNLNmzSI5OZnAwEDOPfdc1q1bV6Vd0zQuuugizjnnHBYuXFhHkTccansPdu7cSWhoKEOGDOHss89mypQpdRj96Y8nxsBx3n//fW688UYvR9ywqG3+Y2NjefTRR2nZsiUJCQkMHDiQNm3a1GEPTn9qew86d+7MypUrOXbsGDk5OaxcuZJDhw7VYQ9OPzw575zI4cOHad68ueP35s2b1+peSJFWzxQWFtK1a1dmzZpVbfuiRYsYO3YsEydOZOPGjXTt2pVBgwaRmZnp0Pnxxx/ZsGEDn332GVOmTOG3336rq/AbBLW9BxUVFfzwww/897//Zc2aNSxbtoxly5bVZRdOazwxBqDyvXqrV6/m8ssvr4uwGwy1zX9OTg5Llixh7969HDp0iNWrV7Nq1aq67MJpT23vwfG1gBdffDHXXHMN5513HlartS67cNrhqXnH6yjBZwDUxx9/XEXWq1cvNXr0aMfvdrtdJSQkqKlTp1Zr49FHH1Xz5s3zYpQNG3fuwerVq9Wll17qaH/++efV888/XyfxNjRqMwbefPNNNXz48LoIs8HiTv7ff/99dd999znan3/+efXcc8/VSbwNEU/8HbjjjjvUkiVLvBlmg6I2Of/uu+/Utdde6/j9p59+UsOGDXP8/uCDD6qFCxe6HZs8SfNhysrK2LBhAwMHDnTILBYLAwcOZM2aNUDlvwby8/OBykXTK1asoFOnTvUSb0PEyD0455xzyMzMJCcnB13XWbVqFR06dKivkBsURvJ/HPmq0/MYyX9iYiKrV6+mpKQEu93OypUrad++fX2F3OAwOgaOP+HZsWMH69atY9CgQXUea0PBzLxzMr169WLLli0cOnSIgoICvvrqq1rdC1nd7MMcOXIEu91OfHx8FXl8fDzbt28HICMjg6uvvhoAu93OXXfdxTnnnFPnsTZUjNwDPz8/pkyZQt++fVFKcemll3LllVfWR7gNDiP5B8jNzWXdunV89NFHdR1ig8ZI/s877zwuv/xyunfvjsViYcCAAQwdOrQ+wm2QGB0DV111Fbm5uYSEhDBv3jzZvFQLjOZ84MCBbN68mcLCQlq0aMEHH3zA+eefz0svvUT//v3RdZ3HHnvM7Z2dIEXaaU/r1q3ZvHlzfYfR6Bk8eDCDBw+u7zAaLREREWRkZNR3GI2WZ555hmeeeaa+w2jUuHrCI3ieb7/9tlr50KFDPfYPFfm604eJiYnBarWe8scnIyODpk2b1lNUjQu5B/WL5L9+kfzXP3IP6h5fyrkUaT5MQEAAPXr0YPny5Q6ZrussX75cDoqsI+Qe1C+S//pF8l//yD2oe3wp5/J1Zz1TUFDArl27HL+npqayadMmoqKiaNmyJWPHjiUlJYWePXvSq1cvZsyYQWFhIaNGjarHqBsWcg/qF8l//SL5r3/kHtQ9p03O3d4XKniE7777TgGn/KSkpDh0Zs6cqVq2bKkCAgJUr1691M8//1x/ATdA5B7UL5L/+kXyX//IPah7Tpecy7s7BUEQBEEQfBBZkyYIgiAIguCDSJEmCIIgCILgg0iRJgiCIAiC4INIkSYIgiAIguCDSJEmCIIgCILgg0iRJgiCIAiC4INIkSYIgiAIguCDSJEmCIIgCILgg0iRJgiCW4wcOZJhw4Z5zf7KlSvRNI1jx44BMH/+fJo0aeI1f8dJTk5mxowZXvdTE/369eOhhx7yqo+ysjLatm3L6tWrAdi7dy+aprFp0yav+nWH8847j48++qi+wxCEekGKNEFoILj7x70uigJfpKaib/369dx99911H9BfLF68mKeeesqrPubMmUOrVq3o3bu3R+1qmsYnn3ziUZtPPPEE48aNQ9d1j9oVhNMBKdIEQRBOIDY2luDg4HrzHxUVRVhYmNfsK6V45ZVXuOOOO7zmw5MMHjyY/Px8vvrqq/oORRDqHCnSBKEBMHLkSL7//ntefvllNE1D0zT27t0LwPfff0+vXr2w2Ww0a9aMcePGUVFR4fQ6u93OHXfcQatWrQgKCqJ9+/a8/PLLpuP66aef6NevH8HBwURGRjJo0CBycnIAKC0t5YEHHiAuLo7AwEAuuOAC1q9fb8r+p59+ytlnn01gYCCtW7dm8uTJjr4BHDt2jHvuuYf4+HgCAwPp3LkzS5YsYeXKlYwaNYrc3FxHvydNmgRU/brzlltu4cYbb6zis7y8nJiYGN58800AdF1n6tSpjlx17dqVDz/80Gnc//3vf2nXrh2BgYHEx8dz3XXXOdpOfLJ5/Cvfk39GjhxpOAcns2HDBnbv3s0VV1xxStv27dvp3bu3I1fff/89UFnYtW3blhdffLGK/qZNm9A0jV27dpGcnAzA1VdfjaZpjt9dxaiUYtKkSbRs2RKbzUZCQgIPPPCA41qr1crll1/Oe++95zSngtAgqfNXuguC4HGOHTumzj//fHXXXXeptLQ0lZaWpioqKtTBgwdVcHCwuu+++9Qff/yhPv74YxUTE6MmTpzo9LqysjI1YcIEtX79erVnzx719ttvq+DgYLVo0SKHz5SUFHXVVVfVGNOvv/6qbDabuvfee9WmTZvUli1b1MyZM1VWVpZSSqkHHnhAJSQkqC+//FJt3bpVpaSkqMjISHX06FGllFLfffedAlROTo5SSql58+apiIgIh/1Vq1ap8PBwNX/+fLV79271zTffqOTkZDVp0iSllFJ2u12dd955qlOnTuqbb75Ru3fvVp9//rn68ssvVWlpqZoxY4YKDw939Ds/P18ppVRSUpKaPn26UkqpJUuWqKCgIEebUkp9/vnnKigoSOXl5SmllHr66afVmWeeqZYuXap2796t5s2bp2w2m1q5cmW1eVm/fr2yWq3qnXfeUXv37lUbN25UL7/8sqP9oosuUg8++KBSSqnS0lJHfGlpaWrFihUqMDBQvf7664ZyUB3Tpk1TZ555ZhVZamqqAlSLFi3Uhx9+qLZt26buvPNOFRYWpo4cOaKUUuqZZ55RHTt2rHLdAw88oPr27auUUiozM1MBat68eSotLU1lZmYaivGDDz5Q4eHh6ssvv1T79u1Ta9euVXPnzq3iZ/bs2SopKanGPglCQ0WKNEFoIJz4x/04//rXv1T79u2VrusO2axZs1RoaKiy2+01Xlcdo0ePVtdee63jd1dF2s0336z69OlTbVtBQYHy9/dXCxcudMjKyspUQkKCev7555VSrou0AQMGqClTplSx+9Zbb6lmzZoppZT6+uuvlcViUTt27Kg2hpPtHefEIq28vFzFxMSoN998s0q/brzxRqWUUiUlJSo4OFitXr26io077rhD3XzzzdX6/eijj1R4eLijyDuZmu7HkSNHVOvWrdV9993nkLnKQXU8+OCD6uKLL64iO16kPfvssw5ZeXm5atGihXruueeUUkodOnRIWa1WtXbtWqVU5f2KiYlR8+fPd1wDqI8//riKbVcxvvTSS+qMM85QZWVlNcb86aefKovF4vjMCkJjwa8+n+IJguBd/vjjD84//3w0TXPI+vTpQ0FBAQcPHqRly5Y1Xjtr1izeeOMN9u/fT3FxMWVlZXTr1s2w702bNnH99ddX27Z7927Ky8vp06ePQ+bv70+vXr34448/DNnfvHkzP/30E88884xDZrfbKSkpoaioiE2bNtGiRQvOOOMMwzGfjJ+fHzfccAMLFy7k1ltvpbCwkE8//dTx1duuXbsoKirikksuqXJdWVkZ3bt3r9bmJZdcQlJSEq1bt+ayyy7jsssu4+qrr3a6Dq68vJxrr72WpKSkKl87u8pBdTaLi4sJDAys1s/5559fpe89e/Z03I+EhASuuOIK3njjDXr16sXnn39OaWlpjffYaIzXX389M2bMcOTj8ssvZ8iQIfj5/f3nKSgoCF3XKS0tJSgoyKk/QWhISJEmCMIpvPfeezz66KO89NJLnH/++YSFhfHCCy+wdu1awza8/ce0oKCAyZMnc80115zSFhgY6DH/w4cP56KLLiIzM5Nly5YRFBTEZZdd5ogB4IsvvqB58+ZVrrPZbNXaCwsLY+PGjaxcuZJvvvmGCRMmMGnSJNavX1/jESP33nsvBw4cYN26dVWKF1c5qI6YmBh+//13l/2ujjvvvJNbb72V6dOnM0L20hQAAAWVSURBVG/ePG688UaXmyxcxZiYmMiOHTv49ttvWbZsGffddx8vvPAC33//Pf7+/gBkZ2cTEhIiBZrQ6JAiTRAaCAEBAdjt9iqyDh068NFHH6GUcjxN++mnnwgLC6NFixY1XvfTTz/Ru3dv7rvvPods9+7dpuLp0qULy5cvZ/Lkyae0tWnThoCAAH766SeSkpKAyqdF69evN3wcyNlnn82OHTto27Ztjf4PHjzIn3/+We3TtOr6XR29e/cmMTGRRYsW8dVXX3H99dc7ioeOHTtis9nYv38/F110kaG4ofIp1cCBAxk4cCATJ06kSZMmrFixotpCZtq0abz//vusXr2a6OjoKm2uclAd3bt3Z/bs2VU+E8f5+eef6du3LwAVFRVs2LCBMWPGONovv/xyQkJCmD17NkuXLmXVqlVVrvf39z8lp0ZiDAoKYsiQIQwZMoTRo0dz5pln8vvvv3P22WcDsGXLlhqfTApCQ0aKNEFoICQnJ7N27Vr27t1LaGgoUVFR3HfffcyYMYP777+fMWPGsGPHDiZOnMjYsWOxWCw1XteuXTvefPNNvv76a1q1asVbb73F+vXradWqleF4xo8fz1lnncV9993HP/7xDwICAvjuu++4/vrriYmJ4d577+X//u//iIqKomXLljz//PMUFRUZPhpiwoQJXHnllbRs2ZLrrrsOi8XC5s2b2bJlC08//TQXXXQRffv25dprr2XatGm0bduW7du3o2kal112GcnJyRQUFLB8+XK6du1KcHBwjU+FbrnlFubMmcOff/7Jd99955CHhYXx6KOP8vDDD6PrOhdccAG5ubn89NNPhIeHk5KScoqtJUuWsGfPHvr27UtkZCRffvkluq7Tvn37U3S//fZbHnvsMWbNmkVMTAzp6elAZVETERHhMgfV0b9/fwoKCti6dSudO3eu0jZr1izatWtHhw4dmD59Ojk5Odx+++2OdqvVysiRIxk/fjzt2rWr8vUoVH6Wli9fTp8+fbDZbERGRrqMcf78+djtds4991yCg4N5++23CQoKchTvAD/88AOXXnpptf0RhAZNfS+KEwTBM+zYsUOdd955KigoSAEqNTVVKaXUypUr1TnnnKMCAgJU06ZN1T//+U9VXl7u9LqSkhI1cuRIFRERoZo0aaLuvfdeNW7cONW1a1fHda42Dhz33bt3b2Wz2VSTJk3UoEGDHBsBiouL1f33369iYmKUzWZTffr0UevWrXNc62rjgFJKLV26VPXu3VsFBQWp8PBw1atXryo7A48ePapGjRqloqOjVWBgoOrcubNasmSJo/0f//iHio6OVoBjx+uJGweOs23bNgWopKSkKpswlFJK13U1Y8YM1b59e+Xv769iY2PVoEGD1Pfff19tTn744Qd10UUXqcjISBUUFKS6dOlSZdfsiRsHJk6cqIBTflJSUgznoDpuuOEGNW7cOMfvxzcOvPPOO6pXr14qICBAdezYUa1YseKUa3fv3q0AxwaPE/nss89U27ZtlZ+fX5XdmM5i/Pjjj9W5556rwsPDVUhIiDrvvPPUt99+67j24MGDyt/fXx04cMBpnwShIaIppVT9lIeCIAhCffDbb79xySWXsHv3bkJDQ01d+8MPPzBgwAAOHDhAfHy8lyL8m3/+85/k5OQwd+5cr/sSBF9DDrMVBEFoZHTp0oXnnnuO1NRUw9eUlpZy8OBBJk2axPXXX18nBRpAXFyc11+TJQi+ijxJEwRBEFwyf/587rjjDrp168Znn312ym5WQRA8jxRpgiAIgiAIPoh83SkIgiAIguCDSJEmCIIgCILgg0iRJgiCIAiC4INIkSYIgiAIguCDSJEmCIIgCILgg0iRJgiCIAiC4INIkSYIgiAIguCDSJEmCIIgCILgg0iRJgiCIAiC4IP8PxPmXamKIgg3AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(7, 5))\n", + "\n", + "ax.loglog(sweep[\"collective_size_bytes\"], sweep[\"model_latency_s\"] * 1e6, \"o-\",\n", + " label=f\"FC model ({LINK_BW_GBPS:g} GB/s links)\")\n", + "ax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"model_latency_s\"] * 1e6, \"s-\",\n", + " label=\"torus model, 152 hops (bounding-box multicast)\")\n", + "ax.loglog(sweep_torus[\"collective_size_bytes\"], sweep_torus[\"minroute_latency_s\"] * 1e6, \"s--\",\n", + " label=\"torus min-route, 96 hops (dimension-ordered unicast)\")\n", + "\n", + "if HAVE_EMPIRICAL:\n", + " alltoall = emp[emp[\"collective\"] == \"alltoall\"]\n", + " fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n", + " torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n", + " if not fc_rows.empty:\n", + " # nccl-tests alltoall size_bytes is the PER-RANK message size (see\n", + " # markdown above); scale to total S to match `sweep`'s convention.\n", + " ax.scatter(fc_rows[\"size_bytes\"] * NODES, fc_rows[\"time_us\"],\n", + " marker=\"x\", color=\"tab:red\", label=\"EC2 nccl-tests alltoall (fc)\")\n", + " if not torus_rows.empty:\n", + " # torus_bench size_bytes is ALREADY total collective size S.\n", + " ax.scatter(torus_rows[\"size_bytes\"], torus_rows[\"time_us\"],\n", + " marker=\"+\", color=\"tab:green\", label=\"EC2 torus_bench alltoall (torus)\")\n", + "\n", + "ax.set_xlabel(\"total collective size (bytes)\")\n", + "ax.set_ylabel(\"all-to-all latency (µs)\")\n", + "ax.set_title(\"Fully-connected vs. 2x2x2 torus all-to-all: model vs. empirical\")\n", + "ax.grid(True, which=\"both\", alpha=0.3)\n", + "ax.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "a8ec5d63", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.497733Z", + "iopub.status.busy": "2026-09-06T20:56:38.497621Z", + "iopub.status.idle": "2026-09-06T20:56:38.505298Z", + "shell.execute_reply": "2026-09-06T20:56:38.504489Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FC calibration : alpha = 74.227 us, BW_eff = 339.48 GB/s, R^2 = 0.9996\n", + "torus calibration : fewer than 2 alltoall rows, skipping fit\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "\n", + "def linear_fit(S: np.ndarray, t: np.ndarray) -> tuple[float, float, float]:\n", + " \"\"\"Ordinary-least-squares fit of ``t = intercept + slope * S``.\n", + "\n", + " Parameters\n", + " ----------\n", + " S : np.ndarray\n", + " Total collective size in bytes for each empirical sample.\n", + " t : np.ndarray\n", + " Measured latency in seconds for each empirical sample. Must be\n", + " the same length as `S`.\n", + "\n", + " Returns\n", + " -------\n", + " intercept : float\n", + " Fitted per-operation latency overhead (the model's alpha), seconds.\n", + " slope : float\n", + " Fitted dt/dS, seconds/byte.\n", + " r_squared : float\n", + " Coefficient of determination of the linear fit; NaN if `t` has\n", + " zero total variance (R^2 is undefined in that degenerate case).\n", + " \"\"\"\n", + " slope, intercept = np.polyfit(S, t, 1)\n", + " t_pred = slope * S + intercept\n", + " ss_res = float(np.sum((t - t_pred) ** 2))\n", + " ss_tot = float(np.sum((t - np.mean(t)) ** 2))\n", + " r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else float(\"nan\")\n", + " return float(intercept), float(slope), r_squared\n", + "\n", + "\n", + "def bw_from_slope(slope: float, slope_factor: float) -> float:\n", + " \"\"\"Translate a fitted t-vs-S slope into an effective bandwidth, bytes/s.\n", + "\n", + " Given the model form ``t = alpha + S * slope_factor / BW``, the\n", + " fitted slope is ``dt/dS = slope_factor / BW``, so\n", + " ``BW = slope_factor / slope``.\n", + " \"\"\"\n", + " return slope_factor / slope\n", + "\n", + "\n", + "if HAVE_EMPIRICAL and (emp[\"collective\"] == \"alltoall\").any():\n", + " alltoall = emp[emp[\"collective\"] == \"alltoall\"].dropna(\n", + " subset=[\"size_bytes\", \"time_us\"]\n", + " )\n", + "\n", + " fc_rows = alltoall[alltoall[\"topology\"] == \"fc\"]\n", + " if len(fc_rows) >= 2:\n", + " # fc slope_factor = (N-1)/N^2 = 7/64 (N=8), from the closed form\n", + " # in the \"Size and latency conventions\" cell above.\n", + " S_fc = fc_rows[\"size_bytes\"].to_numpy(dtype=float) * NODES\n", + " t_fc = fc_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n", + " alpha_fc, slope_fc, r2_fc = linear_fit(S_fc, t_fc)\n", + " bw_fc = bw_from_slope(slope_fc, 7 / 64)\n", + " print(f\"FC calibration : alpha = {alpha_fc * 1e6:.3f} us, \"\n", + " f\"BW_eff = {bw_fc / 1e9:.2f} GB/s, R^2 = {r2_fc:.4f}\")\n", + " else:\n", + " print(\"FC calibration : fewer than 2 alltoall rows, skipping fit\")\n", + "\n", + " torus_rows = alltoall[alltoall[\"topology\"] == \"torus\"]\n", + " if len(torus_rows) >= 2:\n", + " # torus slope_factor = hops/(64*24) = hops/1536; report BW under\n", + " # both routing hypotheses (h=152 bounding-box, h=96 min-route)\n", + " # from a single fit, since alpha/R^2 don't depend on slope_factor\n", + " # -- only the slope-to-BW translation does.\n", + " S_t = torus_rows[\"size_bytes\"].to_numpy(dtype=float)\n", + " t_t = torus_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n", + " alpha_t, slope_t, r2_t = linear_fit(S_t, t_t)\n", + " bw_t_152 = bw_from_slope(slope_t, 152 / 1536)\n", + " bw_t_96 = bw_from_slope(slope_t, 96 / 1536)\n", + " print(f\"torus calibration : alpha = {alpha_t * 1e6:.3f} us, R^2 = {r2_t:.4f}\")\n", + " print(f\" BW_eff @ h=152 (bounding-box model): {bw_t_152 / 1e9:.2f} GB/s\")\n", + " print(f\" BW_eff @ h=96 (dimension-ordered) : {bw_t_96 / 1e9:.2f} GB/s\")\n", + " else:\n", + " print(\"torus calibration : fewer than 2 alltoall rows, skipping fit\")\n", + "else:\n", + " print(\"calibration: no empirical alltoall rows yet\")" + ] + }, + { + "cell_type": "markdown", + "id": "2325bd49", + "metadata": {}, + "source": [ + "## Linear fit: ISL network model vs. profiled GPUs (all-scatter, fully connected)\n", + "\n", + "Each nccl-tests `alltoall` row from the FC leg gives a measured latency $t_{meas}$ at per-rank size `size_bytes`, i.e. total $S = N \\cdot$ `size_bytes`. The ISL model at nominal H100 NVLink bandwidth predicts $t_{model}(S) = S(N-1)/(N^2 \\cdot BW_{nom})$ with $\\alpha = 0$. The fit below is ordinary least squares of\n", + "\n", + "$$ t_{meas} = a + b \\cdot t_{model} $$\n", + "\n", + "so $a$ is the per-operation latency overhead $\\alpha$ that the flat bandwidth model lacks, and $b = BW_{nom}/BW_{eff}$ is the ratio of nominal to achieved port bandwidth; equivalently $t = \\alpha + S(N-1)/(N^2 \\cdot BW_{eff})$ with $BW_{eff} = BW_{nom}/b$. OLS in linear space is dominated by the largest sizes, so the per-size table also reports measured/calibrated to show where a single straight line stops describing the data (NCCL's protocol switches at small sizes are the usual culprit). Rows whose $S$ equals a `MATMUL_SHAPES` operand are labelled with that shape.\n", + "\n", + "Data: `correlation/data//...` (versioned in the `aws-gpu-profiler` submodule), produced from this notebook's directory by\n", + "`aws-gpu-profiler run --workload nccl-tests --param collective=alltoall --param min_bytes=8192 --param max_bytes=1073741824 --data-dir correlation/data`. With no data present the cells below print a notice and draw nothing." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "0714f2dd", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.507533Z", + "iopub.status.busy": "2026-09-06T20:56:38.507438Z", + "iopub.status.idle": "2026-09-06T20:56:38.516282Z", + "shell.execute_reply": "2026-09-06T20:56:38.515831Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "linear fit t_meas = a + b * t_model (18 FC alltoall points)\n", + " a = 74.23 us b = 1.3256 R^2 = 0.9996\n", + " => alpha = 74.23 us, BW_eff = 339.5 GB/s (75.4% of 450 GB/s nominal)\n", + " => t = alpha + S * (N-1) / (N^2 * BW_eff)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
per_rank_bytescollective_size_bytesmatmul_shapemeasured_usmodel_nominal_usmodel_calibrated_usmeasured_over_calibrated
0819265536128x256 bf1686.610.01592974.2485671.166487
116384131072256x256 bf1681.970.03185874.2696821.103681
232768262144256x512 bf1681.370.06371674.3119121.094979
365536524288512x512 bf1681.570.12743174.3963711.096424
41310721048576512x1024 bf1679.780.25486274.5652901.069935
526214420971521024x1024 bf1680.110.50972474.9031281.069515
652428841943041024x2048 bf1681.001.01944975.5788041.071729
7104857683886082048x2048 bf1684.092.03889876.9301561.093069
82097152167772162048x4096 bf1687.684.07779679.6328601.101053
94194304335544324096x4096 bf1688.688.15559185.0382671.042825
108388608671088644096x8192 bf1692.4216.31118295.8490810.964224
11167772161342177288192x8192 bf1690.8232.622364117.4707100.773129
12335544322684354568192x16384 bf16128.1365.244729160.7139680.797255
136710886453687091216384x16384 bf16226.64130.489458247.2004830.916827
14134217728107374182416384x32768 bf16410.00260.978916420.1735130.975787
15268435456214748364832768x32768 bf16776.61521.957831766.1195741.013693
16536870912429496729632768x65536 bf161483.151043.9156621458.0116951.017241
171073741824858993459265536x65536 bf162830.582087.8313242841.7959380.996053
\n", + "
" + ], + "text/plain": [ + " per_rank_bytes collective_size_bytes matmul_shape measured_us \\\n", + "0 8192 65536 128x256 bf16 86.61 \n", + "1 16384 131072 256x256 bf16 81.97 \n", + "2 32768 262144 256x512 bf16 81.37 \n", + "3 65536 524288 512x512 bf16 81.57 \n", + "4 131072 1048576 512x1024 bf16 79.78 \n", + "5 262144 2097152 1024x1024 bf16 80.11 \n", + "6 524288 4194304 1024x2048 bf16 81.00 \n", + "7 1048576 8388608 2048x2048 bf16 84.09 \n", + "8 2097152 16777216 2048x4096 bf16 87.68 \n", + "9 4194304 33554432 4096x4096 bf16 88.68 \n", + "10 8388608 67108864 4096x8192 bf16 92.42 \n", + "11 16777216 134217728 8192x8192 bf16 90.82 \n", + "12 33554432 268435456 8192x16384 bf16 128.13 \n", + "13 67108864 536870912 16384x16384 bf16 226.64 \n", + "14 134217728 1073741824 16384x32768 bf16 410.00 \n", + "15 268435456 2147483648 32768x32768 bf16 776.61 \n", + "16 536870912 4294967296 32768x65536 bf16 1483.15 \n", + "17 1073741824 8589934592 65536x65536 bf16 2830.58 \n", + "\n", + " model_nominal_us model_calibrated_us measured_over_calibrated \n", + "0 0.015929 74.248567 1.166487 \n", + "1 0.031858 74.269682 1.103681 \n", + "2 0.063716 74.311912 1.094979 \n", + "3 0.127431 74.396371 1.096424 \n", + "4 0.254862 74.565290 1.069935 \n", + "5 0.509724 74.903128 1.069515 \n", + "6 1.019449 75.578804 1.071729 \n", + "7 2.038898 76.930156 1.093069 \n", + "8 4.077796 79.632860 1.101053 \n", + "9 8.155591 85.038267 1.042825 \n", + "10 16.311182 95.849081 0.964224 \n", + "11 32.622364 117.470710 0.773129 \n", + "12 65.244729 160.713968 0.797255 \n", + "13 130.489458 247.200483 0.916827 \n", + "14 260.978916 420.173513 0.975787 \n", + "15 521.957831 766.119574 1.013693 \n", + "16 1043.915662 1458.011695 1.017241 \n", + "17 2087.831324 2841.795938 0.996053 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "shape_by_size = {M * K * dtype_bytes: f\"{M}x{K} {MATMUL_DTYPE}\" for M, K in MATMUL_SHAPES}\n", + "fc_fit = None\n", + "fc_fit_table = None\n", + "if HAVE_EMPIRICAL:\n", + " fc_rows = (\n", + " emp[(emp[\"collective\"] == \"alltoall\") & (emp[\"topology\"] == \"fc\")]\n", + " .dropna(subset=[\"size_bytes\", \"time_us\"])\n", + " .sort_values(\"size_bytes\")\n", + " )\n", + "else:\n", + " fc_rows = pd.DataFrame()\n", + "\n", + "if len(fc_rows) >= 2:\n", + " S_fc = fc_rows[\"size_bytes\"].to_numpy(dtype=float) * NODES # per-rank -> total S\n", + " t_meas = fc_rows[\"time_us\"].to_numpy(dtype=float) * 1e-6\n", + " # Bandwidth term only, ALPHA_S deliberately excluded: the fitted intercept a IS alpha.\n", + " t_model_nom = S_fc * (n - 1) / (n * n * bw_h100)\n", + " alpha_fit_s, slope_fit, r2_fit = linear_fit(t_model_nom, t_meas)\n", + " fc_fit = {\"intercept_s\": alpha_fit_s, \"slope\": slope_fit, \"r2\": r2_fit, \"n\": int(len(t_meas))}\n", + " bw_eff = bw_h100 / fc_fit[\"slope\"]\n", + " t_model_cal = alpha_fit_s + fc_fit[\"slope\"] * t_model_nom\n", + " print(f\"linear fit t_meas = a + b * t_model ({fc_fit['n']} FC alltoall points)\")\n", + " print(f\" a = {alpha_fit_s * 1e6:.2f} us b = {fc_fit['slope']:.4f} R^2 = {fc_fit['r2']:.4f}\")\n", + " print(f\" => alpha = {alpha_fit_s * 1e6:.2f} us, BW_eff = {bw_eff / 1e9:.1f} GB/s \"\n", + " f\"({100 * bw_eff / bw_h100:.1f}% of {NVLINK_H100_GBPS:g} GB/s nominal)\")\n", + " print(\" => t = alpha + S * (N-1) / (N^2 * BW_eff)\")\n", + " fc_fit_table = pd.DataFrame({\n", + " \"per_rank_bytes\": fc_rows[\"size_bytes\"].to_numpy(dtype=int),\n", + " \"collective_size_bytes\": S_fc.astype(int),\n", + " \"matmul_shape\": [shape_by_size.get(int(s), \"\") for s in S_fc],\n", + " \"measured_us\": t_meas * 1e6,\n", + " \"model_nominal_us\": t_model_nom * 1e6,\n", + " \"model_calibrated_us\": t_model_cal * 1e6,\n", + " \"measured_over_calibrated\": t_meas / t_model_cal,\n", + " })\n", + " display(fc_fit_table)\n", + "else:\n", + " print(f\"linear fit: fewer than 2 FC alltoall rows (found {len(fc_rows)}) -- run aws-gpu-profiler run \"\n", + " \"--workload nccl-tests --param collective=alltoall --param min_bytes=8192 --param max_bytes=1073741824 --data-dir correlation/data\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "93811891", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-06T20:56:38.518402Z", + "iopub.status.busy": "2026-09-06T20:56:38.518321Z", + "iopub.status.idle": "2026-09-06T20:56:38.987993Z", + "shell.execute_reply": "2026-09-06T20:56:38.987515Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABPoAAAHpCAYAAAAbAFI+AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjksIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvJkbTWQAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXVYVMsbx7+7S3en0qGCimJiYGDjNVCwwVbQa2IH2IHdcQXs7ry2XgPba4siBlIq3bs7vz/47bksu8sGqc7neXh033nPzJx5z57z3TkTLEIIAYVCoVAoFAqFQqFQKBQKhUL5qWFXdgUoFAqFQqFQKBQKhUKhUCgUSumhHX0UCoVCoVAoFAqFQqFQKBTKLwDt6KNQKBQKhUKhUCgUCoVCoVB+AWhHH4VCoVAoFAqFQqFQKBQKhfILQDv6KBQKhUKhUCgUCoVCoVAolF8A2tFHoVAoFAqFQqFQKBQKhUKh/ALQjj4KhUKhUCgUCoVCoVAoFArlF4B29FEoFAqFQqFQKBQKhUKhUCi/ALSjj0KhUCgUCoVCoVAoFAqFQvkFoB19FEoVoFWrVmjVqlVlV4NShICAANjY2Ch0rDzxzMzMhImJCfbs2SN3OREREWCxWIiNjVWobAqlKjFt2jQ0bty4sqtBoVAoFEqVh+o92SiNni8Oi8VCSEhImeRFoZQ3tKOPUuYIOh9YLBb++ecfkXRCCKpXrw4WiwVvb+9KqCFFHlq1agVXV1chW35+PtasWYN69epBR0cHenp6cHFxwYgRI/D69WvGT3AtPHjwoKKr/dOwZs0aaGtro0+fPpVdlXLl5cuXCAkJEeqUFLBx40ZERERUeJ3E8fDhQ3h7e8PMzAxaWlqoU6cO1q5dCx6PV2Zl2NjYSLz3Xbt2DSwWC4cPH2ZsmZmZmDt3Ljp27AgDAwOwWKwS2+vVq1fo2LEjtLS0YGBggIEDByI5OVnEj8/nY9myZbC1tYWamhrq1KmDffv2yXQOISEhzH2++N/mzZuFfHNzc7Fq1So0btwYurq6UFNTg5OTE8aMGYO3b98yfuPHj8fTp09x8uRJmepAoVAolUlZapzs7GyEhITg2rVrpa8YpVyhsaJQKD8DSpVdAcqvi5qaGvbu3YvmzZsL2a9fv44vX75AVVW1kmpGKS0+Pj44d+4c+vbti+HDh6OgoACvX7/G6dOn4eHhgRo1alR2FX8KCgoKsGbNGkyYMAEcDqeyq1OuvHz5EqGhoWjVqpXIm9WNGzfCyMgIAQEBlVI3AQ8fPoSHhwccHR0xdepUaGho4Ny5cxg3bhzev3+PNWvWVEq9vn37hnnz5sHKygp169Yt8cfFly9f0LJlS+jq6mLRokXIzMxEWFgYnj17hnv37kFFRYXxnTlzJpYsWYLhw4ejYcOGOHHiBPr16wcWiyVzx/OmTZugpaUlZCs6Ku/bt2/o2LEj04Har18/aGlp4c2bN9i/fz+2bt2K/Px8AICZmRm6deuGsLAw/PHHH3K0EIVCofzcZGdnIzQ0FADoKK0qDo3V70tOTg6UlGj3CeXngF6plHKjc+fOOHToENauXSt0U9y7dy/c3d3x7du3Sqxd2ZCVlQVNTc3KrkaFcv/+fZw+fRoLFy7EjBkzhNLWr1+P1NTUyqnYT8jp06eRnJwMX1/fyq7KLweXywWfzxfq2JLGli1bAAA3btyAgYEBAGDkyJHw9PREREREpXX0mZubIz4+HmZmZnjw4AEaNmwo0XfRokXIysrCw4cPYWVlBQBo1KgR2rVrh4iICIwYMQIAEBcXhxUrViAoKAjr168HAAwbNgyenp4IDg5G7969Zep87tWrF4yMjCSmBwQE4PHjxzh8+DB8fHyE0ubPn4+ZM2cK2Xx9fdG7d2/ExMTAzs5OavkUCoVCoVB+DhTRZlUJNTW1yq4ChSIzdOoupdzo27cvvn//josXLzK2/Px8HD58GP369RN7DJ/Px+rVq+Hi4gI1NTWYmppi5MiRSElJEfI7ceIEunTpAgsLC6iqqsLe3h7z588XmV4XHR0NHx8fmJmZQU1NDdWqVUOfPn2QlpYGAIiNjZU4Da74OgyCqWovX75Ev379oK+vLzRacffu3XB3d4e6ujoMDAzQp08ffP78WSTfrVu3wt7eHurq6mjUqBFu3rwptS0BwNXVFa1btxbbZpaWlujVqxdj279/P9zd3aGtrQ0dHR3Url27zDop3r9/DwBo1qyZSBqHw4GhoWGZlMNisTBmzBgcOnQItWrVgrq6Opo2bYpnz54BKOyUcXBwgJqaGlq1aiV2SuihQ4eYmBgZGWHAgAGIi4sT8Tt+/DhcXV2hpqYGV1dXHDt2TGydZL0+ZeX48eOwsbGBvb29kP3ff/9FQEAA7OzsoKamBjMzMwwZMgTfv39XqBxxFBQUIDQ0FI6OjlBTU4OhoSGaN28u9H0FgNevX8PX1xfGxsZQV1eHs7OzUOfMx48fERgYCGdnZ6irq8PQ0BC9e/cWikdERAR69+4NAGjdujUzxfPatWuwsbHBixcvcP36dcZe9A15amoqxo8fj+rVq0NVVRUODg5YunQp+Hw+4yP4HoeFhWH16tWwt7eHqqoqXr58KVebpKenQ01NDXp6ekJ2c3NzqKurM5/nzp0LNpuNy5cvC/mNGDECKioqePr0qVzlSkNVVRVmZmYy+R45cgTe3t5MJx8AeHl5wcnJCQcPHmRsJ06cQEFBAQIDAxkbi8XC6NGj8eXLF9y5c6fU9Y6KisKZM2cwdOhQkU4+oPC8wsLChGxeXl5M/SgUCuVnJz8/H3PmzIG7uzt0dXWhqamJFi1a4OrVq4xPbGwsjI2NAQChoaHMs7CoBn39+jV69eoFAwMDqKmpoUGDBiLLHAimEt+6dQsTJ06EsbExNDU10aNHD7HLN5w7dw6enp6MVmzYsCH27t0LoPA5p6ysLPa4ESNGQE9PD7m5uWLPOSwsDCwWCx8/fhRJmz59OlRUVBjdJE2ny4NAp799+xYDBgyArq4ujI2NMXv2bBBC8PnzZ3Tr1g06OjowMzPDihUrhI4vi1gFBARAS0sLnz59gre3N7S0tGBpaYkNGzYAAJ49e4Y2bdpAU1MT1tbWTHsXP4fiiFuTWVa4XC7mz5/PaCMbGxvMmDEDeXl5Qn6CZUX+/vtvuLm5QU1NDbVq1cLRo0dF8iytNpOlrYvnIfj9pKqqioYNG+L+/fsi9ZJVz4vjwYMH6NChA4yMjKCurg5bW1sMGTJEyKdorAV1k/RXlKioKHTs2BG6urrQ0NCAp6cnbt26JXPdKBRFoCP6KOWGjY0NmjZtin379qFTp04ACkVFWloa+vTpg7Vr14ocM3LkSERERGDw4MH4888/8eHDB6xfvx6PHz/GrVu3oKysDKDwgaelpYWJEydCS0sLV65cwZw5c5Ceno7ly5cDKHxgd+jQAXl5eRg7dizMzMwQFxeH06dPIzU1Fbq6ugqdV+/eveHo6IhFixaBEAIAWLhwIWbPng1fX18MGzYMycnJWLduHVq2bInHjx8zHQd//fUXRo4cCQ8PD4wfPx4xMTH4448/YGBggOrVq5dYrp+fH0JCQpCQkCD0o/+ff/7B169fmal2Fy9eRN++fdG2bVssXboUQOGaXbdu3cK4ceMUOueiWFtbAwD27NmDZs2alesQ9ps3b+LkyZMICgoCACxevBje3t6YMmUKNm7ciMDAQKSkpGDZsmUYMmQIrly5whwruI4aNmyIxYsXIzExEWvWrMGtW7eEYvL333/Dx8cHtWrVwuLFi/H9+3cMHjwY1apVE6mPrNenrNy+fRv169cXsV+8eBExMTEYPHgwzMzM8OLFC2zduhUvXrzA3bt3xYpAeQkJCcHixYsxbNgwNGrUCOnp6Xjw4AEePXqEdu3aASjscGzRogWUlZUxYsQI2NjY4P379zh16hQWLlwIoHCE5+3bt9GnTx9Uq1YNsbGx2LRpE1q1aoWXL19CQ0MDLVu2xJ9//om1a9dixowZqFmzJgCgZs2aWL16NcaOHQstLS2mA9HU1BRA4fQYT09PxMXFYeTIkbCyssLt27cxffp0xMfHY/Xq1ULnFB4ejtzcXIwYMQKqqqrMqDxZadWqFQ4cOICRI0di4sSJzNTdo0ePMvcVAJg1axZOnTqFoUOH4tmzZ9DW1saFCxewbds2zJ8/H3Xr1pVaVkFBgdhRzYr8uBEQFxeHpKQkNGjQQCStUaNGOHv2LPP58ePH0NTUZGJR1E+QXnzZBXH8+PFD6DOHw4G+vj4AMD9CBw4cKPM56Orqwt7eHrdu3cKECRNkPo5CoVCqIunp6di+fTuz1ElGRgb++usvdOjQAffu3YObmxuMjY2xadMmjB49Gj169EDPnj0BAHXq1AEAvHjxAs2aNYOlpSWmTZsGTU1NHDx4EN27d8eRI0fQo0cPoTLHjh0LfX19zJ07F7GxsVi9ejXGjBmDAwcOMD4REREYMmQIXFxcMH36dOjp6eHx48c4f/48+vXrh4EDB2LevHk4cOAAxowZwxwneGHv4+MjcXSTr68vpkyZgoMHDyI4OFgo7eDBg2jfvj309fXLTaf7+fmhZs2aWLJkCc6cOYMFCxbAwMAAW7ZsQZs2bbB06VLs2bMHkydPRsOGDdGyZcsyixUA8Hg8dOrUCS1btsSyZcuwZ88ejBkzBpqampg5cyb69++Pnj17YvPmzRg0aBCaNm0KW1tbhc5VFoYNG4bIyEj06tULkyZNQlRUFBYvXoxXr16JdIRFR0fDz88Po0aNgr+/P8LDw9G7d2+cP3+e0YZloc1kaeui7N27FxkZGRg5ciRYLBaWLVuGnj17IiYmhtHe8uj54iQlJaF9+/YwNjbGtGnToKenh9jYWLGdnAKMjY2xa9cuIVtBQQEmTJggNGLxypUr6NSpE9zd3ZkXxeHh4WjTpg1u3rzJ6C4KpcwhFEoZEx4eTgCQ+/fvk/Xr1xNtbW2SnZ1NCCGkd+/epHXr1oQQQqytrUmXLl2Y427evEkAkD179gjld/78eRG7IL+ijBw5kmhoaJDc3FxCCCGPHz8mAMihQ4ck1vXDhw8EAAkPDxdJA0Dmzp3LfJ47dy4BQPr27SvkFxsbSzgcDlm4cKGQ/dmzZ0RJSYmx5+fnExMTE+Lm5kby8vIYv61btxIAxNPTU2I9CSHkzZs3BABZt26dkD0wMJBoaWkxbTJu3Diio6NDuFxuifnJiqenJ3FxcWE+8/l84unpSQAQU1NT0rdvX7Jhwwby8eNHkWOLXgvyAoCoqqqSDx8+MLYtW7YQAMTMzIykp6cz9unTpxMAjK+grV1dXUlOTg7jd/r0aQKAzJkzh7G5ubkRc3Nzkpqaytj+/vtvAoBYW1szNnmuT09PT6nxLCgoICwWi0yaNEkkTdz1vW/fPgKA3Lhxg7EJ2rdoG8lSNiGE1K1bV+j7J46WLVsSbW1tkdjy+fwS63rnzh0CgOzcuZOxHTp0iAAgV69eFfF3cXERW+f58+cTTU1N8vbtWyH7tGnTCIfDIZ8+fSKE/Pc91tHRIUlJSSWeU0lwuVwyZswYoqysTAAQAITD4ZBNmzaJ+D579oyoqKiQYcOGkZSUFGJpaUkaNGhACgoKpJZjbW3N5C/pT9J96/79+xLvWYK0ou0uIDg4mABg7o9dunQhdnZ2In5ZWVkEAJk2bVqJ5yC4Hxb/K/qd6dGjBwFAUlJSSsyrOO3btyc1a9aU6xgKhUKpaGTROFwuV0jzEUJISkoKMTU1JUOGDGFsycnJIrpTQNu2bUnt2rWZ+zchhc9hDw8P4ujoKFIfLy8voef0hAkTCIfDYXROamoq0dbWJo0bNxbSSIJ8BTRt2pQ0btxYKP3o0aMSn+VFadq0KXF3dxey3bt3T+gZJYtOlwfBc2nEiBGMjcvlkmrVqhEWi0WWLFnC2FNSUoi6ujrx9/cX8i1trPz9/QkAsmjRIpGyWCwW2b9/P2N//fq1xN8axVFU7z158oQAIMOGDROyT548mQAgV65cYWwCbXLkyBHGlpaWRszNzUm9evUYW1loM1nbWpCHoaEh+fHjB2M/ceIEAUBOnTrF2GTV8+I4duyYTL9XJMVdQGBgIOFwOEy78vl84ujoSDp06CCinW1tbUm7du1KLI9CKQ106i6lXPH19UVOTg5Onz6NjIwMnD59WuK03UOHDkFXVxft2rXDt2/fmD93d3doaWkJDecuOo0uIyMD3759Q4sWLZCdnc3s+ip4E3jhwgVkZ2eX2TmNGjVK6PPRo0fB5/Ph6+srVG8zMzM4Ojoy9X7w4AGSkpIwatQooTc9AQEBMr21dHJygpubm9AbWR6Ph8OHD6Nr165Mm+jp6SErK0tkCmZZwWKxcOHCBSxYsAD6+vrYt28fgoKCYG1tDT8/vzJdo69t27ZCGzcIFvn38fGBtra2iD0mJgbAf20dGBgo9Ma5S5cuqFGjBs6cOQMAiI+Px5MnT+Dv7y8Ug3bt2qFWrVpCdZHn+pSFHz9+gBDCjH4qStHrOzc3F9++fUOTJk0AAI8ePZKrHEno6enhxYsXiI6OFpuenJyMGzduYMiQIULTQAEIjSgsWteCggJ8//4dDg4O0NPTK3VdDx06hBYtWkBfX1+ozb28vMDj8XDjxg0hfx8fH2ZKjSJwOBzY29ujQ4cOiIyMxIEDB9C1a1eMHTsWx48fF/J1dXVFaGgotm/fjg4dOuDbt2+IjIyUeYRr48aNcfHiRZG/4lNZ5SEnJwcAxG50JPgeCHxycnJk8pPGkSNHhOq/Z88eJi09PR0AhL6rsiCIN4VCofzscDgcRvPx+Xz8+PEDXC4XDRo0kOkZ+ePHD1y5cgW+vr6M3v327Ru+f/+ODh06IDo6WmRJkhEjRgg9p1u0aAEej8dMpb148SIyMjIwbdo0kVF5RY8bNGgQoqKimCVbgMLZHNWrV4enp2eJ9fbz88PDhw+Fjj1w4ABUVVXRrVs3AOWn04cNG8b8n8PhoEGDBiCEYOjQoYxdT08Pzs7OjG4U+JYmVpLqIChLU1NTaE1mZ2dn6OnpCdWhrBGM5J84caKQfdKkSQDA6GEBFhYWQiNEdXR0MGjQIDx+/BgJCQkAykabydvWfn5+Qnq5RYsWAP7T/fLoeXEIZvmcPn0aBQUFUv3FsXPnTmzcuBHLli1jllp68uQJoqOj0a9fP3z//p1pq6ysLLRt2xY3btwQmu5MoZQldOoupVwxNjaGl5cX9u7di+zsbPB4PKG15IoSHR2NtLQ0mJiYiE1PSkpi/v/ixQvMmjULV65cYX5MChBMfbO1tcXEiROxcuVK7NmzBy1atMAff/zBrNuhKMWH10dHR4MQAkdHR7H+giHlAoFV3E9ZWVnmRef9/PwwY8YMxMXFwdLSEteuXUNSUhL8/PwYn8DAQBw8eBCdOnWCpaUl2rdvD19fX3Ts2FHmc5SGqqoqZs6ciZkzZyI+Ph7Xr1/HmjVrcPDgQSgrK2P37t1lUk7xDiZB3IpPcxbYBWu+CNra2dlZJM8aNWrgn3/+EfITFztnZ2chsSHP9SkP5P/Tv4vy48cPhIaGYv/+/SL5yju1UyDMBOjq6kJdXR3z5s1Dt27d4OTkBFdXV3Ts2BEDBw5kpp8IxJOrq2uJ+efk5GDx4sUIDw9HXFyc0PmUZhoqUNjm//77r8TOu+JtU9qpL0uWLMGaNWsQHR3N7CTr6+uL1q1bIygoCN7e3kIdecHBwdi/fz/u3buHRYsWySQmBRgZGTHr0RWlNFPhBZ2uxdfdAcCspSTwUVdXl8lPGi1btpS4GYeOjg6Awpcxxdc9LAlCSJlMT6dQKJSqQGRkJFasWIHXr18LdSLI8sx69+4dCCGYPXs2Zs+eLdYnKSkJlpaWzOfi2knQQSLQSILON2nPdz8/P4wfPx579uzBnDlzkJaWhtOnT2PChAlS79G9e/fGxIkTceDAAcyYMQOEEBw6dAidOnVing3lpdPFaUc1NTWRZ5Wurq7I2seliZUANTU1Ed2iq6uLatWqibSbrq6uwus8y8LHjx/BZrPh4OAgZDczM4Oenp7IOooODg4idXRycgJQuCadmZlZmWkzedpa2jUtj54Xh6enJ3x8fBAaGopVq1ahVatW6N69O/r16yf2pWhxnjx5glGjRqFv375CnaqCl+n+/v4Sj01LSxP70p9CKS20o49S7vTr1w/Dhw9HQkICOnXqJPEHH5/Ph4mJidCIkKIIHiipqanw9PSEjo4O5s2bB3t7e6ipqeHRo0eYOnWq0JuRFStWICAgACdOnMDff/+NP//8E4sXL8bdu3fFPnAFFN/UoyjFfwDz+XywWCycO3dO7C6Vgg6DssDPzw/Tp0/HoUOHMH78eBw8eBC6urpCnXgmJiZ48uQJLly4gHPnzuHcuXMIDw/HoEGDEBkZWWZ1EWBubo4+ffrAx8cHLi4uOHjwICIiIspk7T5Ju35KsovrNCsrZL0+ZcXAwAAsFkuswPP19cXt27cRHBwMNzc3aGlpgc/no2PHjnK/+TM3Nxf6HB4ejoCAALRs2RLv379nvhvbt2/HqlWrsHnzZqE30dIYO3YswsPDMX78eDRt2hS6urpgsVjo06dPqd9S8vl8tGvXDlOmTBGbLhCfAmTtnJLExo0b0aZNG5Hv7B9//IGJEyciNjZWSCzHxMQwIk6wSUxlIoh1fHy8SFp8fDwMDAwYwWpubo6rV6+KdKoJjrWwsCh1fWrUqAGgsG0Eb99lISUlpcSdfCkUCuVnYffu3QgICED37t0RHBwMExMTcDgcLF68WGi0myQEz9HJkyejQ4cOYn2Kd+KUlUbS19eHt7c309F3+PBh5OXlYcCAAVKPtbCwQIsWLXDw4EHMmDEDd+/exadPn5i1owVI0+mKIO78ZWmT0sZKWlmy1EGR3yWyUJYvz8pCm8nb1uWt+1ksFg4fPoy7d+/i1KlTuHDhAoYMGYIVK1bg7t27Jf6WS0lJgY+PD5ycnLB9+3ahNMH3d/ny5SLrDgooy9+JFEpRaEcfpdzp0aMHRo4cibt37wpNOy2Ovb09Ll26hGbNmpX4g/3atWv4/v07jh49yiygCwAfPnwQ61+7dm3Url0bs2bNwu3bt9GsWTNs3ryZmXoKQGS6qbidwkqqNyEEtra2Ig+3ogg2sYiOjkabNm0Ye0FBAT58+CDTAv62trZo1KgRszjy0aNH0b17d5G3TSoqKujatSu6du0KPp+PwMBAbNmyBbNnzxYRhGWFsrIy6tSpg+joaGbqcmUhaOs3b94ItbXAJkgvGpPivHnzRuizrNenrCgpKcHe3l7kuk1JScHly5cRGhqKOXPmMHZJU2ylUXwKt4uLC/N/AwMDDB48GIMHD0ZmZiZatmyJkJAQDBs2jBll+vz58xLzP3z4MPz9/YV2r8vNzRX5TpUkMiWl2dvbIzMzU+zIt/IgMTFRrJgWvGnmcrmMjc/nIyAgADo6Ohg/fjwWLVqEXr16MQtzVwaWlpYwNjbGgwcPRNKKL27t5uaG7du349WrV0IjEaOiopj00tK1a1csXrwYu3fvlqujT9b7IYVCoVR1Dh8+DDs7Oxw9elToWTd37lwhP0nPQcGzWFlZucyehfb29gAKn+/SNOGgQYPQrVs33L9/H3v27EG9evWEdERJ+Pn5ITAwEG/evMGBAwegoaGBrl27iviVpNMrktLGqiwo+ruk6MAIeX6XFMXa2hp8Ph/R0dFCm28lJiYiNTWV0cECBCNIi57j27dvAYBZSqcstJmsbS0r8uj5kmjSpAmaNGmChQsXYu/evejfvz/2798v8QU4n89H//79kZqaikuXLkFDQ0MoXfBd09HRqTAtS6EIoGv0UcodLS0tbNq0CSEhIWIf8AJ8fX3B4/Ewf/58kTQul8t0HAje6hR9i5Ofn4+NGzcKHZOeni70wxwoFBNsNpuZsqajowMjIyOR9SSK51USPXv2BIfDQWhoqMibJUIIMy2gQYMGMDY2xubNm5Gfn8/4REREyLWunZ+fH+7evYsdO3bg27dvQtN2AYhMQ2Cz2cx0TMF5FxQU4PXr12JH/kgjOjoanz59ErGnpqbizp070NfXL9U6aWVBgwYNYGJigs2bNwtNTzx37hxevXqFLl26ACgc1eTm5obIyEihaaYXL17Ey5cvhfKU9fqUh6ZNm4p0yoi7vgGI7GImK15eXkJ/glFfxa8TLS0tODg4MO1lbGyMli1bYseOHSLxLlo3DocjUtd169aJdJhpamoCEO1UF6SJs/v6+uLOnTu4cOGCSFpqaqrI97u0ODk54eLFi0Jtw+PxcPDgQWhrazOCDQBWrlyJ27dvY+vWrZg/fz48PDwwevToSl9bzsfHB6dPn8bnz58Z2+XLl/H27Vv07t2bsXXr1g3KyspC9zpCCDZv3gxLS0t4eHiUui5NmzZFx44dsX37dpE1DoHC+/bkyZOFbGlpaXj//n2ZlE+hUCiVjbhnelRUFO7cuSPkJ+ggKP4sNDExQatWrbBlyxaxmi05OVnuOrVv3x7a2tpYvHgxs1yDgOLP806dOsHIyAhLly7F9evXZRrNJ8DHxwccDgf79u3DoUOH4O3tzWgBQDadDgCfPn1i1t8uT0obq7JAoDOK/i7JyspSeEZO586dAYhqyJUrVwIAo4cFfP36VWgn3vT0dOzcuRNubm7MC/yy0GaytrWsyKPnxZGSkiJy7QteeIpb5kRAaGgoLly4gH379omdcuzu7g57e3uEhYUhMzNTJF2R7y+FIit0RB+lQihpbQIBnp6eGDlyJBYvXownT56gffv2UFZWRnR0NA4dOoQ1a9agV69e8PDwgL6+Pvz9/fHnn3+CxWJh165dIjfoK1euYMyYMejduzecnJzA5XKxa9cucDgc+Pj4MH7Dhg3DkiVLMGzYMDRo0AA3btxg3l7Jgr29PRYsWIDp06cjNjYW3bt3h7a2Nj58+IBjx45hxIgRmDx5MpSVlbFgwQKMHDkSbdq0gZ+fHz58+IDw8HCZ1+gDCh+wkydPxuTJk2FgYCDyhmjYsGH48eMH2rRpg2rVquHjx49Yt24d3NzcmLd5cXFxqFmzJvz9/RERESFz2QDw9OlT9OvXD506dUKLFi1gYGCAuLg4REZG4uvXr1i9erXIEPsdO3bg/PnzInmNGzdO7oX6ZUFZWRlLly7F4MGD4enpib59+yIxMRFr1qyBjY0NJkyYwPguXrwYXbp0QfPmzTFkyBD8+PED69atg4uLi9BDWdbrUx66deuGXbt24e3bt8xoUB0dHbRs2RLLli1DQUEBLC0t8ffff0scsaootWrVQqtWreDu7g4DAwM8ePAAhw8fxpgxYxiftWvXonnz5qhfvz5GjBgBW1tbxMbG4syZM3jy5AkAwNvbG7t27YKuri5q1aqFO3fu4NKlSzA0NBQqz83NDRwOB0uXLkVaWhpUVVXRpk0bmJiYwN3dHZs2bcKCBQvg4OAAExMTtGnTBsHBwTh58iS8vb0REBAAd3d3ZGVl4dmzZzh8+DBiY2NlmuLZqlUrXL9+XeoUj2nTpmHAgAFo3LgxRowYAXV1dezbtw8PHz7EggULmPU2X716hdmzZyMgIIB5eREREQE3NzdmjcyyZv369UhNTcXXr18BAKdOncKXL18AFE6fFqxnNGPGDBw6dAitW7fGuHHjkJmZieXLl6N27doYPHgwk1+1atUwfvx4LF++HAUFBWjYsCGOHz+OmzdvYs+ePRKnycjLzp070b59e/Ts2RNdu3ZF27ZtoampiejoaOzfvx/x8fFCG5BcunQJhBBmsXYKhUKp6pSkcby9vXH06FH06NEDXbp0wYcPH7B582bUqlVLSGOoq6ujVq1aOHDgAJycnGBgYABXV1e4urpiw4YNaN68OWrXro3hw4fDzs4OiYmJuHPnDr58+YKnT5/KVV8dHR2sWrUKw4YNQ8OGDdGvXz/o6+vj6dOnyM7OFupUUlZWRp8+fbB+/XpwOBz07dtX5nJMTEzQunVrrFy5EhkZGSIvpmXV6YMGDZLpGV5ayiJWpaV9+/awsrLC0KFDERwcDA6Hgx07dsDY2FjsS3Zp1K1bF/7+/ti6dSuz9NG9e/cQGRmJ7t27M5tGCHBycsLQoUNx//59mJqaYseOHUhMTER4eDjjUxbaTNa2lgdZ9bw4IiMjsXHjRvTo0QP29vbIyMjAtm3boKOjw3SWFufZs2eYP38+WrZsiaSkJJH1yQcMGAA2m43t27ejU6dOcHFxweDBg2FpaYm4uDhcvXoVOjo6OHXqlELnS6FIpfw39qX8bgi2gJe2Rbm1tTXp0qWLiH3r1q3E3d2dqKurE21tbVK7dm0yZcoU8vXrV8bn1q1bpEmTJkRdXZ1YWFiQKVOmkAsXLhAA5OrVq4QQQmJiYsiQIUOIvb09UVNTIwYGBqR169bk0qVLQuVlZ2eToUOHEl1dXaKtrU18fX1JUlKSxC3vk5OTxZ7PkSNHSPPmzYmmpibR1NQkNWrUIEFBQeTNmzdCfhs3biS2trZEVVWVNGjQgNy4cYN4enoST0/PEturKM2aNSMAyLBhw0TSDh8+TNq3b09MTEyIiooKsbKyIiNHjiTx8fGMj2C7en9/f6lleXp6EhcXF+ZzYmIiWbJkCfH09CTm5uZESUmJ6OvrkzZt2pDDhw8LHSu4FiT9ff78WWK5AEhQUJCQTVDv5cuXC9mvXr1KAJBDhw4J2Q8cOEDq1atHVFVViYGBAenfvz/58uWLSFlHjhwhNWvWJKqqqqRWrVrk6NGjxN/fn1hbW4v4ynJ9yhrPvLw8YmRkRObPny9k//LlC+nRowfR09Mjurq6pHfv3uTr168i16SgfT98+CB32QsWLCCNGjUienp6RF1dndSoUYMsXLiQ5OfnC/k9f/6cqYuamhpxdnYms2fPZtJTUlLI4MGDiZGREdHS0iIdOnQgr1+/JtbW1iLX17Zt24idnR3hcDhC39WEhATSpUsXoq2tTQAI1T8jI4NMnz6dODg4EBUVFWJkZEQ8PDxIWFgYU1dJ14UAd3d3YmZmJrVNCCHk/PnzxNPTkxgZGREVFRVSu3ZtsnnzZiady+WShg0bkmrVqpHU1FShY9esWUMAkAMHDpRYhqR7HyGSr2Vra2uJ36Oi8SekMGbt27cnGhoaRE9Pj/Tv358kJCSIlMXj8ciiRYuItbU1UVFRIS4uLmT37t0l1l2AtPthUbKzs0lYWBhp2LAh0dLSIioqKsTR0ZGMHTuWvHv3TsjXz8+PNG/eXKY6UCgUSmUii8bh8/nMfVZVVZXUq1ePnD59WqzGuH37NnF3dycqKioiz/v379+TQYMGETMzM6KsrEwsLS2Jt7e3kO6SpL8FzxXBM1fAyZMniYeHB1FXVyc6OjqkUaNGZN++fSLnee/ePQKAtG/fXu422rZtGwFAtLW1SU5OjlCarDrd09OTyPKTVdJzyd/fn2hqaor4F9e3ZRErWcsSIE4PPHz4kDRu3JjR8CtXriyV3isoKCChoaHE1taWKCsrk+rVq5Pp06eT3NxcsXW5cOECqVOnDlFVVSU1atQQ0SOElF6bydrWJeVR/DtCiHx6viiPHj0iffv2JVZWVkRVVZWYmJgQb29v8uDBA4llCr5Xkv6K8vjxY9KzZ09iaGhIVFVVibW1NfH19SWXL18usV4USmlgEVLOr0coFAqFIpH58+cjPDwc0dHRZTaKivIfGRkZMDAwwOrVqxEUFFTZ1aGUQEJCAmxtbbF//346oo9CoVCqCE+fPoWbmxt27tyJgQMHVnZ1KOWEjY0NXF1dcfr06cquCoVCKQPoGn0UCoVSiUyYMAGZmZnYv39/ZVfll+TGjRuwtLTE8OHDK7sqFCmsXr0atWvXpp18FAqFUoXYtm0btLS0KnWzKQqFQqHIBx3RR6FQKBQKhUKhUCgUhlOnTuHly5eYPXs2xowZw2zgQPk1oSP6KJRfC7oZB4VCoVAoFAqFQqFQGMaOHYvExER07twZoaGhlV0dCoVCocgBHdFHoVAoFAqFQqFQKBQKhUKh/ALQNfooFAqFQqFQKBQKhUKhUCiUXwA6dbcE+Hw+vn79Cm1tbbBYrMquDoVCoVAoFIpECCHIyMiAhYUF2Gz6LlcWqNajUCgUCoXyMyCPzqMdfSXw9etXVK9evbKrQaFQKBQKhSIznz9/RrVq1Sq7Gj8FVOtRKBQKhUL5mZBF59GOvhLQ1tYGADx8+BA2NjZMrymfz8e3b99gZGRUoW/My6JcefOQ1V+an6R0eeyy2ioCGouqEYuyKpPGovTQWNBY0FiIUtGxSE9Ph7W1NaNfKNIRtNXHjx+ho6NTpa/jn1nzKJJPWcRCkTQaC8X9aSzKPx8aC1FoLGgsfpdYyKPzaEefGDZs2IANGzaAx+MBAHg8HnJzc4WCV9xWEZRFufLmIau/ND9J6fLYZbVVBDQWVSMWZVUmjUXpobGgsaCxEKWiY5GXlwcAdAqqDBTXenl5ecjNza3S1/HPrHkUyacsYqFIGo2F4v40FuWfD42FKDQWNBa/Syzk0Xm0o08MQUFBCAoKQnp6OnR1daGrqwsTExOh4LFYLBgbG1f4F6e05cqbh6z+0vwkpctjl9VWEdBYVI1YlFWZNBalh8aCxoLGQpSKjoWamprCZfxuFNd6xsbGzIi+qnod/8yaR5F8yiIWiqTRWCjuT2NR/vnQWIhCY1G6WBQUFCAyMhxPnzyBsooybKxt8Oe4CQgcPQJKyspQUVEBAPTo4YNmzZoDgMQ0Pp+PDevXIjs7Cyw2G+pq6hgyZBhs7exKLEtam338+BE7/tqGtLRUAEDfvv3RuElTvHj+HAsXzYeFuQW4XC6UlJSwcNESqKqq/pdmYcHku2DBIrBYLKSk/MD27VsBADwuDzVq1kRAwBCmzM+fPwuV59enH+zs7Cv9eyGPzqMdfTLAYrHAZrOFAiPOVll1Ke88ZPWX5icpXR47jQWNRXHKqkwai9JDY0FjQWMhSkXGoqKfg78SRdu2Kl/HP7PmUSSfsoiFImk0For701iUfz40FqLQWCgei717doHNYmP9hk3/7wRL+f8xLEyaFAxbWztxZyQxbeiwEcyyZ1F372DDhnVYuWqNlLLElPD/+hcUFGDZssX488/xqFmzFng8HjIzM8Fms8Fis2BpYYnlYSuRlJQkNDhLkLZi5WomTz6fj/T0DNja2mHZshVQUlICn8/H8mVLcPHvC2jUuInY8tLT05GXl1fp3wt5jqeKkEKhUCgUCoVCoVAoFArlNyI3NxeXL19Cv/4DwGIVTgfV19cvVZ4aGhrM/7Oys5l8SyorLu4Lhg8bgoSEBADAyRPHsX79GvD5fNy8cR1OTs6oWbMWAIDD4UBXV7dUdVRVVYWSUuGYNy6Xi/z8fKZO/9y8UeblVQZ0RF8ZwOPxUFBQUO7l8Pl8FBQUlHpdOHnykNVfmp+kdHnsstoqAhqLqhGLsiqTxkI+lJWVweFwyjxfCoVCqaoU13qVcX//mTWPIvmUxbNWkTQaC8X9aSzKPx8aC1HElUu1qmwkJCRAS0sbR44cwrN/n0JFRRW+fn1Qp05dAMDatasBAjg4OmLAgEFCHV4lpa1buwYvXjwDAMycNUdqWZaW1TDIPwArVyzDIP/BOH/hHCZNmgI2m43PXz5DWUkJixbOx/fv32FtbQ3/gCFMeQkJ8ZgSPAk8Hg/t23dAp85dipxfPCZPmgA2m402bdqifYeOTFpSUiKWLF6ExMQE1K/vjvYdOiIlJUVseQMHBZRTBMoP2tEnA4QQ8Pl85jOfzwchBDweDwkJCUhNTa2wuvD5fGRkZFRoHrL6S/OTlC6PXVZbRUBjUTViUVZl0ljIh56eHkxNTZm3X4L7YtF7pSLIm4+s/tL8JKXLY5fVVt7QWPy+sajIc/vV4PP5zF/RtiaEIDExUazWq4z7+8+seRTJpyyetYqk0Vgo7k9jUf750FiIIq7c4lpVljwqWvcokiaLxpG1blxuAZKTk1DNshr69x+IDzExmD8/BCtXrUXovAUwNjYGl8vF/n17sW7tasyYORsAJKYJyg0aMxZsNhvXrl3Brp2RmDFzdoll6enpoVmz5nj27F/MnxeC2XNCoKWlBT6fDx6Xh3//fYqFi5bCwMAAe/fuxpYtmzB58hTY2Nhi85btUFdXx9u3b7Ft6yZo6+jAw6MZk6apqYnv379h0cIF0NTUhIOjE/h8PoyMjBG2YhVycnKwbu1q3L17B05OzmLL27Z1MwYOCijXWMiCPHnQjj4xFN+JLS0tDUlJSUJvL9LS0pCeno6CggKYmJhATU1N5puIogguEDabrXBZ8uYhq780P0np8thltVUENBZVIxZlVSaNhewQQpCbm4ukpCRkZWUx27sL7ouEkFK/2ZYnH1n9pflJSpfHLqutvKGx+H1jURk/rn5Wimu95ORkZtfdom2dkZGBvLw8Ea1XGff3n1nzKJJPWTxrFUmjsVDcn8ai/POhsRBFXD3EaVVpVIbuUSRNFo0j+7mwwGKx4FyjJpKSkqCppQV9fQP8++9T1Pi/DQAaN2mK8+fPMp8BiE0rXm6tWq7YumUzYmJipJbF4/EQ8/49NDQ08OHDB3A4SiCEQE1NDfb2juDxeEhOToaLS21cu3pFqC4ZGRlgsVioW7ceHj18AAcHRyYtKysLAODmVg+PHj2CkbGJSLu41q6DK5cvwcTEVGJ5qamp5RwL6cij82hHnxhk2XWXEIK0tDSYmZnB0NCwwupWUFAAZWXlCs1DVn9pfpLS5bHLaqsIaCyqRizKqkwaC9nR1tYGm81GUlISDA0NweFwwOf/vDuelZQuj11WW3lDY/H7xoLuuis7suy6SwjBjx8/JGq9yri//8yaR5F8yuJZq0gajYXi/jQW5Z8PjYUoxcsVp1WlURm6R5E0WTSOrHUzMTFB7dp18PVrHOrXd0diYiJSUn7AxcUVampq0NTUBABERd2BnZ09TExMkJubCx6PJzYtIyMDaWlpTN/JvXtR0NbWga2tLVgslsSyDA0NsTMyAtY2NpgwcTJCQ2ZjzJhxMDExQbv2HbBwQSi0tLSgoaGBO3duwfb/5aWk/ICurh4AIC8vD2/evkbbNl5CaWw2Gzk5OXjz5jVat24DPT098HhcGBkZQUlJCQUFBXj9+hUcHR2hp6eHdu3bY9HC+cLl2dpBT0+P7rr7qyFulxQ+v3DYpKamZoWOXhKUVZpRZPLkIau/ND9J6fLYZbVVBDQWVSMWZVUmjYX8CO59PB6PEVbi7pWKIG8+svpL85OULo9dVlt5Q2Pxe8aiIs/rV6No2wraWrA4tzitVxn3959Z8yiST1k8axVJo7FQ3J/GovzzobEQRVK54rSqNCpD9yiSJovGkbVuI0cFYuPGddizexdYLBZGjgoEj8dDaMhs8Ph8gACmpqb4c9x4sNlspKenI2z5ErFpubk52LZ1M/iEDzaLDR1dHcyYOYvpaBVXlrGxMR48uI8nTx5j6bIwqKqqwt9/MHbs2IYlS8NgamqKnj69MWvmdLDYLBgaGGLU6ECw2WxERUXhwoVz4LA5yMvPQ4vmLdHWqx1YLJZQGo/Pg0fTZmjT1gvJycl48eIFwpYvA5vNBo/PQ53addCrtx9SU1NhYmIiUt6IkaOZUaPlGQtpyHM87egrJRV5E6NQKJSqAr33USiU3wV6v6NQKJSfD3rvlg0zMzPMm7dQxB62YrVEf0lpxsYmmDJ1utBsSFnKatCgIRo0aMh8burRDPYOjlBRUQEAtGrVGq1atRY5rnPnLujcuQv4fD6SkpJgYmLCxF2QVhTBYK127dqjQ5GNOYqmiStPkP/PBH31S6FQKBQKhUKhUCgUCoVCofwC0I4+CoVCoVAoFAqFQqFQKBQK5ReAdvRVAXh8gnsxWTjzJA33YrLA45PKrtIvz7Vr18BisZCamlqi3+XLl1GzZk1mV77yJiQkBG5ubhVSVmXAYrFw/PhxAEBsbCxYLBaePHlS7uUGBASge/fuzOdWrVph/Pjx5V5ucc6fPw83NzehoeEUCuXXhscnuB+TTZ/xvzFU51U8suo8CoVCoVBKS9Hn/P2Y7CrxnKdr9FUyF5+nY/HpRCSmcRmbqa4Spnubop2rTiXWjAIAU6ZMwaxZs2TaqamiaN26Ndzc3LB69eoyyzMgIACpqak4dOhQmeUpL9euXUPr1q2RkpICPT29SquHJMSt89GsWTP8888/zOerV69i+fLliIqKQk5ODmxsbNChQwdMmjQJ1apVQ8eOHTF79mzs2bMHAwcOrMjqUyiUSuDSiwwsOpmG5MxUxkaf8b8XF59nYMmZYjpPh4Np3mZoX5teAxQKhUKh/MzcfJePTeExSEz/7zlvrMXCjD800L62bqXVi3b0yQAhRGgEDp/PByGESRP8X14uPs/AxL1xKH50UhoXE/bEYWU/oJ2rtkhdiv6rCPLmIau/ND9J6fLYi7c7j8dTqE1kid8///yD9+/fo2fPnmJ9yiMW8rRhSeVWdCxkLbdo3Yufi6LfKXniULyu8tZ9x44d6NixcOHWgoICaGpqMv5btmxBUFAQBg0ahMOHD8PGxgafPn1CZGQkVqxYgZUrVwIA/P39sXbtWgwYMKDEcisiFqVB0H58Pp/5K36vVAR585HVX5qfpHR57LLayhsai6oRi0svMjBx79fCZzwhcOR9RLSSTZFnPIGXi7a0bADI17YUxRB3LxP8X9JzSdp99tKLDAQfSBTVeek8TNwrqvNkuZeX971dEmVVrrzP7IKCAigpKcmtQWV9HpbmWfs7xUIef0Xau6Q0GgvF/X/3WBTXqtKoDN2jSJosGqcy9GdZlvuzxOLvZ+kIOZMFANDlp0MFBUhmGyI5k2Di3q9Y2Q9oU1OzzGIhTx60o08MGzZswIYNG5jpmmlpaUhKSmJ2juHz+cjIyAAhBFwuF1wut6TsxMLjEyw5nSAi/gCAAGABWHI6ES2d1MBh/7eFuaBOiu4iRAiBl5cXXF1doaSkhF27dkFFRQWhoaHo06cPxo0bh6NHj8LU1BSrVq1Chw4dmDJfvHiB6dOn459//oGmpia8vLwQFhYGIyMjEEJw7tw5LFu2DC9evACHw0GTJk2wYsUK2NvbgxCCnJwcjBs3DsePH0dKSgpMTU0xfPhwTJo0CbGxsXB2dsa9e/fg5uYGQgi+f/8OCwsLXLx4EZ6enrh27Rrat2+PEydOICQkBM+fP8eZM2fg4eGBZcuW4a+//kJCQgIcHR0xY8YM+Pj4MOd97tw5TJo0CV++fEHjxo2Z0VQlxW/fvn1o27YtlJSUGJ958+bh5MmTGD9+PEJCQpCamooOHTpg8+bN0NYuFOt8Ph8rV67EX3/9hc+fP8PU1BTDhg3D9OnTAQBfvnzBtGnTcPHiReTl5aFGjRpYu3YtGjVqxNxExNWJEIKhQ4fi+vXruH79OtauXQsAePv2LWxsbPD8+XOJ8QGAI0eOYMGCBXj//j00NDRQt25dHD16FCtWrEBkZCQAQFVVFQBw8eJFNG3aFMHBwTh27JhQvKZOnSr2Wrx//z5mz56Np0+foqCgAHXr1kVYWBjq1asndB48Hk+o3blcLt69e4c2bdoAAAwMDAAAAwcOxF9//YW8vDxMmzYNBw8eRHp6Otzd3REWFoYGDRqAEIL8/HyMHDkS165dQ0JCAqpXr45Ro0Zh7NixTJmCB7ygTMGDv+hnWb5b2trazPXO4/HA4XDA5XLx5csXjBs3DmPGjEFYWBjjb2lpiUaNGiEjI4Mpq1OnThg7dizevHkDOzs7seVKqo84u6y2sobL5YLP5+P79+9QVlYGn89HWloaCCGl2jpe3nxk9ZfmJyldHrustvKGxqLyY8HjEyw6mQYCQJ+fhknZ4WjGfYwxWrPwQskRALDoZDxcDLOZZ3xJyFr3jIyMsjqFX57iWi85ORm5ublCbc3j8ZhnR/HnsrT7LJfHx7Jz32TWebLcy+W5twu0HofDkar1BC+wJOkIQ0ND8Hg8XLhwAUuWLBGr8wAgPz9fom6IjY2Fk5MT7ty5g/r16zPTak1MTBidd/36dbRr1w4nT57E3Llz8fz5c5w6dQqenp4ICwuTqPMIIThz5gymTJkiovMKCgoktllpnrUV8ZwVR1mVK28+svqX5KdIGo2F4v40FqJaVRqVoXsUSZNF41SG/pR2PuWZT2XEgscnWHw6DQDQIv8+JuVE4CPbAuO1poOw2CAo1Ho19bWRmZFeJrGQR+fRjj4xBAUFISgoCOnp6dDV1YWurq7QFtEC0ZeZmQklJSUoKcnfjI9ispCYLnndNwIgMZ2Lp1/y0chOUyhNlhtVSbBYLOzevRvBwcGIiorCgQMHMGbMGJw8eRLdu3fHzJkzsWrVKgwePBgfP36EsrIysrKy0KFDBwwdOhSrVq1CTk4Opk2bhv79++Py5csAgNzcXEycOBF16tRBZmYm5s6dC19fXzx+/BhsNhsrV67EmTNncODAAVhZWeHz58/4/PkzOBwOc05F21MwXZbD4QjZZ8+ejeXLl8POzg76+vpYsWIF9u3bh02bNsHR0RE3btxAQEAAzMzM4Onpic+fP8PX1xeBgYEYMWIEHjx4gMmTJ4uUV5xbt26hb9++QulsNhsxMTE4ffo0jh8/joyMDPj5+SEsLAwLFxZuFT516lRs374dK1euRPPmzREfH4/Xr19DSUkJmZmZ8PLygqWlJU6cOAFDQ0M8e/YMLBYLSkpKYLPZzP/FsXLlSrx//x4uLi6YN28eAMDY2BgZGRlC8cnIyMCsWbOY+MTHx2PgwIFYunQpevTogZSUFNy5cwccDgdTpkzB27dvkZ6eji1btkBZWRkGBgZYu3YtTp8+jQMHDsDc3BwJCQn4/PmzUN2KXos5OTkYOHAg1q9fD0IIVqxYgT/++ANv375lOkHFxVNJSQm2trY4fPgwevXqhdevX0NHRwfq6upQUlLCpEmTcOzYMURERMDa2hrLly9Hly5dEB0dDQMDAxQUFKB69eo4ePAgDA0Ncfv2bYwcORKWlpbw9fVl4sZms5kyWSyW2HaW9t0S1L24/7Fjx5Cfn4+pU6eKjZ2xsTHzfzs7O5iamuLOnTtwdnYusVx57LLaygrB9WpoaAg1NTXw+XywWCwYGxuX+sEuTz6y+kvzk5Quj11WW3lDY1H5sbgfk43kzFS0zo/C+JxI6JFMAMD07K0YrL0IBSxlJGcSfMnWRkM7Dan5yVp3NTW1MjuHX53iWs/Y2Bg6OjpCbZ2fn4+MjIwStYKk++yjT1lIUkDnyXIvl+XezmKxsGvXLpm1Xn5+vlSdl5eXV6LOW716NaMbiuq84tquqOYT2JSUlBjdN2vWLEbnaWlpISwsDHv27ClR5/Xt21eszlNWVhYqUxyledaW53O2JMqqXHnzkdVfkfYuKY3GQnH/3zkWxbWqNCpD9yiSJovGqQz9Ke18yjOfyojF/Zhs5Kd/xuycXfAquAsA0Oe9Qbf8Kziu6gUAhVovRxs2euwyiYU8Oo929MkAi8ViOgqK2gT/KvLG4luGbJs7fMvgCb01KVquIgiGM9etWxezZ88GAMyYMQNLly6FkZERRowYAQCYO3cuNm/ejH///RcNGjTA+vXrUa9ePSxevJjJa8eOHahevTqio6Ph6OiInj17QklJianbjh07YGxsjFevXsHFxQVfvnyBo6MjWrRoARaLBRsbG5HRa4L2LH6uRc83NDQU7du3B1DYubh06VJcvHgRHh4eAAB7e3vcunULW7duRatWrbB582bY29szUydr1KiB58+fY+nSpSXG7+PHj7C0tBRKZ7FY4PP5CA8PZzqiBg4ciCtXroDFYiEjIwNr167F+vXrERAQAABwcHBAixYtABSOEkxOTsb9+/ehr68PLpeLmjVrisRVXJ0IIdDT04OKigo0NTVhbm7OpG3YsIGJj6BN//rrL1hZWSE6OhqZmZngcrnw8fGBlZUVqlWrhnr16jHlqKurIy8vD+bm5kwMP3/+DEdHRzRv3hw8Hg8ODg4lXott2rQBl8tljt+2bRv09PRw48YNeHt7i8S46PFKSkowNDQEAJiamjJr9GVlZWHz5s2IiIhA586dAQDbtm3DxYsXsWPHDkyePJkZpSDIz87ODnfv3sWhQ4fg5+cn1IbFYynvd6tfv35C6zXu2rULPXr0wLt376CjowMLCwuRmInL18LCAp8+fRKyF39rKqtdVltZI2i/ovdGcfdKRfOWJx9Z/aX5SUqXxy6rrbyhsajcWKQm/8DcrPVoU3CPsaWwtLFFzQ8FrP9+iHzL5JVp21bkNfarIeleVvyZJUDafVZenSfLvVzee7usWu/Zs2e4dOmSRJ339u1b2NnZwcfHR6jcojrP1dWV0Q1FdZ6A4vUtrgOKfp43bx7at28PQgiysrKwePFiXLp0CU2bNgUgqvM2bdoEOzs7rFixAiwWS0jnFS1b1hiWRyzKirIqV958ZPUvyU+RNBoLxf1pLMRrVWlUhu5RJE0WjVMZ+rMsy63qseA+v4mIjBUwJGmMz01ld1xXbih03PdMPmy1y6ZN5DmedvRVEsY6sjW9rH7yUrt2beb/HA4HhoaGQjZTU1MAQFJSEgDg33//xdWrV6GlpSWS1/v37+Ho6Ijo6GjMnz8fUVFR+PbtGzOH/NOnT3BxccGgQYPQqVMnODs7o2PHjvD29ka7du3krnuDBg2Y/7979w7Z2dlMx5+A/Px8Zsroq1ev0LhxY6F0gVgsiZycHLG95jY2NtDW1mY6KM3NzZl2evXqFfLy8tC2bVuxeT558gT16tWDgYEB0+kqiZs3b6JTp07M582bN4t0XAl4+vRpifFp37492rZti9q1a6NDhw5o27YtfH19mWmy4ggICEC7du1Qo0YNtG/fHl27dkWHDh0k+icmJmLmzJm4ceMGkpKSwOPxkJ2dzXRoKcL79+9RUFCAZs2aMTZlZWU0atQIr169YmwbNmxAeHg4Pn36hJycHOTn55fL7sWrVq2Cl5cX05lavXp1AMICQxbU1dWRnZ1d5vWjUCiVT97T66h7eCGUClIY2zXlhlip7o80tvDmC+X1jKdUPkbasm3iVZ7XQJ06dZj/S9N60nSEnZ0doqOjMXfuXLE6z9XVldENRXVecX0mC+J0XnG9WFTnvX79Go0aNRJKl0XnUSgUCoUiL/ysdGQdWgnnqDOMLYOlgdXqg3BJuSlQ7DehsXblaD2qMCsJdxsNmOoqISmNK3b9FhYKd+Zzt5E+pUcRig9tZrFYQjZBp4VAxGVmZqJr167M29GiCEaW9ejRAzY2Nti2bRssLCzA5/Ph6uqK/Px8AEC9evUQExOD8+fP49KlS/D19YWXlxf27dvH9E4X7fwqKCgQW3dNzf+muGRmFk6HOn36NKpVqybkJ1hvTlGMjIyQkpIiYhfXdoJ2UldXLzFPaelFadCgAZ48ecJ8NjExkehbND6CTijByDpzc3NwOBxcvHgRt2/fxoULF7BhwwbMmTMHUVFRsLW1FZtn/fr18eHDB5w9exYXL16En58fvLy8cPjwYbH+AQEB+PbtG1avXg0bGxuoqqqiadOmTPzLiwMHDiA4OBgrVqxA06ZNoa2tzex8W9aYmZnBwcFBqI0BwMnJCWlpaYiPjxcaaSmJHz9+CE3npVAoPz+Fwm8F8qLOMuIqjaWJ1er+uKLcWEj4lfcznlL5uNtowFSHg6R0XqXoPEA+rVeSzjMzMwMA/PHHH7C2tpao8wS64dy5c0I67/DhwwrrvKyswkXOz5w5A0tLSyG/0uo8CoVCoVDkIf/FbWTvWQR+WjJju61UF2EaQ/CdrS/kK3jO17dRx/dvmRVcU9rRV2lw2CxM9zbFhD1xYAFCIlDwU2Cat6lMi3RXBPXq1cPRo0dhY2Mjdp2ab9++4e3bt9i2bRtatmwJoHDX2uLo6OjAz88Pfn5+6NWrFzp27CjU6REfH8+8oX369KnUetWqVQuqqqr49OkTWrVqJdanZs2aOHnypJDt7t27UvOuV68eXr58KdWvKI6OjlBXV8fly5cxbNgwkfQ6depg+/bt+PHjB/T19cXk8B/q6upwcHBgPgs6l1RUVJgFZwXUr18fR44cgY2NDbNBRNFp1EChoG/WrBk8PDwwY8YMODg44NixY5g4caLYPIH/4uXj44PevXujU6dOEut+69YtrF27Fp07dwaLVTj199u3b1LbTICKigoACNXD3t4eKioquHXrFqytrQEU/jC4f/8+xo8fDwC4c+cOPDw8EBgYyBz3/v17mcstC3r16oVp06Zh2bJlWLVqlUh6amoq02a5ubl4//69yCYlFArl5yX/+S1k7l4oJPzSbJpi8I+++M7WE/Ktis94StnDYbMwpbMxJu9P+Cl0XlEdUVznEUKQmJiIN2/eYNu2bcxyJIrovISEBMa36MtMSdSsWZPReZ6enmJ9atSooZDOo1AoFApFFkhOJtgn1yLz8UXGxlLTxAePQEx/6CpxZldlPufpYi6VSDtXHazqbwkTXWFBZaqrhFX9LdHOVUfCkRVPUFAQfvz4gb59++L+/ft4//49Lly4gMGDB4PH40FfXx+GhobYtm0b3r17hytXrmDixIlCeaxevRr79u3D69ev8fbtWxw6dAhmZmbQ09ODuro6mjRpgiVLluDVq1e4fv065s6dK7Ve2tramDBhAiZOnIjIyEi8f/8ejx49wrp165idZEeNGoXo6GgEBwfjzZs32Lt3LyIiIqTm3aFDB7EitiTU1NQwdepUTJkyBTt37sT79+9x9+5d/PXXXwCAvn37wszMDN27d8etW7cQExODI0eO4M6dOzKXYW1tjaioKMTGxjJTZ6TFJyoqCosWLcKDBw/w6dMnHDt2DMnJyahZsyaAwunI//77L968eYNv376hoKAAK1eulBgvcTg6OmLv3r149eoVoqKi0L9/f7lGMFpbW4PFYuH06dNITk5GZmYmNDU1MXr0aAQHB+P8+fN4+fIlhg8fjuzsbAwdOhRA4RqIDx48wIULF/D27VvMnj0b9+/fl7ncsqB69epYtWoV1qxZw+yM/PHjR9y6dQuBgYGYP38+43v37l1mtCOFQvm54edkImPXAqRvGM908rHUtaA1aC7spqzBrAG1YKwlLPCq4jOeUj60raWFlf1EdZ6JjhJW9qta14CsOm/r1q0SdV5JukGg85YvX87ovFmzZkmtl7a2NiZNmoQJEyaUqPPevXsnt86jUCgUCkUa+a+ikLawH9hFOvmUazaB3uz9qO/TCyHeWjAptgyHsRYLK/tZVOpzno7oq2TaueqgTS1tPIzNRnI6F8Y6hdM4qsobXgEWFha4desWpk6divbt2yMvLw/W1tbo2LEjMx1j9+7dmDhxIlxdXeHs7Iy1a9cKjbITTKmMjo4Gh8NBw4YNcebMGeb4HTt2YOjQoXB3d4ezszMWLVrEbMBQEqGhoTA1NcXixYsRExMDPT091K9fHzNmzAAAWFlZ4ciRI5gwYQLWrVuHRo0aYdGiRRgyZEiJ+fbv3x9TpkzBmzdvmN1RZWH27NlQUlLCnDlz8PXrV5ibm2PUqFEACket/f3335g0aRK6dOkCLpeLWrVqYcOGDTLnP3nyZAQEBKBWrVrIycnBhw8fYGNjw8SnQ4cOIvHR0dHBjRs3sHr1aqSnp8PKygphYWHMGoDDhw/HtWvX0LRpU2RmZuLq1avQ1tbGsmXLhOJ19uxZsNlssesLbt++HSNGjIC7uzuqV6+ORYsWMbveyYKlpSVCQ0Mxbdo0DB48GIMGDUJERASWLFkCPp+PgQMHIiMjAw0aNMCFCxegr68PQgiGDx+Of//9F35+fmCxWMzOe+fOnZO57JCQEERGRuLDhw8yH1OcwMBAODk5ISwsDD169EBOTg5sbGzQuXNnTJo0ifHbt28f+vfvDw0NDanrNFIolKpL/ut7yNw1H/wf/41QUq7VBFoDZoGjX7j2mZeLNlwMs/ElWxvfMnlV9hlPKT/auWqjrct/Os9Im4O61VSgqlI5u1JKQprOI4Rg3759GDduXIk6T5JuAIC//voLQ4cORYMGDeDs7Ixly5bJtIbf/PnzYWJiUqLOEyzjsX79epl1HoVCoVAokiC52cg6tha5N478Z1TVgFav8VBt1p1ZvquFgwq6N7HE40+5hc95LQ6qaWTA3Ey78ioPgEXoL02JpKenQ1dXF2/fvoW9vb3Q9tVxcXHIyMiAnZ2dXNscl4bia69VRB6y+kvzk5Quj11WW1kSHByM9PR0bNmyRa7zlQUai9JTVmX6+/uDEILIyMhyjcW3b9/g7OyMBw8ewNbW9qeORW5uLj58+ABbW1uoqamBz+cjKSkJJiYmpdpRSt58ZPWX5icpXR67rLbyhsaifGNRKPzWIffGf+uVslQ1oFlE+MnbJtKQNR+BbklLS4OOTtUZLVaVKd5mRds6Pz9f6D5XlMp41lbGc7akOpd3PmWhexRJo7FQ3J/GovzzobEQRVK5xbWqNCpD9yiSJovGqQz9Ke18yjOfso4F991jZO6cB/73r/+l29SB/pBQKBtXKzG/8oyFPDqPjuijUEpg5syZ2LhxI/h8foVvTU6pGAghuHbtGq5evVruZcXGxmLjxo0SN0ChUChVm4LoR8jYOQ/8b3GMTdmpAbQGzQbH0KISa0ahUCgUCoVCKRX5ucg6uAJ51w/+Z1NRg0aPsUh3bg6OoVnl1U1OaEcfhVICenp6zNQQyq8Ji8VCbGwsuFxuuZfVoEEDNGjQoNzLoVAoZQvJz0XWiY3IvbofEEyEUFGDZo+xUGvZCyz6IohCoVAoFArlp6Xg/VNwIuYi70c8Y1NyqAftQXPAMrRAelJSJdZOfmhHnwwQQsDn85nPfD6fWVuLEFKh62wVLbei8pDVX5qfpHR57LLaKgIai6oRi7Iqk8ZCPgT3Pj6fz/wVv1cqgrz5yOovzU9Sujx2WW3lDY1F2caC++FZ4fSNpE+MTcm+LjQHzAbHpDoIACImz7y8POTm5lRoLCryOvvVEHcvE/xfktarjPv7z6x5FMmnLJ61iqTRWCjuT2NR/vnQWIgiqW5F7+fSqAzdo0iaLBqnMvRnWZZbkbEg+bnIOb0FuVf2gSW4fpRVofHHaKi28gPr/9NvxeWfnJxcobGQJw/a0SeGDRs2YMOGDeDxeACAtLQ0JCUlCc27zsjIACGF6wFUxEggoPBmJahTadaFkycPWf2l+UlKl8cuq60ioLGoGrEoqzJpLOSHy+WCz+fj+/fvUFZWBp/PR1paGgghpV6TQ558ZPWX5icpXR67rLbyhsaijGLBLQD72h6wbh8DixQKK8JRBr/tIHAbd0UuOICEt7sJCfH4+8I5aGhooHWbdhUWi4yMDIXL+N0orvWSk5ORm5sr1NY8Hg98Pl+s1quM+/vPrHkUyacsnrWKpNFYKO5PY1H++dBYiCKp3OJaVRqVoXsUSZNF41SG/pR2PuWZj8Kx+PIGnOOrwPr+35Is/GrO4HefgHRDS+DbN7HH8Xg83Iu6g4cP76NFy1aoXbtuhcRCHp1HO/rEEBQUhKCgIGaxQ11dXZEFFrlcLjIzM6GkpAQlpYptRlluVGWdh6z+0vwkpctjl9VWEdBYVI1YlFWZNBayo6SkBDabDUNDQ2YzDhaLBWNj41I/2OXJR1Z/aX6S0uWxy2orb2gsSh8L7qdXyNo5D7z4GMbGsXaB1qA54JjZSDyOx+Ph+rUruH796v/fuhIocThlsii1LHWvqM3BfgWKaz1jY2NmMw5BW+fn5yMjI6NErVcZ9/efWfMokk9ZPGsVSaOxUNyfxqL886GxEKV4ucW1qjQqQ/cokiaLxqkM/SntfMozH3ljYaSni7zzfyH34m7g/y9zoaQCXqt+MOg6DBwlZbHHGRsb49u3bzh+7DC+fi3sHExLSxXpKyqvWMij82hHnwywWCyw2WyhwAjeFrBYrAodvVS03IrIQ1Z/aX6S0uWxy2qrCGgsqkYsyqpMGgv5Edz7it4bxd0rFc1bnnxk9ZfmJyldHrustvKGxkKxWBBuAbLP7UDO+XCAXzg6ABwlaHiPgHq7gWBxJMumb9+Scejgfnz58hkAULeuG7p4d0NGRkaFxYJuGqU4ku5lgnuduFEvFX1//5k1jyL5lMWzVpE0GgvF/Wksyj8fGgtRJJUrTqtKozJ0jyJpsmicytCfZVluucUi/j0yt60TepmrZF0LmgNn4ztHCxwlZYl53Lt3FxfOnwWXy4W6uga6de8BY2PTCouFPMfTjj4KhUKhUCi/Pdwv0ciIDAHvy1vGxqnuDG3/EChZOkg8jhCC+/eicPbsKRQUFEBNTR3duvVAnbpuzFIfFAqFQqFQKJTKg3ALkH12OzjnI8ATjOLjKEGjywiotx8IwmJLXJIlPT0dJ08cxadPHwEAjo5O6OnTG1pa2kiqopt00I4+CoVCoVAovzw8PsGTLwXgxqfDRFcZ7jYa4LBZIDwucs6HI/vcDoD3/3XY2ByodxoCjU5DShzFBxRO142KuoOCggLY2TnAp5cv9PT0yv+EKBQKhUKhUCgMPD7Bg9gsJKdzYayjxGg97pe3/3+ZGw3BmM/iL3PFbawmID7+Kz59+gglJSV07NQFTZp4gMViVelN0GhHH4VCoVAolF+ai8/TsfhUIhLTuQAyAQCmukqY1zgDLrfDwPsSzfhyLOwLhZ9VjRLzFEwVUlJSQm/fPnj/LhpNPZrT6bMUCoVCoVAoFczNd/nYFB7zf61XiIUOsMLsCiwe7GKWZCFsDjQ6DoFG55Jf5hadEu7sXAPNmrVAffeGMDMzK98TKSOoGqVUaTw9PbF3797KrgauXbsGFouF1NTUMs1XRUUFx48fL9M8FaV169aYNGmSzP47d+6Evr5+mdYhNjYWLBYLT548qdR6VBRNmzbF0aNHK7saFMovzcXn6ZiwJ05I+CkRLjonHITDwcD/OvnYHKh3HAy9aTtL7OTLy8vDsaOHcfXKJcZmZmaOZs1b0k4+ym9Hq1atMH78eJn9f7ZnNovFKnOdJm+bUeQjIiJC7lHVAQEB6N69e7nUh0KhlD+XXmQg5EyWkNaz5X1BSNwcWNyLYDr5OBYO4A0Lg3qXYSV28sV++ICNG9YiLS2NsdV3bwgTE5NyO4eyhipSSpXl1KlTSExMRJ8+fSq7KvDw8EB8fDx0dXUruyqUn4hDhw7B1dUV6urqqF27Ns6ePSuUPnPmTMyaNatKD/umUH5meHyCxacTQYrYnLgfsDVjLgLyjkMJ/wk/vSnh0OwWCJayisT8Pn36iPXrVuPBg3u4du2KkACkUCi/HvHx8ejUqVOllX/r1i0oKSnBzc1NyB4SEiK0UQyLxUKNGsIvKHJzcxEUFARDQ0NoaWnBx8cHiYmJUst89+4dhgwZAisrK6iqqsLS0hJeXl7Yu3cvuNz/fkQXLVtJSQlWVlaYOHEi8vLyRPKMjIxEixYtFGuE3xhCCObMmQNzc3Ooq6vDy8sL0dHR0g+kUH4jeHyCJaf/WyePQ3jol3sKWzPmwJkXW+gDNtQ6DobOlHDAXPK6yzweD3//fR7bt2/G169xuHTxQnlXv9ygHX2UKsuGDRsQEBBQJUZIqKiowMzMrEJ3kKL83Ny+fRv9+vXD4MGD8ejRI3Tv3h3du3fH8+fPGZ9OnTohIyMD586dq8SaUii/Lo9ic5CYVvjDVJkUYFjOIWzKDIU9v3BnXC44iFDtjug+m6BkXVNiPjweD5cuXsDWLRvx48d36OnpYfDgYfTlD4Xyi2NmZgZVVdVKKTs1NRWDBg1C27Ztxaa7uLggPj6e+fvnn3+E0idMmIBTp07h0KFDuH79Or5+/YqePXuWWOa9e/dQv359vHr1Chs2bMDz589x7do1DB06FFu3bsWLFy+E/MPDwxEfH48PHz5g48aN2L17NxYtWiSS74kTJ9C1a1c5W4CybNkyrF27Fps3b0ZUVBQ0NTXRsWNH5ObmVnbVKJQqw8PYbGYknzUvDusyF2Bk7iGooNAWy7ZAoNYcvKztX+LL3KSkRBw6uA83b1wDIQT13Rugi/cfFXIO5UHl96D8guTn50v8KygoKHNfedi5cyfMzMxE3rZ1794dAwcOVOyEZShTW1tb6A1UYGAgatSogezsbLHHJCcn4+rVqyKiQEVFBdu3b0ePHj2goaEBJycnnDp1Ssjn+vXraNSoEVRVVWFubo5p06YJvYFs1aoVxo4di/Hjx0NfXx+mpqbYtm0bsrKyMHjwYGhra8PBwUGo86X41N2IiAgYGxvjwoULqFmzJrS0tNCxY0fEx8czx9y/fx/t2rWDkZERdHV14enpiUePHsnVdkXramBggGrVqkmtKwDcuHEDjRs3ltgGWVlZ8Pf3h5aWFszNzbFixQqRsvPy8jB58mRUq1YNenp6aNKkCa5duyZX/Ytz/vx5NG/eHHp6ejA0NIS3tzfev38v0V/Q7mfOnEGdOnWgpqaGJk2aCHWWCSjvWMjLmjVr0LFjR0yaNAk1a9bE/PnzUb9+faxfv57x4XA46NixIw4cOFCudaFQfleSMwrvezW477EtYw4G5p2CEgpH0EZzrDBSKwTh6j2RnCX5JU5ychK2bNmIq1cvgxACN7f6GPvnBNja2VfIOVCqJhWp9eRFEZ0DSNdPWVlZGDRokEzawdLSEpqammjcuHGptUPr1q3x559/YsqUKTAwMICZmRlCQkKEfD59+oTu3btDS0sLOjo68PX1FRq9FhISAjc3N+zYsQNWVlbQ0tJCYGAgeDweli1bBjMzM5iYmGDhwoVC+RaduhsbGws2m42jR4+iTZs20NXVhZubG+7cucP4f//+HX379oWlpSU0NDRQu3Zt7Nu3T6HzHjVqFPr164emTZuKTVdSUoKZmRnzZ2RkxKSlpaXhr7/+wsqVK9GmTRu4u7sjPDwct2/fxt27d8XmRwhBQEAAnJyccOvWLXTt2hWOjo5wdHRE3759ce3aNdSpU0foGD09PZiZmaF69erw9vbGH3/8IbL8Sm5uLv7++2/88UfhD+aNGzfC0dERampqMDU1Ra9eveRql5UrV6JevXrQ0tJC9erVERgYiMzMTIn+gthv2bIF1atXh4aGBnx9fcWOyA4LC4O5uTkMDQ0RFBQk9P3ctWsXGjRoAG1tbZiZmaF///7lutsmIQSrV6/GrFmz0K1bN9SpUwc7d+7E169fceLEiXIrl0L52UhO50KJcDEo9zj+ypgFF17hb0seWNir2gXDtefhtZIdkotM6y0Kn8/Hndu3sGnjOiQnJ0FDQwP9+g2Ej48v1NTUKvJUyhS6GUc5EBoyS2Kak3MN+PsPYT4vWhgqIvIE2NraYdjwUcznsOVLkJ2dJeK3cNEymevWu3dvjBs3DidPnoSvry8AICkpCWfOnMHff/8t8ThXV1d8/PhRYnqLFi1w8uRJsWmDBg3C6dOn4e/vj9u3b+Pvv//G9u3bcefOHWhoaAiJSAH//PMPNDQ0ULOm6AiLefPmYdmyZVi+fDnWrl0Lf39/eHp6wtDQEHFxcejcuTMCAgKwc+dOvH79GsOHD4eampqQKIyMjMSUKVNw7949HDhwAKNHj8axY8fQo0cPzJgxA6tWrcLAgQPx6dMnaGhoiD2v7OxsrFixArt27QKbzcaAAQMwefJk7NmzBwCQkZEBf39/rFu3DoQQrFixAp07d0Z0dDS0tbUltmVxBHWNiorCvn37EBgYiOPHj0usa1xcHP744w/4+/tLbINp06bh+vXrOHHiBExMTDBjxgw8evRISMCNGTMGL1++xL59+2BiYoJTp06hY8eOePbsGRwcJA95LomsrCxMnDgRderUQWZmJubMmYMePXrgyZMnJY7cDA4Oxpo1a2BmZoYZM2aga9euePv2LZSUCm9h2dnZCAsLK9NY7NmzByNHjizxfM6ePYuWLVuKTbtz5w4mTJggZOvQoYPIWj8NGzbE8uXLSyyHQqEohok6HyNzDsAv7yw4/5/AWwAOdqp1wx5Vb/BYhfcQYx3xcig/Px9bt2xEdnY21NXV0a1bT9SuU7fC6k+puswLnS0xzcm5BgYNGsx8Lq3WW7Bwqdz1k1fnlKSf5s6dC6DwWSxOOxSdVirQDvv374eFhQWOHTtWau0gOJ+JEyciKioKd+7cQUBAAJo1a4Z27dqBz+fDx8cH2trauH79OrhcLoKCguDn54erV68yebx//x7nzp3D+fPn8f79e/Tq1QsxMTFwcnLC9evXcfv2bQwZMgRt27aFu7u7xLrMnDkTy5cvh62tLUJCQtC3b1+8e/cOSkpKyM3Nhbu7O6ZOnQodHR2cOXMGAwcOhL29PRo1aiTz+YaHhyMmJga7d+/GggULxPpER0fDwsICampqaNq0KRYvXgwrKysAwMOHD1FQUAAvLy/Gv0aNGrCyssKdO3fQuHFjkfyePHmCV69eYd++fRI1WUkzW96+fYurV6+KDBy4fPkyLC0tUaNGDURFRWHcuHHYtWsXPDw88OPHD9y8eVNqexSFzWZj1apVcHBwwIcPHxAYGIjp06dj06ZNEo959+4dDh48iFOnTiE9PR1Dhw5FYGAgoxMB4OrVqzA3N8fVq1fx7t07+Pn5wc3NDcOHDwcAFBQUYP78+XB2dkZSUhImTpyIYcOGiSzLUpRRo0Zh9+7dJZ5PRkaGWPuHDx+QkJAgFENdXV00btwYUVFR6N+/f4n5Uii/C9Wy3mFrxkJmtgYAfGKbYYnGcLxQcmRskrTevai7OH26sPPcytoGffr0g66uXrnWuSKgHX2/Gerq6ujTpw8iIiKYjr7du3fDysoKrVq1knjcmTNnxHbICZDW271582bUrVsXf/75J44dO4aQkBC4u7uDECLW/+PHjzA1NRUrNPz9/dG3b18AwKJFi7Bu3Trcu3cPnTp1wsaNG1G9enWsX7+eWa/k69evmDp1KubMmcPkV7duXcyaVdghO336dCxZsgRGRkbMw3zOnDnYtGkT/v33XzRp0kRsHQsKCrBp0yZGuI4ZMwbz5s1j0tu0aSPkv3XrVujp6eH69evw9vYusb2KIqgrIQRTp07F8uXLS6zrxo0bUa1aNaxfvx5sNlukDbKyshAeHo5du3Yx00EiIyNRrVo1psxPnz4hPDwcnz59grm5ObhcLiZPnowLFy4gPDxc5I23rPj4+Ah93rFjB4yNjfHy5Uu4urpKPG7u3Llo166dUF2PHTuG3r17AyiMxebNm2FvXzjCpixi8ccff4gVwkDhW1Yulwtra2uJdU5ISICpqamQzdTUFAkJCUI2c3NzfP78GXw+v0pMU6dQfhk+v4bDmfWwy/vvJdUbjg2WaAxHDKc6AICFwt133W3Ev9BRUVFBm7bt8OrVS/j4+NKpupSfBnl1Tkn6afbs2cjMzMSOHTuwe/dumbSDhYUFAGDy5Mk4f/58qbQDANSpU4fpcHR0dMT69etx+fJltGvXDpcvX8bz588RExPDdHTt3LkTLi4uuH//PurVqwegcNTGjh07oK2tjVq1aqF169Z48+YNzp49Cw6HA2dnZyxduhRXr14tsaNv8uTJ6NKlC7hcLkJCQuDq6op3796hRo0asLS0xOTJkxnfsWPH4sKFCzh48KDMHX3R0dGYNm0abt68ybzQLE7jxo0REREBZ2dnxMfHIzQ0FC1atMDz58+hra2NhIQEqKioiGxKIU6HCHj79i0AwNnZmbElJSXBzs6O+bx06VIEBQUxn/v27QsOhwMul4u8vDx4e3tj6tSpQvmeOHGCGc336dMnaGpqwtvbG9ra2rC2tka9evUk/h4Qx/jx48HlcqGkpARbW1vMnz8fo0ePLrGjLzc3Fzt37oSlpSUAYN26dejSpQtWrFjB7KCpr6+P9evXg8PhoEaNGujSpQsuX77MfGeGDPlvoIadnR3WrFmDRo0aITMzU+JL43nz5gldD/IgiJMsWpJC+R0h+bnIPr0Vlpf2AKRwtgYXbOxX7YxIte7IZxVO0xXWeqL3mvruDfDw4X3Ud28AGxs7aGvrVOBZlB+0o68cmBsi/s0bIPombMbMuTL7Tg6exjzYSrNW3JAhQ+Dh4YG4uDhYWloiIiICAQEBJeZpbW1dYrqg40MS+vr62LJlC7p06QIPDw9MmzatxDrm5ORIXBOl6KgzTU1N6OjoMEPnX716haZNmwrVtVmzZsjMzMSXL18YAVg0Dw6HA0NDQ9SuXZuxCR6qJQ3J19DQYDqWgMIOm6L+iYmJmDVrFq5du4akpCTweDxkZ2fj06dPJZ57SecrS11fv36NJk2aSGyDHz9+ID8/X6gTy8DAQEjYPXv2DDweD05OTkJ1ycvLg6GhoVz1L0p0dDTmzJmDqKgofPv2jdmE4tOnTyV29BWdtiKo66tXrxhbecRCW1tbonATXO+SBLg8qKurg8/nIy8vD+rq6qXOj0L53SH5ucg+uQmcy/vA/7+gy4cSItW6Y59qZ2YUn+AOOc3bFBz2f/fLly+fQ0tLG1ZWhR35TZp4oHHjprQjniLEnLnzJWqykrRe0eeHYCODokwOnsbYpGmrkpBXO0jTT8nJyZWmHYqfDyD8nH/16hWqV6+O6tWrM+m1atWCnp4eXr16xXT02djYCD3XBS+Ui363TU1NpU7HLFoXc3NzAIXtWKNGDfB4PCxatAgHDx5EXFwc8vPzkZeXJ3F2SHF4PB769euH0NBQkXYsSqdOnZhY1alTB40bN4a1tTUOHjyIoUOHylSWLBgaGuLJkycghKB169YiU8lXrVoFLy8v8Hg8vHv3DhMnTkRAQACzJAkhBKdOncLBgwcBAF5eXrC2toadnR06duyIjh07okePHnLpn0uXLmHx4sV48+YN0tPTweVykZubi+zsbGhqaoo9xsrKiunkAwp1JZ/Px5s3b5iOPhcXF3A4HMbH3Nwcz549Yz4/fPgQISEhePr0KVJSUoQ0rIuLi9hyTUxMpO7SKU8nJ4VCKST/7UNk7l4IfvJ/o/iiOVZYpj4Mb5VsGFtxrcfnE+Tl5eH69avw9GwNNpsNFRUVjA4cC6Dk3/4/G7SjrxxQUZG8yGNpfdlsdqk7+urVq4e6deti586daN++PV68eIEzZ86UeExppu4KuHnzJjgcDuLj45GVlVXi9FUjIyNmPbziKCsrC31msVhy71oqLo+iNkH7lpSvuDyKPqz9/f3x/ft3rFmzBtbW1lBVVUXTpk3lXm+nLOoqL5mZmeBwOHj48CHYbLbQjxItLS2F8+3atSusra2xbds2WFhYgM/nw9XVVaE1iIpSHrEo7dRdMzMzkd3tEhMTGUEp4MePH9DU1KSdfBRKGVDw7gkyds0HP+kTI+6UbFzwqslEXLytDl6R9VlMdZUwzdsU7VwL39zm5eXizOlTePjwPvT1DTD2z/FQVVUT2xlDoUjTZEWfQUW1HiGkxONUVFSEOvoU7WCubO1QtMMEQKm0A1B+2q/4CztZ8i2pHZcvX441a9Zg9erVqF27NjQ1NTF+/HiZdU5GRgYePHiAx48fY8yYMUzehBAoKSnhwoULYnWHnp4enJyc8O7dOwCFGiQ/Px+pqalCo/rE6RABjo6FU9zevHnDdI5yOBw4ODgw5RfHzMyMmdni7OyM9PR09OvXDwsXLoSjoyPu3bsHLpcLDw8PAIUvUR8+fIjr16/j77//xpw5cxASEoJ79+7JdI3Exsaia9euGDlyJBYuXAhDQ0PcvHkTw4YNQ35+vsSOPlko6RrLyspChw4d0KFDB+zZswfGxsb4+PEjOnbsWGJsSzN1VxCnxMREpkNZ8Ll4xzeF8rvAz8lE9rF1yL159D+jkgrUOw9FnGYrpN3iASVovdgPMTh4cD8yMtKhxOGgRctWAAqXBCjL52FVgHb0/aYMHToUa9asQVxcHLy8vITegoqjtFN3b9++jbCwMJw8eRLTpk3DmDFjEBkZKdG/Xr16SEhIQEpKCgwMDEo+mSLUrFkTR44cASGEEV+3bt2Ctra20PSSiuDWrVvYuHEjOnfuDAD4/Pkzvn37Vu7l1qhRo8Q20NfXh7KyMqKiopippykpKXj79i1atGgBoLD9eTwekpKS0Lx5c5FRC4q8ffz+/TvevHmDbdu2MeUU3yFOEnfv3mVGYwrqKm79RkkoEovSTt1t2rQprly5wgh1ALh48aLIotovXrxgBDWFQlGQ/FxkHV6JvGsHgf/fnwhHGRpdR0LDqz9acJRwoRkPl598BZejDRNdZbjbaDAj+T5+jMXhQ/vx48cPsFgsuNauAw6HSiTK74M0/aSjo8Noh+LPY09PTwDC2kHwnC9KeY1cqlmzJj5//ozPnz8zdXv58iVSU1NRq1atcilTErdu3UK3bt0wYMAAAIWddG/fvpW5Hjo6OkKjyIDCzSuuXLmCw4cPw8bGRuxxmZmZeP/+PbM+nru7O5SVlXH58mVm2ZQ3b97g06dPEjf3qFevHmrUqIGwsDD4+voq1Mks6ODNyckBUDhtt0uXLuBwOEz8lZSU4OXlBS8vL8ydOxd6enq4cuUKM723JB4+fAg+n49ly5YxneKybGj26dMnfP36lZlSfvfuXbDZbKERqSXx+vVrfP/+HUuWLGF+M92/f1/qcaWZumtrawszMzNcvnyZWQczPT0dUVFRzHRiCuV3Iv/ZP8jcuxj81P9G3SnZ1YHWwNlgm1iheVISunkY4/GnXCSnc2Gso8RoPS6Xi0uX/sY/N6+DEAJ9fQNYWdtU3slUAFTF/qb069cPwcHB2LZtG3bu3CnVvzRTdzMyMjBo0CCMGTMGnTp1QvXq1dGwYUN07dpVZM02AfXq1YORkRGz65esBAYGYvXq1Rg7dizGjBmDN2/eYO7cuZg4cWKFT7tydHRkduhKT09HcHBwhYzaCgwMxJo1azB27FiMHTtWpA20tLQwePBgTJkyBUZGRjAxMcHMmTOF2sfJyQn9+/fHoEGDEBYWhtq1ayMlJQVXrlxBnTp1mA4zedDX14ehoSG2bt0Kc3NzfPr0SeoUbgHz5s2DoaEhTE1NMXPmTBgZGaF79+4yl61ILEo7dXfcuHHw9PTEqlWr0LVrVxw4cAAPHjzA1q1bhfxu3brFrD9IoVDkpyD6ETiR85CX8t9O2xxbV+R1DoR6LXew/n9v47BZcKumDBMTHeZ+x+PxcOXyRVy/fhWEEOjp6aNXbz/Y2tqJLYtC+VWRpp+0tLQwZMgQBAcHw9DQUKp2WLFiBerVq4fk5GRcvnxZYe0gC15eXnB1dcWAAQOwevVqcLlcBAYGwtPTEw0aNFB4+rMiODo64vDhw7h9+zb09fWxcuVKJCYmytzRx2azRZYyMTExgZqaGlxdXRn9MXnyZPzxxx+wtrbG169fMXfuXHA4HGYNa11dXQwdOhQTJ06EgYEBdHR0MHbsWDRt2hRNmjQR2+nKYrEQHh6Odu3aoVmzZpg+fTpq1qyJgoICXL9+HcnJySIjNVNTU5GQkAA+n4/o6GjMnz8fjo6OzMvYkydPCq2ZfObMGXz8+BGenp7Q19fH2bNnwefzZe5wc3BwQEFBATZs2IBu3brh9u3b2LJli9Tj1NTU4O/vj7CwMKSnp+PPP/+Er6+vxNGNxbGysoKKigrWrVuHUaNG4fnz5xI3SSlKaabuslgsjB8/HgsWLICjoyNsbW0xe/ZsWFhYoFu3bjLVm0L5FeBnpCDr0Ark3b/wn1FVHZrdx0CtZS+wiozG47BZaGQnPLI3MTEBBw/uQ0J8oU6sVcsFPX18f/nZVL/8gjOpqalo0KAB3Nzc4Orqim3btlV2laoEurq68PHxgZaWllwdJoowbtw4aGpqYv78+QCA2rVrY9GiRRg5ciTi4uLEHsPhcDBo0CDs3btXrrIsLS1x9uxZ3Lt3D3Xr1sWoUaMwdOhQZkHqiuSvv/5CSkoK6tevj4EDB+LPP/+U+rAvCywtLXHy5Encv39fYhssWbIELVq0QNeuXeHl5YXmzZuLLDwdHh6OQYMGYfLkyXB1dUWPHj1w//595m15cWJjY6GiooJr166JTWez2di/fz8ePnwIV1dXTJgwQebdZpcsWYJx48bB3d0dCQkJOHXqlFzT3isjFh4eHtizZw+2b98ONzc3HD58GMePHxcS8HFxcbhz5w4GDx5cQk4UCkUc/JxMZO5ZhIzVo8ESdPIpq0LTZzx0Jm4FjEoeqZ6Tk43Nm9fj2rUrIITArV59jP1zPO3k+8mgOq9skEU/LV++XGbtMGnSJDg7O6N79+5StQOLxZKoHWSBxWLhyJEj0NfXR8uWLeHl5QU7OzuZRnqVNbNmzUL9+vXRoUMHtGrVCmZmZuWis+Pi4tC3b184OzvD19cXhoaGuHv3LoyNjRmfVatWwdvbGz4+PmjZsiXMzMxw9OjREnIFmjRpgocPH8LZ2RlBQUGoVasWPDw8sH//foSFhWH06NFC/oMHD4a5uTmqVauGvn37wsXFBadOnYKSkhLev3+Pd+/eoUOHDoy/np4ejh07hjZt2qBmzZrYvHkz9u3bx6xxFxERUeLAgrp162LFihXMS+g9e/Zg0aJFUtvLwcEBPXv2ROfOndG+fXvUqVMHGzdulHqcAGNjY0RERODQoUOoVasWlixZIrOGLQ1TpkzB2LFjMWLECDRs2BCZmZk4d+6c1JlUFMovASHIe3ABKfN8hTr5lGs1gf7sA1Bv5cu8zJXE06ePsXHDWiTEx0NDQxN9+w1EW68OEvcC+JVgkV98BVAej8csgpuVlQVXV1c8ePBApkWB09PToauri7dv38Le3p55a8nn8xEXF4eMjAzY2dlV2M22+ALOpc3Dy8sLLi4uWLt2banLlOYnKb0k+5cvX+Dm5oZHjx7B2tparG9ZtIkilHUsZMmjMmMhS7tfuXIFPj4+eP/+vVzTrUvi2rVraN26NVJSUkR2jpPlXGWlsmIxZcoU/PjxA9u2bSvTWFTE9yI3NxcfPnyAra0t1NTUwOfzkZSUBBMTk1KNnpU3H1n9pflJSpfHLqutvPkdYlHw7Cay9i0FPy2ZSVeyqwPtQXPAMbWWKRaEEOzeFYGPH2PRrbsPatcu+zWPKjoWAt2SlpYGHZ1fY9c4aZRG5wGibVa0rfPz84Xuc0WpjGftz6x5JOVz9epV9OzZEzExMdDX11eo3JL8FEn7XWNRFv5F/VatWoVLly7h7NmzUvMQpM2fPx83btxgOn7LIhYhISE4fvw4njx5ImPryMbPFIvf9XtRXKtKozJ0jyJpsmicytCf0s5HHrjf4/Fj53yw3/43RZ6loQPNXhOg2qSLyPUlqdz4+K/YtHEd7O0d0NPHF5qamj91LOTReb/81F0Oh8PsdJWXlwdCyG+/u1FKSgr++ecfXLt2Ta63WRWNmZkZtm/fjk+fPpW4FhqlanD27FlMnTpVRKhTJGNiYoI///yzsqtBofw8ZKYg89Qa5D+69J9NVQO8NgOh3zkAHCk7YWdlZSI3VwcaGhpgsVjo2bM3uDwedHV1y7nilPKC6ryfm7Nnz2LGjBlUO/yCVKtWDdOnT5frmPPnz2P9+vXlVCMKhVLVIXw+8m4dR9bRtWDnZjF2lfptoeU7GWxdI6l5JCcnwdi4cOaWubkFRgeOhZmZuUKbOP3MVPmpuzdu3EDXrl1hYWEBFouF48ePi/hs2LABNjY2UFNTQ+PGjXHv3j2h9NTUVNStWxfVqlVDcHAwjIykXyC/Mo0aNcLgwYOxdOlSmdfEqCy6d+8udkFnStVj+fLlmDRpUmVX46di0qRJMDU1rexqUChVHkII8qLOgrMxSKiTT9nFA7qz9oE08pY6fePly+fYu2cnTp86wdg0tbRoJ18lQ3Xe783y5csRHBxc2dWglAO+vr5ya/ioqCg0atSonGpEoVCqMrykz0hfE4jMvYtB/t/Jx9IxhPaIpdAZvkRqJ19eXh4OHzqAtWtW4suXz4zd3NyiQkeYVhWq/Ii+rKws1K1bF0OGDEHPnj1F0g8cOICJEydi8+bNaNy4MVavXo0OHTrgzZs3zBpcenp6ePr0KRITE9GzZ0/06tVL7I/rvLw85OXlMZ/T09MBFP7AKNr7K9jmXpBWkW+Oi5arKG/fvmW2kJclH1nLlOYnKV0eu6y2iqAsypU3j98tFp6ensx3T95zkRcaC/kQ3Pv4fD7zV/xeqQjy5iOrvzQ/Seny2GW1lTe/Wix4378ie99SFLy6C4FMY2nqQqPXBKg07Fh4LSYnS2z3vLw8nD1zEo8ePQQAJCUlIicnp0LWZ6noWPyMb6orUucBkrWeuHuZ4P+StF5l3N9/Zs2jSD5l8axVJI3GQnH/qhaLuXPnYu7cueUSJxqLqv29KK5VpVEZukeRNFn0ZmXoT0XLJTwucq/uR87prUDBf89nvltb6PWdAiUtPan5xbx/h8OHDyAjIwMsFgufP3+ChYWlXPX7GWIhTx5VvqOvU6dO6NSpk8T0lStXYvjw4cxi9ps3b8aZM2ewY8cOkR09TU1NUbduXdy8eRO9evUSyWvx4sUIDQ0VsaelpSEpKUlo3nVGRgYIKVwPoKJ28yKEgMfjAUCp1oWTJw9Z/aX5SUqXxy6rrSKgsagasSirMmks5IfL5YLP5+P79+9QVlYGn89HWloaCCGlXotMnnxk9ZfmJyldHrustvLml4kFjwvOw3NgX94FVkEuk85zaQnSaTjSNPWA/3fwSWr3r1/jcOniBaSnpwEAXFxqo0XLVkhLS5OzNRSjomORkZGhcBmVRUXqPECy1ktOTkZubq5QW/N4PPD5fLFarzLu7z+z5lEkn7J41iqSRmOhuD+NRfnnQ2MhiqRyi2tVaVSG7lEkTRa9WRn6U9r5iCUxFpyTa8H6Gs2YiJ4JuF2CkGpoC25mLtjZSRIP53G5uHv3Nh49egAA0NHVRfv2nWBuboGkJNHjfvZYyKPzqnxHX0nk5+fj4cOHQus/sNlseHl54c6dOwCAxMREaGhoQFtbG2lpabhx44bIjlECpk+fjokTJzKf09PTUb16dejq6oossMjlcpGZmQklJSUoSVkTqKyR5UZV1nnI6i/NT1K6PHZZbRUBjUXViEVZlUljITtKSkpgs9kwNDRkNuNgsVgwNjYudYeGPPnI6i/NT1K6PHZZbeXNLxGLb5+htmcZeLHPGTtLzxgFHUfBqFlnqbHIz88vFH4P74MQAj09PfTs2Rsamlq/dCx+tZ0Yy1rnAZK1nrGxMbMZh6Ct8/PzkZGRUaLWq4z7+8+seRTJpyyetYqk0Vgo7k9jUf750FiIUrzc4lpVGpWhexRJk0VvVob+lHY+RSEF+ci5EIHcCxEAv7CTFiwWVFv5QsN7FIiKGpCcXGI+iYkJOHr0EBIS4gEAtWq5onsPH6irqytUv58hFvLovJ+6o+/bt2/g8Xgi0zNMTU3x+vVrAMDHjx8xYsQIZuju2LFjUbt2bbH5qaqq/hZbLVMoFAqFUpnw+AT3P2TjeyYfxtpKqG+jDg6bBcItQO6FCCidjwCP/98IKtUWPlDrOhrfMrNlyj8/Px+vXr0AIQT16tVH5y5/QEVFBcnJydIPplQZylrnAVTrUSgUCoVS3vD4BA9js5GcwRXSeQDA/fAcWXsWghcfw/izzWyg2X8WlO0Kn99EhimqMTHvkZAQDw0NTXTr1gOGRsb0+V6En7qjTxYaNWok93bqGzZswIYNG5ghwHTq7u89RVEcNBZVIxY/87SJktJ/hljQqbt06q6i/tff5mHD9Wx8z/5v+qyxFgvTXOLg/ngTWEkfmbX4iKEleF3HgGvtioyMrBJjwefzwWazmV3VmjZpBmVlZTg6OSM9Pf23iMXPOHW3tCii8wBRrUen7krmZ37W/i5TFMs7HxoLUWgsqn4s6NTdypm6e+HfNEQ8TMO3zP/WSzTWYmFsMzZafDwI1t2TYOH/ayqyOSDNeoHb0g/5SsrA/6fbSqo/IYSJsZ2dAxo1aoLadepCTU0dqampv3wsfpupu0ZGRuBwOEhMTBSyJyYmwszMTOF8g4KCEBQUhPT0dOjq6tKpu3L4/4pTFEuCxqJqxOJnnjZRUnpVjgWdukun7irif+lFBuafS0HRJbrVSS58k4+g3oW//xN+LDbUvAZAo8swsJRVS8yfz+cjKysLF86fhVu9+nBzq/fbxuJXm7pbXjoPENV6dOqudH7mZ+2vPkWxovKhsRCFxqLqxoJO3a34qbt/P0tH2PU0AMKbsVilPIfjoR1g8/+bWcGxqgHN/jOhVM1JJB9x9f/336e4fesmBg8Zzozc6/pHd4n+4vjZY/HbTN1VUVGBu7s7Ll++jO7duwMobNjLly9jzJgxZVYOi8UCm80WCoygJ5nFYlXo6KWi5VZEHrL6S/OTlC6PXVZbRUBjUTViUVZl0ljIj+DeV/TeKO5eqWje8uQjq780P0np8thltZU3VTEWPD7BkjNJQtLPo+AxxmdHwpT8YGzsak7I7xwIzbpNZWrz58+f4cTxI8jNzUViYjxq164DDofzW8aiIs+rIqgonQdA4r1McK8TN+qlou/vP7PmUSSfsnjWKpJGY6G4P41F+edDYyGKpHLFaVVpVIYGVSRNFo1T0ZqHxydYdlZ4iRQtfhZG5+6Hd/71/4zKqtDwHgn1tn3B4kjujhLUPy8vD6dOHsPTp08AAFFRd9CqVRuJ/r9yLOQ5vsp39GVmZuLdu3fM5w8fPuDJkycwMDCAlZUVJk6cCH9/fzRo0ACNGjXC6tWrkZWVxezOVhYQIn7LZEGa4P8VQdFyFYXP52PEiBE4cuQIUlJS8OjRI0yYMAF169bF6tWrFS5Tmp+kdHnsstoqgrIoV948CCFo3bq1xFjJmu+vFAtFykxOTkbLli0RHR2NAwcOwMfHR6FYyOL/q8ZCcO/j8/nMX/F7pSLIm4+s/tL8JKXLY5fVVt5U1Vg8iMlGYlrh9EdDfgr+zNmNVgX3mfQ8KCNcrQfa9hoGG51cqW2em5uLs2dO4vHjRwAAc3ML9OrdBxwO57eNRUWeW1lRFXQeALH3MsH/JWm9yri/VzXNI4smKUpkZCQmT56MlJSUUpUrj9+8efNw6tQpPH78GAAwePBgpKam4tixYxJ1VVWMRUhICE6cOIF79+6VWO7s2bORmJiIrVu3lphfZegeRdIqOxaCdhdcP7LUWd7vBY2FfHz79g0uLi54+PAhqlWrVmLdit7PpVEZGlSRNFk0TmVongcx2UhM/2+Zi+b5DzEhJxJGJJWxPeE4w2DATLg1cAYBJK7FJ6j/+3fROHbsMNLS0sBms9HSszWaNWshs16XlO/PGgt58qjyHX0PHjxA69atmc+CndL8/f0REREBPz8/JCcnY86cOUhISICbmxvOnz8vsnCzPPzqa/SdO3cOkZGRuHTpEmxtbWFkZIQDBw5AWVmZORdHR0eMHTsWY8eOrfJrkYWGhmLRokUiZWpoaCA1NVXEfuDAAQwcOBBdu3bFkSNHJJ7TsWPHsHXrVjx9+hR5eXmoVasWZs+ejfbt25dZLBRZk0Pw0JJ03ZVHLL58+YIZM2bg4cOHeP/+PYKCgrBs2TIh3+LHf//+Hf7+/nj27Bm+f/8OExMTdO3aFfPnz4eOjg4A4NatW5gxYwbevHmD7OxsWFlZYfjw4Rg3bpzU9ihappOTEz5+/AgAUFdXh52dHcaOHYshQ4aIHJORkYFOnTrB0NAQffr0wYABA6ClpQVPT0+h81m6dCmOHz+ON2/eQF1dHU2aNMGiRYvg7Owsc+zkjcWLFy8QGhqKx48f4+PHj1i+fLlQW0j6Djx58gQTJ07EgwcPYGxsjNGjR2PChAlCfocPH0ZISAg+fvwIBwcHLFq0CJ06dRLKOzQ0FDt27EBqaio8PDywbt06ODo6ij03ukYfXaNPXv/3cflgEz7+yL+CETkHoYlcJu2+kitWqvvjK8cUNvFp0OPnlNjm8fFfcenieaSnp4PFYqGWS220aFH4HU5KSvptY/EzrtFXGToPoGv0yUNJ5UrTJMV9BT9SZPUvi7XIBD+yBGWGhYWBEIKCggKxuqosYzFv3jycPHkSDx48kHq+0hCcR0nlJiQkYO3atXj06BFzPlu2bMGWLVsYnVSrVi3MmDED7dq1Y/IJDAzElStX8PXrV2hpaTGap0aNGsz5X7p0CfPnz8fz58+hqamJAQMGYP78+ULT2sXV7/Hjx1i+fDn++ecf/PjxA2ZmZnB1dcWwYcPQpUsXsFgsfPjwAc7Ozkw+ysrKsLKywsCBAxEcHCyUHyEE8+bNQ0xMDCIjI8v9e1H8+inpXIumyfO9qIg1+mJjY+HkJDpFEgD27t2LXr16CR33/ft3NGzYEHFxcUhKSoKurq7Y70VMTAyWLl2Ka9euISEhAebm5ujXrx+mT58OFRUVqeejCHp6ehgwYADmzJmDrVu3SmwTukZfxWqe93H5AABzXhL+zNkND+4TJi0Latis3genVFphRo4mLP6/Fp8kCgrycf36Vbx6+QIAoKuri/btO8HM3ALfv38X8f9dYvFLrdHXqlUrqW8ExowZU6ZTOH71NfpiY2Nhbm6OFi1aMDZx873ZbDZTVlVei2zSpEki8ffy8kLDhg1FYhMbG4tp06ahRYsWYLPZJcbu1q1baNeuHRYtWgQ9PT2Eh4ejR48euHv3LurVq1diveVB3jwEw9ClXXdlGQsulwsTExPMmjULq1evBpvNBofDKTE+Kioq6NatGxYsWABjY2O8e/cOY8aMQWpqKvbs2QMA0NHRwZgxY1CnTh1oamrin3/+wahRo6CtrY0RI0aUWP/ihIaGYvjw4cjOzsahQ4cwatQoVK9eXagzKy8vD7169YKBgQGOHTsGTU1NmJiYwM/PD+fPn0eTJk0Y33/++QdBQUFo2LAhuFwuZs6ciS5duuDFixfQ1NQssQ0ltYm09Pz8fNjb28PX1xcTJ04Eh8MRG+ei+aWnp+OPP/6Al5cXNm/ejGfPnmHo0KHQ09PD6NGjAQC3b9/GwIEDsWjRInh7ezOC7uHDh3B1dQVQ2LG5YcMGREREwNbWFnPmzIG3tzdevHgh9v5A1+ija/TJ61/zwzOsz1wMF957xpbC0sZ69f64pNwU+L84t7c0gJ5WlsQ2V1Li4Pixw+Dz+dDT00dPn97Q0NCkscDPuUZfZeg8gK7RpwjiypVVkwgQXL/yaOfSPmsFU7AFZRoaGgqlSzqHkmKRn58vUQeVVHZpEORVUrkRERHw8PCAvb09Y7OyssKSJUvg6OgIQggiIyPRq1cv3Lt3D3Xr1gUANGjQAAMGDICVlRV+/PiB0NBQdOnSBTExMeBwOHj69Cl69OiBGTNmYOfOnYiLi8Po0aNBCEFYWJhIPQT1O3HiBPz8/ODl5YWIiAhYW1uDz+fj9u3bCAkJQatWraCnp8f4X7x4ES4uLsjLy8M///yD4cOHw8zMTEQTnjlzBtOmTRNq1/L6XkiLYVl8LyTlUxb+Aj9bW1t8/fqVsRcUFCA8PBxhYWHw9vYWqevo0aNRp04dxMXFQUlJSeJvwujoaADA5s2b4eDggCdPniAwMBA5OTlir42yYsiQIWjQoAHCwsJgYGAgtm50jb6K1TwOqakYlLsHA3JPQRUFjP2OUl2s0AhAMrvw3mtvaQATE40S8zp9+gTTydegQSN07NSlxB11f5dYyKXzCEUiaWlpBAB5+/Yt4fF4jJ3H45FPnz6RFy9ekJycnAqrD5/PJ/n5+YTP5yuch7+/PwHA/FlbWxNCCPH09CTjxo1j/l/UB4DUMgV1S05OJn369CEWFhZEXV2duLq6kr1790qsuzx2WW1PnjwhAMiNGzeE8uRyucTDw4Ns376d+Pv7k27dusnabAy1atUioaGhEus9d+5cUrduXSHbqlWrmHYmhJCrV6+Shg0bEg0NDaKrq0s8PDxIbGys1LIFZXp6epKgoCASFBREdHR0iKGhIZk1axZTF2nXSWlj4enpSf7880+ZYlGcNWvWkGrVqpV4nj169CADBgwo0ad4mdbW1mTVqlVCaQYGBmTChAnMZy6XS3r06EG6du1KcnNzhXwjIiKImZkZefXqlcSykpKSCABy/fp1mb+LpYmFtbU1WblypVT/DRs2EH19faFzmjJlCnFycmL8fH19SZcuXYTyaty4MRk5ciSTr5mZGVm+fDmTnpqaSlRVVcm+ffvE1j0nJ4e8fPmSuQfyeDwSHx8vdK9UBHnzkdVfmp+kdHnsstrKm6oWC35eDsk8upYkBzYmyaMaMH/hf04ijafcI7WmvSS1pr0kLtNekjaL35L8Aq7UNj9z+iQ5fOgAycnJobEogkC3pKWllaq834nibVa0rYvf54pS3s9acTZZnz2enp5kzJgxZNy4cURPT4+YmJiQrVu3kszMTBIQEEC0tLSIvb09OXv2rNBx165dIw0bNiQqKirEzMyMTJ06lRQUFDDlZmRkkIEDBxJNTU1iZmZGwsLChPQjIYTk5uaSSZMmEQsLC6KhoUEaNWpErl69ytR/+/btRFdXt8T6Fz9/LpdLli5dSuzt7YmKigqpXr06WbBgAeMXHBxMHBwciLq6OrG1tSWzZs0i+fn5TB6zZs0S0mYCDViSrpo+fbrQ983a2prMnTuXDBw4kGhraxN/f3+Sn59PgoODiaOjo9iyw8PDRfR0eHg4IYSQlJQUMnToUGJkZES0tbVJ69atyZMnT4TOf/HixcTExIRoaWmRIUOGkKlTp5K6deuWeA24uLiQ9evXS21bfX19smXLFon5PH36lAAg7969I4QQMm3aNOLu7i7kf/LkSaKmpkbS09MZW9FrNDMzkxgaGpIePXqIpBX1J4SQmJgYAoA8evRIqB5t27Ylo0aNEjrm48ePREVFhaSmphI+n0/mzJlDqlevTlRUVIi5uTkZO3as1PMvypQpUyTGkBBRbS+4fubOncvEb+TIkSQvL4/x8fT0JGPHjiXBwcFEX1+fmJqakrlz5wqVu2LFCuLq6ko0NDRItWrVyKhRo0hGRobU+paFBhWkubm5kSFDhoikrVu3jnh6epLLly8TACQlJUWue9TSpUuJra2txLp9+PCBACCPHz9mbCkpKQQAc7/48eMH6devHzEyMiJqamrEwcGB7NixQygfW1tbsn37donnWtI9XByVoUEVSZNF41S05sl7cZt8n91DSOe9HN2OBE0MJ7WmvhDSeVye9L6M1NRUErZ8CXn+/JlM5f8usZBH51X5EX1VASJh3rUgjRR5E526ZBBI+g+RPMqyLuKGabN0DKA3bafU41evXg0bGxv89ddfuHfvHjgcjsi5HDlyBG5ubhg+fDiGDx+OgoICJp3NZmPHjh0ICAgQW7fc3FzUr18fU6ZMgY6ODs6cOYOBAwfCzs6OGQVHir25//jxI9zc3Eqs9/Tp0zFjxgyhuhYtt+i/27Ztg5OTE5o3by7kFxoaChMTEwwZMgQ3b94UW5eS4P9/yra+vr7YcmWxcblcdO/eHcOGDcPevXuRlZXFrPkhS10EPpGRkRgyZAiioqLw4MEDjBw5EtWrV8fw4cPF1uPmzZvo3LlziXlv3rwZvr6+YuuiyLkW5+vXrzh69Cg8PT0lnuvjx49x+/ZtzJ8/X+bYFL9++Xw+jh07hpSUFCgrKzPpbDZbaKp20fwHDRqEvn37CvkXRzANvKT479mzB6NGjSqxvmfPnhUaTVtSm4mzF/e/e/cumjdvDhUVFcbWoUMHLFu2DCkpKdDX18edO3cwYcIEobzat2+PEydOMNMuEhIS0LZtW8ZHR0cHjRs3xu3bt+Hn5ye2XoL2FvwVv1cqgrz5yOovzU9Sujx2WW3lTVWKRf6LO8g+sBT87/GM7SPbHGEag/GvUg3GJniqTe1iAhaIUD58Ph/37t2Fg4MTY2/foZPQm1Mai//8KIoh7l4m+L8krSdJkwmQlC6PXWAT6DxpzwwBkZGRCA4ORlRUFA4cOIDRo0fj2LFj6N69O6ZPn45Vq1Zh4MCB+PjxIzQ0NBAXF4fOnTvD398fkZGReP36NUaMGAFVVVWEhISAEILg4GBcv34dx48fh4mJCWbOnIlHjx6hbt26TH2CgoLw6tUr7Nu3DxYWFjh27Bg6duyIf//9lxlRJkv9i57/9OnTsX37dqxcuRLNmzdHfHw8Xr9+zeShra2N7du3w8rKCs+ePcOIESOgpaWFKVOmiOQlLn9BexXXVTY2NoyuAoBVq1Zh9uzZmDNnDnOstrY2wsPDYWFhIVK2r68vnj17hgsXLuDixYsACqefEULQu3dvqKur4+zZs9DV1cWWLVvQtm1bvHnzBgYGBjh48CBCQkKwfv16NG/eHLt27cK6detgZ2cnsQ1//PiBly9fwt3dXWL78ng8HDp0CFlZWWjcuLHYfLKysrBjxw7Y2tqiWrVqIIQgLy+PGUki8FdTU0Nubi4ePHiAVq1aibTphQsX8P37dwQHB4vUWZx2LP75wYMHePjwIfr16ydkP3nyJFq2bAkdHR0cPnwYq1evxq5du1C3bl0kJCTg6dOncul7LS0tiTGUVOfLly9DTU0NFy9exJcvXzBkyBAYGBhg4cKFjE9kZCQmTJiAu3fv4s6dOxg8eDA8PDyEpkyvWbMGtra2ePv2LcaNG4fg4GBs3LhRYl07d+7M/IYRh7W1NZ4/fy7SluLa4+HDh3jy5AnWr18vlP7y5UssXLgQd+/eRUxMjNh8StL+hBCkpaXBwMBAqrYtem8tbps1axZevnyJs2fPwsjICO/evUNOTo5Qno0aNcLNmzcxZMgQiXUrej+XRmVoUEXSZNE4FaV5eCmJyD68CgVPrjI2Ltg4otoB4WrdkcNSByCq8/h84WsjLTUVz188Q7Nmhb+RNDU10bffQJiYmJRp7H72WMiTB+3oE0Np1ujjp30HSUsWm29ZIe6WySKyrQOhoaEBTU1NcDgcGBkZASiclim4EXK5XOjo6IDD4UBTUxOGhobg8XgoKCgAi8WCk5MTtLS0JK5TYWJigvHjxzP20aNH48KFC9i/fz/q1KnzP/bOOzyqou3D99mSnmx6o/fekY4gCEiVZsGG7bW8KJZPEazY66uIoNiwKwIqIkWaSFFBRFFB6R0SNo30bDvz/bHskk12k93NJiQw93XlgszMeZ4553fKkzlnnrH3tUxOmqSkJH799ddy5aqqOj+Xj42NdeZSKW3D4ddRVlJSwueff86DDz7o0seffvqJ+fPns23bNme+BvXMFGxveeWVVygoKGD8+PFu+wLu83g4Lkir1Up2dja5ubkMHz6chg0bYrPZaNOmDYqiVNoXx74KIahfvz4vv/wyiqLQrFkz/vzzT1577TVuuummcscEoHPnzmzbts3tsXWQmJjoVd4gx/be5FAEuO666/juu+8oLi5m5MiRvPXWW+X2tUmTJmRkZGC1Wnnssce48cYbfc5rMn36dB577DFMJhNWq5XY2Fi/7Lj740tVVe6991769OlD69atPeo/YsQI53F22C17rOvVq1dhHiBHuaOudP/dtT958iSNGjVyXqNwdlrSsWPHiIyMJD09nfj4eBdbCQkJpKenY7VaOXHihHO7sm3S0tLcHkOZo0/m6KuwfVEumlXvodm50VkvtDrU/ldxIHEUJzZboOjstvERClMGhNExoRijsdBpp6iokLVrVnPs2BGSk1MYNHio1OI8y9F3rqhKjr7SsV5lwwme6n0pF9jjPE/PnnLthaBjx45Mnz4dgAcffJAXX3yR2NhY5yImDz/8MPPmzeOPP/6gZ8+ezJkzh/r16zNr1iwURaF58+Y8/vjjPPzww8yYMYP8/Hzmz5/Phx9+6Mxp+95779GkSRNn3HP06FE+/PBDDhw4QGpqKgD33nsv33//Pe+//z5PP/20S0xUGUIITp8+zezZs3n99de59tprAftgRq9evZw2HnroIWw2G1qtlvr163PfffexcOFC7r//fudzuHRs5ogBS+foKx1XNW3alB07djjjKgcDBgxg6tSpLjn6HnroIacWZX3r9XrCwsJcYm6ADRs28Ouvv3LixAnnVLQXXniBb7/9loULF3Lrrbcya9YsbrrpJiZPngzYF4RYu3YtJSUlHs+BgwcPIoQgMTGx3PH9+++/ufjiiykpKSEiIoKFCxfSsmVLl9hh3rx5zJgxg8LCQlq2bMmKFSvQaDRYrVYGDx7M66+/zqeffsoVV1xBeno6Tz31FADHjx93G9fs3r0bgGbNmjn/1ti6datLSpVPP/2UkSNHOj8q6Nu3LxqNBrPZjMVi4ZZbbmHSpEku/VyyZIlzm8OHD5OUlMTAgQMJCQkhNTWVrl27+hTfO64Tdxo6zpey509QUBDz5s0jODiYtm3b8sQTTzB9+nSeeOIJNBoNQgg6dOjAI488Atjj3Dlz5rBmzRpnTlJHOgIhBMnJycycOZO77rqL2bNne+zrW2+9RVFRkds4HnDJt15RbCuEYP78+bRu3ZoePXo4tzGZTEyaNIlnn32WlJQU57Rcq9Xq1d9hQgj27t3LnDlzePHFFz3q4CgvfW91/OuIfY8cOUKnTp2cH4M4Ft0obTM5OZkdO3Z4vDfKHH3VGPPYrChblqLZ8AWK5WzOZbVhWzY3u5aF+xpSXHD2ieYa5xW7mNqz519+XP8DZrMJBYXmLVo6+w/erTR7oWhxXuXoOxdUJUefxhCHqMYEyUJ4/qLP2zwQju1Lt3eXS8Jdjj7HQ9sTGo2G5557jkWLFnHixAnMZjMmk8k5uOjpJtumTZtyZRaLxatcNaXLvvvuO/Lz87npppuc++L4/Z133iE5OdnZz8py9JXm888/55lnnmHJkiXO4NVdX9zl8SidjyYxMZEbb7yRkSNHMmTIEAYOHMikSZNISUnxqh9g16pXr14uvvv27esMzh2+S9dHRkY6EyqD52Prqby0PUVRvMrR52DWrFnMnDmTvXv38vDDDzNt2rRybys3btxIQUEBW7ZsYcaMGbRs2ZJJkyZVdihceOCBB7jxxhtJS0tj2rRp3HnnnS777A2e9v3OO+9k165dbNq0qcJ8MDExMc4v/hxUdEwr8uvIw1NZjj7HOVe6rPQ54Ph/WVulz0utVuv8f9k2nnLMyBx9Mkef2/ZCJWrvZkqWzkMUnw1GdC27EX71dLRJDRmvqvRpaeRYUQRZBSoJkTq6Ng5Fq1Fc/BpPpfPdd0soLi5Gp9PRtVt3YmJiXJ7Hnvp5oWpRF3P0nSuqkqPPEet5iskceKr3pdxRpkTFep03WVEUOnbs6OyzTqcjLi7OpaxevXoAZGVlodPp2Lt3L3369HGx3b9/fwoKCjh16hQZGRmYzWb69OnjtJGYmEirVq2cz4l///0Xm81Gu3btXPpjMpmIj49Hr9f7nKNv3759mEwmhgwZ4nGbL7/8ktmzZ3Pw4EEKCgqcL60d7cs+yxwxYOm4pmxc1atXL15//XXnsxjseezKHvuvv/6aN954gwMHDnjlG2Dnzp0UFBQ4Y1IHxcXFHDp0CJ1Ox+7du7njjjtctuvduzc//vijx/jLMVgWERFR7li1a9eOP/74g9zcXBYvXsytt97K2rVrnS/hAa6//nqGDRtGWloa//vf/7j22mvZvHkzISEhjBgxgueff5677rqLm266ieDgYB599FE2b97sEms4KKu1o75z587OmSwtW9q/0i6d/23BggW0adMGi8XCzp07mTp1KjExMc4F4PLy8ti0aRNvv/02er2eq666ijfeeIN27doxbNgwRowYwejRo33Kjffll1/6pKFGo6FTp04YDAZnjNe3b18KCgpIS0ujUaNG5a5BgNTUVDIzM51la9eu5YUXXmD37t3k5eVhtVopKSnBbDYTFuY+f1mjRo2cWlcld2VxcTELFy7k0UcfdenjtGnTaNu2Lddddx16vd4lPqzo/uMoO3HiBJdffjkTJ07k9ttv99in0vem0v+Hs/Hqf//7XyZOnMiOHTsYMmQIY8eOpU+fPi52wsLCKC4u9tg3maOvemIey97tFH35Mrb0Q84yJTKGsHFT0XUfRt/MTEaPjWfHURMZ+dZycZ6D4uIivlv6LX///ScA9es3oFWr1sTFx0stPOBLnCcH+rzAMbBRWhhHIOYYIHMQM+OTauuH422STqerMLCszEbpvpem7L6UbedNMPvyyy8ze/ZsZs2aRYcOHQgPD+fee+/FbDa7tSOE4NixY85kwJ5wvFEua6Ps/rz//vuMGjXKJXg6ePAghw8fZsyYMc4yxxtlvV7Pnj17XJIWl2XBggX85z//YdGiRc7P7T0dR8dU6LJvk0q3++CDD5g6dSorV650roS6Zs0al4Ug3FHWpzdaOf7dtGmTyxtUd8ybN885RbOsRu72tTItHKSkpJCSkkKbNm2Ii4ujf//+PP744y6Dm02bNgWgY8eOGI1GnnzySedUDW+PSUJCAi1atKBFixYsWrSIDh06cNFFF9G2bVuf7JQ9z++66y6WL1/Oxo0badCgQYXtP/vsswoDG4CVK1c6p+56slN62kFlWiQnJ2M8s3KVo+zUqVPOOkVRnG1K2zIajc56hxZGo9FlIPvUqVN07tzZ/cuFM+dg6Xuju3ulP/hqx9v2lbXzVO9Lubdl1c250MJ65F90nz5D8cn9Z7cPNxA+4V6Ce410OY90Wg09m0W4tVtSUsKaNd+zZ/e/AKTWq8eVV0wiLj7e+XW91MI9Nblf5xue7mWOe13ZWK+ymMxTvS/lZcsqelaVJSgoqNwzu3SZu2d3WbsV3fvdlRUWFqLVatm+fbtzgMBBRESEi01v4lghBKGhoR79Avzyyy/O1TeHDx9OdHQ0CxYs4H//+5/zmFW2P2Xte4q3HAtxOexu3bqV6667jieffJJhw4ZhMBhcfHva38LCQlJSUvjxxx/L9Sc6OtrnWM9BQkICYE81kpiY6FIXHBxMixYtAPuA5W+//cacOXN45513nHaio6OJjo6mZcuW9O7dm5iYGJYsWcKkSZMQQnDffffxwAMPkJ6eTkxMDIcPH+bhhx+mWbNmbo+dY4XXvXv30qtXL4QQhISE0Lx5c4/72KBBA2c/27Zty4EDB3j88cd56qmnCA0N5fvvv6dt27Y0bNgQsC80snv3blatWsUPP/zAlClTeOWVV9iwYYNXA2GO88dXDSvSxPF/d9eg4/gcPnyY0aNHc+edd/LMM88QFRXFli1buPXWW12+XizL8OHDK526u2uXfQGDiu4XixcvpqioiBtuuMGlbv369fz9998sXrzYaQPs59bDDz/MY4895mKvtI+0tDQGDRpE7969Xc4rd5S+Pzjalf57SVEURowYwZEjR1ixYgVr1qzh0ksvderrICcnx3nee/pbpbpiSl/bV9TOnzpvYpxAxzxqbiaFX7+O6dfvS3VEQ8jFEwgbcyeasEjnoJZep6Vn8wiPtg4c2M9Xi78kNzcXjUbDwEsGM3DgoHLnhtTCFV+2lwN9XuBp3rWjrnQQURN9Kf1vIG2U3pegoCCXT7+98SmE4Oeff2bMmDHO6RWqqrJ3717atm3r0U5KSorz7V5pSr+tcqym5M6G4/8HDx5k/fr1zrxjDlq1asVff/3lYvuxxx4jPz+fWbNmOfOPuOOLL77glltu4YsvvmDEiBFu/ZYui4+PJz093XmTA9ixY0e5dp07d6Zz58488MADDBgwgM8++8yZK6UiHDa2bt3qYu+XX36hRYsWzou/bN+6devmcozdvQlMSkqq8NwoW+5tWWkcn9WXlJRU2MZkMnl9jru7FuvXr8+VV17JjBkzWLJkic92HP/efffdLFmyhPXr19O4ceNK93X06NH06NHDxW7ZY12vXj2fjlllWvTq1YtHH33Uxc+aNWto2bIlMTExCCHo3bs369at45577nHacQwuCyFo3LgxycnJrF271jnonpeXx9atW7njjjvc9s1xvB1ToAKVf8JXO4HIyVFRfVVycvizP4GgprVQi/IoXjoP0+avUUqdK0E9RxA2/h40EdEu12dFdrMyM/nww/c4ffo0iqJw8YBLuOSSwWi1WqmFl+e5xD/c3csc//cU63l7//bmmeqp3N2zqSKfpberKM4rW9a6dWu+/vprl/hl8+bNREZGUr9+fSIjI9Hr9WzZssX50isnJ4e9e/dy8cUXI4Sgc+fO2Gw2Tp065ZKL1pt990Tz5s0JDQ1l7dq13HrrreXqf/rpJxo1asT06dOdz8DDhw+79VGRhmXjqq1btzrjKk/H/ueff6ZRo0Y8/PDDzu3K+tbr9c7pwQ66dOlCeno6Wq2Wxo0bu+1TmzZt2LJlC9dff72zfMuWLW774aBp06ZERUWxa9cu52CZJ1RVxWQyeTwujnO/dLzmGNRxvBz8/PPPadCgAV26dHF7zg4ZMoTY2FhefPFFvv76a7d993ROOtBqtVitVsxmMyEhIXz77beMGTPGZfvQ0FBGjhzJ2LFjmTJlCm3atOGvv/6ia9euFR4DOHv+VKShu+P9559/Or82B3scHhER4fI3RUX3jd9++w1VVXnllVfQaDRYLBaXY+Tp+nj33XcpLi6ucNaTNzHm/PnzGTVqFAkJCS51ixcvpri42PmCYdu2bdxyyy1s3LiRZs2aebxHnThxgkGDBtGtWzfeffddl+vGHY6p7CdPnnROzS2ds9yxbXx8PDfccAM33HAD/fr1Y9q0abz88stOOzt37nSmEvDUt9L388o4FzGoP3XexDiBjHmEzYpp41cUL3sbUVLoLNc2akf41dPQNWzt9OmN3x9//IF1a1cDEBsXx8SJV9GgQUOnDX/6f6Fo4YsNOdDnhqrk6KtOhKg4j5i3Nhw3wLK5vxz7A/Y3Qhs2bGDChAnodDqSkpJQFIX27dvzzDPPMHbsWLd9a9q0Kd988w2bNm0iOjqa119/nVOnTtG6dWuPucgURXF+5l7WnlardZZXlhvivffeIyUlhSFDhrjsm06nKzeF0zE9x1FutVp55JFHOHnyJB988AFwdpDv1VdfpVu3bhw/fhyA0NBQoqKi3O5Pv379yMjI4IUXXmD8+PGsXr2alStXEhUVhdVq5dChQ7z33nuMHj2a5ORkdu/ezb59+7j22mt9ytF39OhR7rvvPm699Vb++OMP5syZw0svveTMgVK2b3q93hlMuju2ZY+lu/I//7R/Vp2fn4/RaOT3338nNDTU+cXckiVLePTRR/n7779RFIWVK1diNBrp1q0bERER/PPPP0yfPp0+ffpQv359rFYrb731Fg0aNKBVq1aA/Y+K//3vf0yZMsXn3Hqq6ppzccqUKXTp0oWtW7fSrVs3r+049v3uu+9mwYIFfPXVV4SGhjr1NxgMhISEuD1WoaGhLkG7p2PtKX+K2Wzmn3/+AexTnY4dO8Zvv/1GREQEzZs3RwjBnDlz+O6771i92v6AvOKKK3jyySe5+eabeeCBB9i1axezZ8/mxRdfdL4VnjJlCoMHD+bll19m+PDhLFy4kN9++425c+c6+3L33Xfz7LPP0rRpUxo3bszMmTNJTU1l1KhRMkefzNHn3o5QUXb8gGbtByhFeWe3S2iIOvJOrI3aU1RkhiKj13ZtNhs6nZ6IiEiGDhtOvXr1ycrKqnA7qcVZZI4+76lKjj6oPCar7JnqTXnZMm/jwLIxnYOyz0k4mw/rtttu4/XXX+euu+7izjvvZO/evcycOZN77rkHm81GaGgoN954I9OmTSM6OpqEhAQef/xx5x/0VquVpk2bMmnSJCZPnsyLL75I586dyczM5IcffqBDhw4MHz7c+UeKt894vV7PAw88wEMPPYRWq6VPnz5kZmbyzz//cNNNN9G0aVOOHj3KggULuOiii/j++++dL/gcMZHjj6yKcvSVjavefPPNcjnGhBDO56oQwun7s88+o3v37qxcudLFN9i/UDt06BC//fabc8B04MCB9OrVi7Fjx/L888/TokUL0tLSWLFiBWPHjqVbt25MmTKFW2+9la5du9K7d2+++OILdu3aRZMmTSo8BwYNGsSmTZsYNWqUs+yRRx7hsssuo0GDBuTn57NgwQJ+/PFHli5disVi4dChQ85ZK/Hx8Zw4cYKXXnqJ0NBQhg4d6jyOr7zyCpdddhkajYYlS5bw4osv8vnnn7sc29LnaEhICPPmzePaa69l5MiR/Pe//6Vp06YUFxc7YxjHsXJMOz516pQz59/OnTuZPXs2AwYMIDQ0lJKSElauXOk8JwE++eQTrFarM9b8+OOPCQ0NdcmHXBHeaOju/DGbzdx8881MmzaNY8eOMXPmTO68885yLwjK5ux2XIONGzfGYrHw+uuvM2LECH766Sfefvttp19PfXe8lHcXW5Y+nmW1KN1u//79bNy4kW+++abc14ONGjVyse+YIdKiRQsMBgM2m41t27Zx8803s2rVKlJTUzl69CiXXXYZjRo14rnnnnMOYiuKUm56ugO9Xk/Pnj154YUXaNCgARkZGTz66KPA2XvSzJkz6dq1K23btsVkMvHdd9/RunVr5/4VFRWxfft2nnrqKZmjz01ZwGKeY7vRrngLJf2gs0iERKBeOhlr16GYFA0Yz8Z53vgNDbVPTW/fviP9+g9Ar9c7Zyf5Ysef9nVaC3yM84TEI47li/fs2SMsFouw2WzCZrMJi8Uijhw5Inbt2iWKioqEqqo19mMymaps4+WXXxaNGjVyKRswYICYOnWq8/eff/5ZdOzYUQQHBwvAWQ6I+fPne+xbZmamuPzyy0VERIRITEwUjzzyiLjhhhvE5Zdf7rHvvpR7KrNaraJ+/fpixowZXh2DyZMni8svv7xc2YABA1yOCVDuZ/LkyRX2+8033xQNGjQQ4eHh4oYbbhDPPPOM83inpaWJsWPHipSUFBEUFCQaNWokHnvsMWG1WoWqquLgwYMCED/88IPHYzJgwABx5513ittvv11ERUWJmJgYMWPGDGGz2bw+T/zRwt2xKH0ezZ8/3+VcWbdunejdu7cwGAwiJCREtGjRQkybNk1kZ2c727z++uuiXbt2IiwsTERFRYkuXbqIuXPnOo+HO7vu+taoUSPx6quvlqsbNmyYGD58uF/Xlrv9LX3+e3st+qKFQ/+yP6XPy0cffbTc9fvbb7+Jfv36ieDgYFGvXj3x/PPPl/P75ZdfipYtW4qgoCDRrl07sWzZMpd6m80mHn30UZGUlCSCg4PF4MGDxe7duz32u6ioSOzatUsUFhY674snT550uVf68+OrHW/bV9bOU70v5d6WVfdPTWhhOrpb5Lx0s8i4o/vZn6n9RfriucJcUuyT3fT0dGE2m531RqNRHD58WGrhh52cnBwBiNzc3HMdQtUZHLFeTk5OuWNdWFhYYaxXHc/aysq8efaUjelUVXX7nATE119/7fx9/fr14qKLLhJBQUEiOTlZTJs2TZjNZqffvLw8cd1114mwsDCRlJQkXnzxxXK+TCaTeOyxx0Tjxo2FXq8XKSkpYty4ceLPP/8UqqqKd999VxgMhnLPvYriHqvVKp5++mnRqFEjodfrRcOGDcWzzz7rbPPAAw+IuLg4ERERIa666irx6quvuvh49NFHRadOnZy/l44BPcVVDz30kLDZbC7H7+WXXy7Xt8p8FxcXiwkTJojo6GiXGCI3N1fcddddIjU1Vej1etGgQQNx7bXXiiNHjji3feaZZ0R8fLyIiIgQkydPFg8++KDo1KlThefA8uXLRb169VziqJtuukk0atRIBAUFiYSEBDF48GCxatUqp53jx4+L4cOHi8TERKHX60X9+vXFNddcI/79918X2wMHDnTGdD179hTLly/36jz+9ddfxcSJE0ViYqLQ6XQiLi5ODBs2THzxxRfOY1w2/tFqtaJ+/fri1ltvFcePHxeqqoo1a9aI+vXru/j4+uuvRc+ePUVUVJQIDw8XvXr1EmvWrHH6fvzxx8vFTGV/KtPw8ccfd3v+PPbYY87tbr31VlFcXFzhNXj55Zc7/4ZQVVX873//EykpKSI0NFQMGTJEfPTRRwJwiZN9vbd402769OmiQYMGLv31tN0PP/zg0ieTyeQsO3jwoPOa9hQzV9S3Xbt2id69e4vQ0FDRuXNnsWrVKpd7wVNPPSXatGkjQkNDRWxsrLj88svFgQMHnNt/9tlnolWrVhXua9lYNdDP7EDEoP7UeRPjVDX+sORmibyPn3KN8+7oLvI+fkpYcrN82leLxSLS0tJc2p04cUJq4cePL3GeIkQF39Ve4DgSNO/du5dmzZq5jNKeOHGC/Px8mjZtWmPJr4UITI4+X2x4276ydp7qfSn3tqwmqC4t1q9fz/jx4zl48GC5RR0uVC2eeOIJNmzY4DaXTaB8yuvCd0pKSjh06BBNmjRxLsZhNBrLLZTgK77a8bZ9Ze081ftS7m1ZdVOdWqjFBRQte4eSHxeCanO2Dep2KWHjppJpUbzWIj4+nl+3bmHVqhX0v3ggl146tML+Sy0qt+OIW3Jzc4mKivLb34VE2WNW+libzWaX+1xpzsX9vS7HPJ7sVHfc409dXdZCCEHPnj257777KlzU7FzEPVXV4p577sFqtTpnI3ijxeTJk1EUhQ8//LDSffSWG2+8kdOnT/PNN9/U2Rj0fLguevXqxdSpU7nmmms8+i0bq1bGuYhB/anzJsbxN/4QqorppyUUfvsmojDXWa6t35KISQ+hb9qxgq3L9yMnJ4evFn9JWtpJ7p56P9HR0W63e+rJJ8g5nYNG0RAaGspNN91CUHAwn336McePHycoKAiDwcBtt9/pTB8w/aEHsFgcqzVbOXbsGK+88hpNzuR+/37lClasWO78wvSFF19Gp9Oxc+ffLF60kMLCQswWM926dueGyTcCVKjFY48+TFFRERqNvY833/IfGjduzIkTJ1z6GWUwMH78RNq3b49Go+GhaQ9QXGKf6q/aVI4dO8r/Xn3dOQvMXT+DgoI4efIk8+bNpbDgbD/HjhtPTEyMV3GenLorkdQiVqxYwcMPP1wu2L2QWblyJXPmzDnX3ZBILmiEEJh/W03hV7NQczOd5ZrEhkRcPY2gNj3tU/LKTL3wREFBPiuWL+XAAfvCHWknT6CqqlxMQiK5wJBxT2BRFIV33nmHv//++1x3JeC0b9+e3r17e91eCMGPP/7I5s2bq7FXknNBZmYm48ePr3AwW+I71iP/UrDgRayHdznLlJBwwkbfQciAiSha74eOhBDs+ON3li5dgslUQlBQEKdOpXkc6Pu/Bx4kPNy+eMfWLb8wd+4b3Hf/A1w6ZCjdunVHURRWrFjOW2/O4amnnwXghRfPLszy00+bWfDFZzQ6M3j2669b2bhxA8+/8BLh4eHk5uY6F/r45uuv6NGjJ6NGj8FsNvPQtAf444/f6dKl4ryet9x6G40bN0aj0bB1yy/MeeN1XvnfawAu/Vy+fBmff/YJzz3/IgDPv/CScwBx65ZfWLjwS+cgX0X9/PjjD+nZsxcjR4529rNZ8+ZeayAH+iSSWkTpBLMSO7/++uu57oJEckFjSz9E0ZevYNn729lCfTBhw28m9NLrUPRBPtn7++8/+XbJ15hMJvR6PcOHj6JHz141+lWMRCKpHci4J/A4Fnw737jtttsA+wCCNyiKwpEjR6qzS5JzRHx8PNOmTTvX3ThvUIvyKVr6FiUbF0Op6yu4x2WEj78HjSHeJ3slJSUsXPgFO/+2L4bZoEFDrrjyauLiPNtxDPIBFBYVgWLP5di1azdnfNiyZUuWfrvE7fY/rFtL7z59nb9/u+Qbrrzqaudq6QaDwb6vqn3BqaKiIgDMZjNWm5WYmFhUVWXOnNfp3u0ixo4bT3p6Go89+ggzn3yKlJRUwsLCXPpYOhe+Sz9btGTJN1+77ee6dWsZPPjSSvsJoABFha799DRQ6g450OcFQlwYq+5WtX1l7TzV+1LubVlNILWoHVoEyqfUwjcc9z5Vlavu+lpW3QTKp60wH2X1fHJ//Q5sZxOD6zv0J+yK+9HGpTqS8Hjlt7i4mGXLvuWvP3cAkJpaj4lXXO1c9c9xrkot/LdTk/t2vuHuXub4v6dY71zc3+tyzOOPnUA8a/2pk1r43/5818KxaF8g/Uotqo6nvpW+n1fGuYhB/anzJsbxpm9CCMy/rqTom9mI/BxnuTa5CWFXPYi+ZTenLW/Zv28vixcvpLCwAI1Gw8BLBnPxxQPRarWV2nlj9uvs2mX/Gnn6jEfK9X/Zsu/oftFF5ezYF2jaxdWTrnXWHTt+jP379rHwywVYLBYGDBjIiJGjUFWVCROu4L333mb16u8pKCxk4oQraNy4MaqqMnnyzfzvlZdo1qw5H338AdffMJmUlFTn8Zw9exb/7NoJwIyHH3N7nJcv/46OHTuW08JoNLJr107uuvueSvsJcONNN/PC88+xatVKZz8dqxN7gxzoc8P5vuquLza8bV9ZO0/1vpR7W1YTSC1qhxaB8im18B256u55vNKrUFF2rEOz7iO0pfKziOhE1OG3Y23Zg2Ib5abpVuY3Jyebf3btPLOCe0f69rvYGfh4Y+eC1MJHO3LVXe85n1fdDTR1+VnrT53Uwv/2UovqtyO1KI8nv3LV3Qr6ZjyCdvlbKEfPTtMV+hDUAZOw9hqNSav3Oh1Laf7Y8TuFhQVER8cwdNhwkpKSycrK8mrbq66eBExiy5Zf+GD++1xz7fXO/n///QqOHzvK1HvuLxc3rlyxnHbt2mO1Wp1jNjarlSNHj3DX3fdQVFTEa6+9QnBICO3atWfNmtV06tSFy4aPID8/j1mzXiUuPp5WreyrOl933Q08/fRMevXqTcuWrTAajc7jeeWVV6PRaNiy5Rfmz3+PO++c4nKcv/9+BceOHeOGyTc5++LY9peff6Jd+w6UlJRQUlIC4LGfHTp0ZMk3X9G5cxeGXTbc2c+wsHCvtZADfW6YMmUKU6ZMcSZoNhgM5ZJd2mw28vPz0Wg06HQ1exi9uVEF2oa37Str56nel3Jvy2oCqUXt0CJQPqUW3qPRaNBoNMTHxxMcHOz8FD4hIaHKAxq+2PG2fWXtPNX7Uu5tWXXji0+bKvj9cDEZ+VYSInV0FAco+epVbEf+OdtIF0TIkOsIHToZJchzMmtP++/4f2JiIuPGTcRgiCYkNFRqUQ12ampxsPOBsrFeQkKCczEOx7E2m82Vxnrn4v5el2Mef+wE4lnrT53Uwv/2UovqtyO1KE9Zv2Vj1co4FzGoP3XexDiO32Pj4tlx1OSM87okq5hXvk/Jj1+6LKqm73wJYRPvQxuTVOl+u9s/Rz/GjBlHSHAIQ4Ze5ndMMmbM5Sz44nN0Oh2JiYks+24pu3bu5ImZTzunuDoQQvDrr1u55db/EB0d7dz/hIQELr10CMnJyQBcdFEPjMZTXHLJILZu3cLrs+cQHx9PYmIiF13UgxMnjtO//8UoisKBA/uIjIyiqLiYhIQEFEUpd3zHjLmcBQs+JzQ01Fnu6OfjTzxJUVGRixYA27Zt5T+33UFiYqKz/576mZiYyMZNG5k9ey6xsbHOfqanp3l9HOVAnxcoiuK8UTjQ6XRoNBrS0tJISEggKCio2t9cOL4gtNlsVfqKzBcb3ravrJ2nel/KvS2rCaQWtUOLQPmUWniPEAKz2UxGRgZarZbg4GDnvdHdvdIffLXjbfvK2nmq96Xc27Lqxhufa3bm8fyyU5zKtRKn5nB78UKaWX5yaaO26UPMpAfRJ9T32W96ehqLF3/JqJGX07hJEwA6d+mKqqrOt5xSi8DakYuZ+E/pY+s41iEhIWi1Wrex3rm4v9flmMcfO4F41vpTJ7Xwv73UovrtSC3K464fnmLVyjgXMag/dd7EOJsPWHjrw8OcyrOCEFxi+ZUY0+fE2s5O09UkNCDi6gcJauv9QjcOVFVl8+aN7N+3lxtvuhWNRkNwcDA9e/UhJCTE62NYWFiAyWQiNjYOgK1btxAZGUlERATLl3/HTz9t5omZTxEREVFu27/++hNVtdG5cxcyMzOd+9+//wD+/HMHnTp1xmQy8c+uXYwdN9458PvXnzu4dMhQSkpK+GfXTkaPGYtGo+HYsWMs+24pr/zvNd6c+wZLv13CuPETKCwsJDc3l6SkJDQajb2PEZFERUVhMplc+hkWFkZxcbGLFnv37sGmqnTu3MXluFTUz6SkJP78cweDB1/q7OelQ4Z5rY8c6PMTRVFo3Lgxp06d4uTJkzXiUwjhHDGvyuCSLza8bV9ZO0/1vpR7W1YTSC1qhxaB8im18J2wsDAaNmwoBxbqKGt25nHfZyfQCQvXmL7n+pKlhGFy1hfHNiHxugfIiWmINi6xAkvlUVWVn3/ezOpVK7HZbKxYuYw777yrRu/REkkg0Gg0NGnShLS0tHKx3rm4v9flmMcfO4F41vpTJ7Xwv73UovrtSC3K48nvhRyrrt2Vz8zlhQA0tJ3knuJP6G49O01X1QYRMeImQodcj6Kv/GvHsuTk5LB40QIOHz4EwK5dO+nQoaNffS0sLOJ/r7yE2WxGURSiDFFMn/EIp0+f5uOPPiQpKZknHn8UAL1e57La7rp1a7lk0OByGo8eczlvz3uTe6ZOAUWhV6/e9OnTF1VVuWHyjXzz9VesWLEcq9XKRRf1oF+//hQWFvLB/He58793ERMTw9R77uWhaQ/Spk1bYmJjeOftt1CFikbREGWI4uFHHkVRFHJyclz6KbDniXzlldec/fnl55+45JJBXvcT4O677+W9d99m+bLvnP3s2bOX18dVDvRVgaCgIBo2bOh8g1DdOHIMxMXF+X3D8tWGt+0ra+ep3pdyb8tqAqlF7dAiUD6lFr6h1WrR6XRy4KaOYlMFz3+XTm/L70wp/pz66tlcJ3lKOB+EjGdLxFBWtGgBmRk+2c7Pz2f5sm85ePAAAK1atWb8+CvkuSKps3iK9c7F/b0uxzz+2AnEs9afOqmF/+2lFtVvR2pRHnd+L+RY1aYKXlhmJF7N5qaSb7jMvAkdZxeL+EXXic8Tb+TTy/qiaHw7PkIIduz4ne+WfovJVEJQUBAjR42hffsOfvc3MTGRF196xaXMMftj0eJvKjyX7rvv/5ztSxMUFMTdU+91u02DBg155tnny9kNCwvjiZlPO6fWRkZG8eZbbzvtT3tohks6N0d5TEyMSz8dfS/NTTff6jJl15t+Nm3alOeef9GlLC8vz21bd8iBPi9wvCVwoKqqy5sDrVaLVqut9n6oqopOpyMoKKhKg0u+2PC2fWXtPNX7Uu5tWU0gtagdWgTKp9TCd4RwXYWy9H2xKvhqx9v2lbXzVO9Lubdl1U1lPv/a8hfTTs5yebNrQ+G7oEt4P2QCeZpIyBP8dqiQxhHe9/3PP//gu6VLMJlM6PV6hg8fRfeLeqAoik/H5ELSItB2anLfzjdUteIVxMvGeufi/l6XYx5/7ATiWetPndTC//ZSi+q3I7Uojye/ZWNVb+zUdAzqT11lMc4fuzMYe+ozJphWE4zF2SZNieeNsGv5SdcVihR+O1jIRU3DvNpXgKKiIpYu/YZdO+0r4zZo0JCJE68iNi7Oeazr8t8D1aGFP/tSWf+9RQ70ucGbVXdreuW+QPn11Ya37Str56nel3Jvy2oCqUXt0CJQPqUWVUdqUQe0yM9C88On1N+xjgacDXz/0LbmjbDrOKBt6GLnwIlsopOKver78WNH+eabxQAkJiYxdNgIYmJiyMgo/0Wg1KL67MhVd73Hm1V3a9s9pS7HPP7YCYQW/tRJLfxvL7WofjtSi/JILc6UZWWgbF5M/U1fcY2l0Nm+gFC+CBnJouBhmJSz03QPnMimUURBpfvpYOm3X3PkyGE0Gg09evSiW/ceWG02l6/XpBbVe134EufJgT43eLPqrqLU7Mp9gfLrqw1v21fWzlO9L+XeltUEUovaoUWgfEotqo7UovZqIUqKKF77KSXrPgNzibPdSU0Cb4dcxY/6i8DN1JZm9WKJjij0qu8JCQns2bub8LBwho8YVeGKexeyFtVtR6666z3erLpb2+4pdTnm8cdOILTwp05q4X97qUX125FalOdC10LYrJT8/B365e+g5Gc725nQ803wpXwWPMo+W6MMzerFkpjo/Rd9o0aPZfHiBYwbN5F69dwv0naha+GuLJDXhS9xnhzo8wJFqR0r9wXKr682vG1fWTtP9b6USy2kFmUJlE+pRdWRWtQuLRQE5p+XUvjdPERe1tm60Ag+CrmcT9RBmJXyA3IKkGTQ0b1JOFmZRW77brVa2bhhPb379CM0NBSAq6++loyMDPR6vdTCiz5Xh52afg6eT5Q+trX5nlKXYx5/7ARCC3/qpBb+t5daVL8dqUV5zjct3n/vHbZt+5WMjAxeevlVmjVrBoDFYuGDD97n9+3bCQkJoaEhlJs0B7GdOsLrObHkqQkoQIkukgMJV5AW2g6A8ILdJGSuBgSKULHWG0j3pq3RlMrR9/fff/HUk08wefJNjBo9hpMnT3DixHEuuqgnAKmpqdx9930oin2bl196gT17dtsXo/jkM8LDI5z7U1RUxPz332X//v1odVq6d7+I66+fTElJCTOfeBSz2T6lOCYmhtvvuJPExCQA1q//gaXfLkHRKNisNq6/YTLdu1/kcrx+WLeWuXPfYNpDM1wWp7gQrgtftpcDfRKJRCKRnGcoB34n791PsJ3cf7ZQoyVkwBWEjbiFDoc1WD47gQKUzl7jCPemj0pC6yFBc3p6GgsXfsGp9HQyMzO48qpr7NsqF17Ca4lEIpFIJJJA06t3X8ZcPpaHZ0x3Kf/0k49QFIWZkycSvPELsg/+ik1rz9t2myEHQ+f+hF/+X97ffJjj3yxEadwOIQSpaV9ytMFtmENS0FmyaXn0NcymsYSG2r/oKyws5NNPPqZL124IIdiwYT3r1q5GCEFKSir16zcAXGO9ocMu47bb7uDmmyeX6/+bb86hTes23HtmsYycnBzAvvjEEzOfcvr97rtvmf/+e0yf8Qj5+fm8/947vDHnLQwGAz///DNz58zmgw8/cdo1Gk+xdu1qWrZsFahDfd4iX/1KJBKJRHKeYD2xn/y596L99AmXQb6gzgOJeWIhEVf+H5qIaIa0j+K1a+uRaHB935dk0PHatfUY0j6qnG1VVdm8aQNvzp3NqfR0wsPD6dCxc3XvkkQikUgkEskFRbt27YiLi3cpKykpYd2a1YzO/wvdJ49iO7ILw5lBPl2zzgTd+iKRt7+MLrUZzeMEyVFaEqMccZ6CRi0myaBj+mUGDFGR6HRnZ3W89947TJx4BcFBQWzZ8jOrV63EYrFw4sRJ9u3bB8CePbu54/b/kJubC0CnTp0xREeX67vRaOTggf2MHnO5sywmJgawf5HmGOQTQlBcVOxMH+NY0KO4uBiA4uIil2OgqipvvjmHW269rcIUMRI78os+iUQikUjqOLaM4xQtewfTtu+h1ApzukZtCJ9wL/oWXcttM6R9FIPaRrL9cBEZeVYSonR0axzm9ku+06dP89XiLzl48AAArVu3Ydz4K4iIiKi+nZJIJBKJRCKRYMs4zsFPXyPMUsjS3zP51xxPkCK4vFEk3a+/B23b3pRkZPDG7NfZtcu+Ku5/75hCx05N+eNoCTt23MXGr98kvDiUZX8X8OC06c7Bsl9+/glFUdDqtPz77z/o9TqSk5MZPXosScnJPDnzcRrUb8CsWa8y9Z57MRgMFfY1PT2N2Lg43nl7HgcO7CcyMpLrrp9M06ZNnW1mznyMo0eOEBVl4LHHZwIQFRXF7bffyYMP3EdERKR9mu/Mp5zbfLf0W1q3bkOzZs0DfHTPT+RAn0QikUgkdRQ1N5OiFe9T8tMSsFmd5cKQQMS4uwm5aBhKBfk8tBqFHk3DK/Rx+PAhPvv0I0pKStDr9YwcNYbu3XvIqboSiUQikUgk1YmwUbz6I3J2rqbEpCFLTSBFZ2Vc/VAye13Bi9+spW2D9kSdicnunnoPGo2GH35Yy5IlX9OlSxe6NQph2QcrmDHjYdq1a8f+fft4/vlneW3WbGw2G4sXL6Jbt258tXghNpuNxMRE7p56H7GxcQBcf/1kZsyYxtVXX0Pbtu0q7bJqs7F//36uvfZ67rjzv/z++3aef+5p3pr3Ljqdffhp5synUVWVrxYv4qvFi7jt9jsoLCxk+fJlvPjSK6Sm1mPd2jW8/PILvD57LmknT7Jlyy88/cxz1XeszzPkQJ9EIpFIJHUMtSif4tUfU7x+gctKukq4gZChk8lvczHB9RpUOMjnLYmJSej1ehISEpl4xVXExydU2aZEIpFIJBKJxD1qcQFFaz6BvCws2/eC3kasVqAguOTa2yhoM4BWqfVI/Olvjhw5TIcOHV22HzhwEO+8PY/8/DwyMjLIzs6mXTv7IF3zFi2Ii4vj0MGDWK1WcnKyWbNmtXPK7PETJ1i5cgXXXns9AAcPHiQqykBmVqZXfY+JjSU2NtbZp65du2G1WsnIyCAlJcXZTqPRcOmQodw15Q5uu/0O/vpzB+Hh4dSv3wBVVenQsROfff4pGRkZ/PPvPxiNRu6acicAp0/ncOytY+Tk5HDZZcOrdrDPU+RAnxcIIVBV1fm7qqrlymqCQPj11Ya37Str56nel3Jvy2oCqUXt0CJQPqUWVUdqUTNaCHMJJT8upGT1x4ji/LMVQaGEDJ5EyOBrITiMvIyMKvlMT08jISERIQQhISHccsvtRMfEoNVqfT6GvrarK1p4Q01fFzX9LDyfUFXV+VNbz+O6HPP4YycQWvhTJ7Xwv73UovrtSC3Kcz5pISxmTJu+onjVh4iC0ygi0d4wKJT4wZPo8Ntx/jW0oL5WR3p6GkbjKVJT65Gfn09OTg7x8facdlu3/EJ4eDhhYeHExqrk5GRz9OgR6tdvQFpaGunpaQSHBNGhZUfee/9DhBBkZmbw5ZcLaNK4CSNHjUZVVbb/to0/dvzOq6+9ztNPzWTTpo307dvPzX4I5zO0QYOGhIaGcujgQRo1bsy+fXsRQhAbG0tWVhZ6vd6Z+mXz5o00atQYVVVJSEzk0KGDZGVlYTAYOHDgAKrNRmxsLEOHDmPo0GFOf088/igjR42mR4+eAXt214XrwhcbcqDPDXPnzmXu3LnYbDYAcnNzMRqNzuWMVVUlNzcXIUSNLh0eCL++2vC2fWXtPNX7Uu5tWU0gtagdWgTKp9Si6kgtqlkLmxXljzVoNixAKch2FgutDtFtOOrFV1IQHk1BfhFqboHfPq1WK7/8vJkdO35n8KVDSUmp57STlZVV4bYXjBY+UNPXRX5+vsc6iStlY72MjAxKSkpq9Xlcl2Mef+wEQgt/6qQW/reXWlS/HalFec4LLWxWtLs2oVn/GUqukU/zDPxtSiJP1fB6QSrBioEne4xjQtMMPv3kY/LyctHqdFx19TXYbDaOHTvKO2/PQ1VVNBoN4RHhXHvtDWRkZKDRaJg06VpefukFFEWDxWImJSWFb5d8w6RrricoKMjZn5KSEvIL8jEajWRnZzNv3lvcdfdUiouLmXzjzcx67X8YDAYSE5N4c+4bnDhxHIB7pt5FYmIiU++5j9zcXCZNupY35ryOxWxBp9Nxyy23kZOTw+HDh/ji88/ODFgJ4uMTuOba6zEajURERDJ06GU8/tgjaLQaEHDjjbdw+vTpcsfUYrGQe9o+RhNwLWrxdeFLnCcH+twwZcoUpkyZQl5eHgaD4czJnOginqIoJCQk1PhNrKp+fbXhbfvK2nmq96Xc27KaQGpRO7QIlE+pRdWRWlSPFsJmxfzrSoq//wA188TZCkVDUM/hhI74D9q4FJdt/PWZlnaSr776EuOpUwAUFxcTHR0ttagCNX1dhISE+O3jQqNsrJeQkEBUVFStPo/rcszjj51AaOFPndTC//ZSi+q3I7UoT13Wwmazodn3G0GLPkNNO+gsvy4qF333nhT1nkhCyw7O7RITE3nm2efJyMhwsRcfH8+Mhx910aJ0mxEjR3HZ8BFs2rSBH9atQVVVhFDRajUkJiY6/T7wwDTn/xMTE3n3vfkef5/55NMe97VFixb06NGzXH1iYqLbcgdXXT2Jq66eVK7/ZXnu+Rfd+j3frwtf4jw50OcFiqKg0WhchHFXdq76Ut02vG1fWTtP9b6USy2kFmUJlE+pRdWRWgROC2GzYtq6gqKV810H+ICgzgMJG3MnupSmfu9raVRV5afNG1mzZhU2m43w8AjGT5hIy5atnV+zX8haVJWavC5q+jl4PlH62Nbm87guxzz+2AmEFv7USS38by+1qH47Uovy1EUtLAf+pPCbN9Ad+JPSEzL1bXsRPvYuNPVaUOwhDvNVi+zsbBYvWsCRI4cBaNeuA2PHjScsrOIF2fyhLmpRlbqavC582V4O9EkkEolEUkuoaIBP37oHYWPuRN+kfcD85eTk8NXiLzl0yP4WuXWbtowbN5GIiIgz0yokEolEIpFIJIHCevIARd++ifmvjS7lusbtCBt7F0GtugMEJA4TQvD79t9YtuxbTCYTwcHBjBp9OV26dENRlCrbl9Re5ECfRCKRSCTnGKvFwt4V3xL188eE5KW51Onb9CRs5H/QN+sUcL852VkcPnyIoKAgRowcTffuPWTgJ5FIJBKJRBJgzKeOkvb1+0TsWo0izg7iibh6RI6/m+Aug6olBtu5629MJhONGjVm4hVXExsbG3AfktqHHOiTSCQSieQcIcwl7PhqIcE/fUGiLdOlLq9BNxpcdWfAB/gciZoBmjZrzugxY2nevAVxcfEB9SORSCQSiURyISOEwPLvVk5//RGak78TiXDWZWtjyO9/I/X6DyAoOSWgg3yOrwEVRWH8uIns+PMP+vTpJ1N8XEDIgT6JRCKRSGoYtbiAko2LyV31GQ2KT7vUbdO156OQsezMb8lrxfUYEkC/+/fv47ulS7hh8k3Ogb2ePXsH0INEIpFIJBLJhY0wl2DauoLi9V9iSztI6Vep+UoYnwWP4pvgIZh2BPNEPRsTkgPj12w2s3rVSkwmE/0vHghARGQk/fpdHBgHkjqDHOiTSCQSiaSGUPOyKV6/gJINixDFBehL1W3RdeSTkDHs1LUEQAFeWHaKQW0j0Wqq9pbXYrGwetVKfv55MwDr1q3hyisnVcmmRCKRSCQSieQstux0SjYsouSnbxGFuS51p5RYvgkewndBAynQ2BfBUIC5G4oY20tQ1Y/tjMZTfPH5J2RmZgDQunVblxV1JRcWcqBPIpFIJJJqxpZ5gqL1X1Ly0xKwmJzlKgo/6i/i8+BR7NM1dtlGAOm5VrYfLqJHU/9XRTt58iSLFn6B0XgKgB49ezN8+Ei/7UkkEolEIpFI7AghsBz4k+L1CzD/sR5Um0v9X9oWfBU8jE36btgUreu2QEaB4PfDxfRsHuGXf5vNxoYN61n/w1pUVSUyMpLx468gyhDt5x5JzgfkQJ9EIpFIJNWAEALLwb/QrPiA3N1boFTiZbQ6jC2GcN+JgRzXplRoJyPP6pd/VVXZvHkja9eswmazERERwfgJV9CqVRu/7EkkEolEIpFI7AirBeWv9eRtX4nt6L+ulVodwd2H8lu9Udz9Q+UDeBn5/sV62dlZLFq4gKNHjwDQrl0Hxo4bT0hIKEaj0S+bkvMDOdAnkUgkEkkAETYrpj82ULz2c6yHd+IyEyMohJB+4wi99FoO5kRw/N2jldpLiPLvUb39t22s+n4FAG3atmPc2AmER/j3tlgikUgkEolEYk/DUrL5a4o3LEabl0Xp7/eUyFhC+o8n9OIJaAzxhB8shB+8iPUifY/1VFXlow/fJzMzk+DgYPpffAkXXzwArVbrXIxDcuEiB/q8QAjhcrGoqlqurCYIhF9fbXjbvrJ2nup9Kfe2rCaQWtQOLQLlU2pRdaQWYCvMR/llCad/W47ITndpr0TFETLgSoL7jUMTYQCgS6QgKUqHMc9aag22UtsASQYdXRqG+HRcHX3r1LkLf/21g85dutK1a3cURfHLTl3U4kK9LmRg7z+qqjp/aut5XJdjHn/sBEILf+qkFv63l1pUvx2pRXlqSgvrsb2U/LgA829rwGp2qdPWb0nIJVcT1O1SFH2w016XhiEkRek45WF2hgLERyh0bhjs8/MCYMSI0Wzc+CNjx03EarU625/vWvjbvq5fF77YkAN9bpg7dy5z587FZrOPz+fm5mI0Gp3LUauqSm5uLkKIGl2iOhB+fbXhbfvK2nmq96Xc27KaQGpRO7QIlE+pRdW5oLXIPIZm23KUv9ajNZe4DNqpiY0p6DSEkO7DsAQFk19kgqKzUynu7B/MzOXugz8B3NEvmKwzSZUro6SkmD9+3073i3pSUFCAEIJRo8eiKAoZGd7ZKE2d1OICvy7y8/P99nGhUTbWy8jIoKSkpFafx3U55vHHTiC08KdOauF/e6lF9duRWpSnWrVQbSh7fkWzdSnKkZ0u7YWiwdy0K5q+47E2bo9JUSAnt5zd2/sG8dRKz7He5G6CrMwMr7Q4fPgQFrOZ+IREhBAYomMYNXosFovl/NciAO3r+nXhS5wnB/rcMGXKFKZMmUJeXh4GgwGDwUBiYqKLeIqikJCQUOMXTlX9+mrD2/aVtfNU70u5t2U1gdSidmgRKJ9Si6pzoWlhs5jR/PMTwStWY9u/o5w9fdvehAy+Fk2LrtgyMz36nZAIBkM+LywzurztTTboeGhkIpe2i6z0GADs37+Pr79aRH5+HmHh4XTs2PmC0UJeF2cJCQnx28eFRtlYLyEhgaioqFp9HtflmMcfO4HQwp86qYX/7aUW1W9HalGe6tCCkkJMPy/FtHERalaaSzslNILgPpej7z+BbFVXqd8J8SqKcoJ5m83lYr0HhyfQKbG4Ui2sViurVq3g161bCAkJ4aqrr61wfOJ80kJeF2fxJc6TA31eoCgKGo3GRRh3ZeeqL9Vtw9v2lbXzVO9LudRCalGWQPmUWlSdC0EL2+kMSjZ/Q8nmb9DlZrrkZSEoFLXDAGKGT0Zfrzlw9mFfkd+hHQwMbhfF9sNFZORZSYjS0a1xGFqNUum+WywWVn2/gl9++QmA+PgE2rZtd0FoUVn5hXhd1PRz8Hyi9LGtzedxXY55/LETCC38qZNa+N9ealH9dqQW5QmYFtknKf7xY0xbl4Op2KVOm9SIkEuuIqTnSJSQMHuMd2bWX2V+L24RzLje9fnjaIlLrKcgMBpLKtQiLe0kixd9SeaZGR5dunYjLCz8/NdCXhfl8GV7OdAnkUgkEklFqDbMf2/C/Mt3mP/eDKrL8B7a5MaEXDwRfY/hZOYXoU1M9NmFVqPQo2m4T9ucPHmCRQsXYDSeAqBnrz5cdtkIdDqdXGlNIpFIJBKJxAuEEFj+3Urxus/R/fMLpjL1+ra9CB00CX2bXihVGKhxF+upqrsszXZsNhvbft3Cr79uQVVVoqIMTJh4JU2bNpNxnqRS5ECfRCKRSCRusBmPUvzTt2h/XkZBQbZrpUaL2qoHhkuvJahNDxTlzEIX+UU10rcdO37n668WYbPZiIiMZMKEK2nZshVg/4pQIpFIJBKJROIZYSqmZOsKStZ/iS39kGtlUAghvUYSMvAqdClNarxvVquV+e+/w9GjRwDo0KEjYy4fT1hYmIzzJF4hB/okEolEIjmDMJdQsmM9JT9/i3XfH4B9RTQHGkM8wX3GENx3LJkWBX1iIopS+RTbQFO/fgM0Gg2tW7fh8rETCA/37WtAiUQikUgkkgsNIQS2o7sx/baakp++RRS7Lm4gDAmEDbqa0L5j0YRHnaNegk6no379BqSnpzF6zDi6dOl6TuJNSd1FDvRJJBKJ5IJGqDYs//6KZtMScvZsgRLXr/KERktQh36E9h2Lvm0vFK3O/ja1BqdNCCE4efIE9erVB+y5+O66+17i4uJl4CeRSCQSiUTiAWGzYjnwJ+Yd6zH/uQE1O71cG13zLoQMvJLTKW0ITU45JzlvCwsKsFitREdHA3DpkGG0bNWaZs2ay1hP4jNyoE8ikUgkFxyON7ol277HtG0VIi+LsiGdNqkRQb1Hk9+sB5FNW52zhQ6KigpZ8s3X/PPPTm655TaaNG0G2Af7JBKJRCKRSCSuCHMJ5n+3Yv7zR8x/bUIU5pZvpNMT3G0ooYOuRtewdY2/xC3Nnj27WfLNYuLi4rn1P3eg0WjQ6/VERp67rwoldRs50CeRSCSSCwab8Sglv6/F9Ov32E4dKVevhIQT1HUwIX3GoGvaESEE+ecw4fG+vXv46quF5Ofno9FoMGYYnQN9EolEIpFIJBI7alE+5n9+xrzjR8z//FJu1VwAtDr0rboT1PkSgjsNRBMVW/MdLYXZbGb9+rXs/PsvAMLCwikoyCcqynBO+yWp+8iBPolEIpGctwghsJ08gOn3H9D+toZc4+HyjbQ69O37UtKyNzG9LmNHOmTkWUk4VESXhiE13mcAi8XCqu9X8MsvPwGQkJDIFVde7Zy6K5FIJBKJRALww7q1zJ37BtMemkHPnr3IPX2a12fP4uSJE4SEBPOf2+6kXbt2AOzft4/589+joKAAnV7HTTfdQocOHZ22Vn2/ku++W0pQcBAaRcMLL75MUFCQV/348ssvWLxoIS+9/CrNmtlfSp48eZI5b8wiLy+fsLAw7rr7Hho2bAjAH7//zhdffIrFaiU4KJg77vgvjZvYF74QQrDwywVs2rTR/mVbVCRPPfVsOZ9qbiYlO35Es201pw//DTZr+Y4FhxLUrg9BnQYS1KEfIjic7YeLyDhoJSGqkG6Nw9Bqan5q7LFjR1m0cAFZWZkA9O3bnyFDL0Ov19d4XyTnH3KgTyKRSCTnFUIIbMf2YPrjB8y/r8NmPAq4LqoBoGvRhZAewwnqMhhCI1j983Heev0kp/LOBolJUTru7B/MhMSa6/+JE8dZtHABGRn2Lwl79erDsMtGeB1oSyQSiUQiuTAwGk+xdu1qWrZs5Sz79NOPadmiJbfddgd5ebm8/NKLvDXvHbRaLS+++DxT7rqb5OQUrFYLTz81k9lvvElwcDC//rqVTZs28uC06TRq1Ij8/Hy0Wq1X/di3by8H9u8nNjbOpfzteW9y6ZBhDBo0mF9+/ok5b7zOSy//j4KCAmbNepWnn3mOhg0b8s8/u5g161Vmvf4GAMuXL+PIkcO8Nms2er2enJwcp02b8RimHT9i/nM91kM7QYhy6VeUiGiCOl5McOeB6FtdhBJkf3G7Zmcezy/bz6ncUrGeQceMUUkMbhvhw5H3H5vNxoYff2D9+nWoqkp4eARXXHk1LVq0rBH/kgsDOdAnkUgkkjqPsJgw7/8D886fMP+9GTXrpNt22ibtCe4yiOBuQ9DGJjvLV/+dy8zlheXaG/OszFxuxWDIZ2iHmplGkZ6eRkaGkcjISCZMuJIWpYJ3iUQikUgkEgBVVXnzzTnccuttfPThB87yn3/+idlvvInFYqF58xbExsaya9dOmjRpSl5eLh07dsJoNJKaWo+w8HD++ON3evXqzbdLvuGKK64iNDQUAIPBHveYTCZmTJ/GpUOGctllw9mzZzevvfo/nn/hJWf9e+++w/89MI1HHp7u7Efu6dMcOLCfx594EoBevfvw3nvvkJaWRmFhAZGRkc6v+9q2bUdmZgYHDxygabNmfLvkG5586mn0ej1CCCILjBRu/hLzjh+xnTzg9nhoYpIJ6jyQ4M4D0TXrhKJ1HepYszOP+z47gSiznTHXyn2fneDVa1LpWAPpj4UQ7PpnJ6qq0qFDJ3r17us8DhJJoJADfRKJRCKpk9hOGzH9tQnN7+vJOfQnmEvKN1I06Jp3JqjzJeTVb09s87blFtWwqYIXlrnPw+cIBl9cbmRwu6hqm9qhqqqzX127dqe4uJiuXbsRFhZeLf4kEolEcmHw/nvvsG3bNjIyjLzyv9do0qQpYP8S7PVZr5Kf7zql0mw28+qrr3D82DGCgoIwGAzc+p/by33Z9ffff/HUk08wefJNjBo9ptJ+vPLKS+z+9x9yc3P5+JPPCA8/+/XUvr17mTv3DVTVRlxcPFPvuY+4uDiys7OZ88brnM49jV6vJyUllf/85/Zytn/55Sc+/eRj59TVC4Xvln5L69ZtaNasubMsPz8Pq9VKTEwMxjM5hhMSE8nMzKRTp87ExMTy888/0bx5C/bv38fJEyec7Y4fP8aBA/v5/PNPARh4ySWMHDma4OBg7v+/B3ji8cdo3rw5s2a9ytR77sVgMGA0Gvnkk48YNuwy4uPjXfqXmZVJTEyM89xRFIX4+AQyMzNo2rQZ+fn57N79L61bt2Hbr1spLi7GmGEkOSWF3NzTbFnxNb9s3YoozGdwcDYXhZSP87QpTdF3GkBBw44kdOzt8QtEmyp4ftmpcoN8YI/1FOyx3sc3VM9XfUIIZ6yn0+m48spJpKWdpGPHzs7jL5EEEjnQJ5FIJJI6gbCYsBz4C8vuXzHv+hnb8b0A5aZroNWhb9mNoC6DCO40AE1UHKqqkuchkNp+uMhluq470nOtbD9cRI+mgR14E0Lwx+/b2bjxR267/b+EhYWhKAr9+l0cUD8SiUQiuTDp1bsvY8eO55FHZriUf/H5Z1w6ZCiDB1/qMqUSYMiQoXTt2g1FUVixYjnz3prLlLumOrctLCzk008+pkvXbl73Y+jQYYwbN57pDz3oUq6qKq/Pfo2rr7qGfv37893Sb/lg/ns88OBDaDQaLhs+kj59+qDRaPjoow/45JOPuPLKq53bG41Gftq8mRYtL6xpj0ePHmHLll94+pnnfNpu+vSH+fiTj1i08EuaNGlCmzZt0WrtkZTNZsNoPMV99z9AWFgYTzzxKElJyXTvfhGpqfUYO248jzw8nauvvoa2bduhqir//vsPmRkZ3HbbHfZVa70kPDycBx6cxmeffkJJSQktW7Wifv36qMf2kLtrFTabjbwNX/NQRAGZEVpeyo4nWWulgd6KrkkH55d72sSGqKpKgdGIonh+Gbv9cJHLdN2yCOyx3t8nraQke2zmFwUFBSxftpTGjRtz6ZBhACQlJZOUlOzTMZNIfEEO9EkkEomkViJUG7ZjezHv/hXLnm1Y9u8Ai8ltWyUylqD2fQlq3w99mx5oQr1/I5tRySCfr+28pbCwkG+XfMWuXTsB2PLLTwwaPCSgPiQSiURyYeNYhKE0ubmnOXr0CBdfPABwnVKZkpJCt27dnW1btmzJ0m+/cdn+vffeYeLEK9iydYuzzNP0zhdfegWDweCcLlqWgwcOoNVoadnKnqZi6LBhfPHFZ5jNZqKjo2ne/OzXai1btGTFiuXO31VVZd5bc7nyqqv5bum3fh6husm///6L0Wjkril3AnD6dA7H3jrGVVdPQqvVuuS0yzAanV/bNW7ShEcffRyj0UhiYiL33nM3DRrYp43GxyfQt19/NBoNUVFRdO3ajb1799C9+0UAHDt6lKgoA5lnFo8A2LtnDwcPHeSO2/8DCE6fzuH5557m9jv+S8sWLcnJycFms6HVahFCkJmZQXy8fX5shw4dade8KZadP1Pw+zo2njhK9Ort6HU2gpVkeobYV82N10OLuAjSOval47W3oo32fX6ttzFcdqG7b/78Z/fuf/j6q8UUFhZw/PhRevfuS3hEzeQClFzYyIE+iUQikdQKhGrDemwfyo5N5Kfvw7rvd0Rhrsf2ukZt0LXrS0FqGxI69UGr8++RlhDl3XbetvOGvXv38NVXCyk4k+h68KVD6d9/QMDsSyQSiUTiiczMLKKiDG6nVKakpLi0Xb58Gd0v6uH8/Zeff0KjKFzUo6fLQF9F0zsrIiMzg4SEswM3oaFhhIaGkZ2dTWLi2ZWwbDYbK1eu4KJSfflu6be0at2ahg0b+Xcg6jDDhl3G8OEjnL8//tgjjBw1mp49e7Fv317WrF7FwEsGsX//PrKzs2jXrj0AOdnZGKKjAVi7ZjUhIcHOVXf797+YHTv+YNiw4ZhMJnbt3MnYceMB2P7bNv79dxevvvY6Tz81k582b6J3n75cPnYc/7ntdjQaDaqqcvtttzJ9xiPOVXebNm3Ghg0/2hfj2LSB2Igwov/dQP73/5B54B8is4+CECwpiKRVkI5EnQ2Ai8LM7EnsQJMx4zA17szhxx5n4tAJfg3ygfcxXGx4YFK0mM1mViz/jm3btgIQFxfH1ZOuk4N8khrjvB/oO3bsGNdffz1GoxGdTsdjjz3GFVdcca67JZFIJBc8wlyC9fAuLAf+xLL/D6wH/0aUFKIFLG7aa2KS0Lfugb51D4JadUdjiD87XUNTbgKv13RrHEZSlK7C6bvJBh3dGof57cOB2Wzm++9XsHXLzwAkJCRy5VWTSE2tV2XbEsmFiIzzJJLq46vFi0hPS+PxJ54kNzeXnJwcFi9exFNPP+O2vbvpnYFACMG777xNeHg4I0aOIjMz0zl19cmnniE7Ozsgfs4Xrr9+Mq/PepUff1xPcHAwU++5D92Zl6Fr1qxm48YfsVptNGzUiGkPzXBOeR095nLmvTWXp5+aiU6no1fv3vTp05eMjAzeffdt7vzvXURGRvLAg9N4/LFHaNykCVqt++EEYbVgO7Gfyd2a8fZn77P43TcIVs3cGJlD4aLfAFiSZ2CfOR4VhaZ6M5OTVYK7jCKo00Buqd+WuW+/zY+frwRWMm7c+CqtStutcRhJBh3GXKvbPH0K9tV3O6RWfXjk2LGjLFr4BVlZWQD06duPTp26kpycUsmWEkngOO8H+nQ6HbNmzaJz586kp6fTrVs3RowYQXi4THAukUgkNYVQVdSMY1iO/IP18C6sh//BevRfsHkeXFNCI9G36m4f2GvTA01Cgwrzr/iLVqMwfVQi931+EgVcAkDH7w+NTAzIQhzr1q52DvL17t2XYZeNQK/XV9muRHKhIuM8icR34uPjyMvLxWazodFoyk2pBPh2yTds3foLT8x8iuDgYAAOHjxATk42/3f/fYB94Yfftv1Kbl4u1157PeB+emdFJMQnkJGR4fy9uLiIoqJCYmNjnWXz579HZlYmDz00w7lwlWPq6tS7/4vNppKfn8e8t94kJyeHyy4bXrUDVAd56ulnnf+Pjo7mscdnOqfnll6E7MqrrmbiFVe6rQsKCuKuu+8pV5eQkMC8t99zTr1OTk7hnXfno6oqRqPRPiMj7SCWw7t4tnd99IueIuvEPrBaMADTgoCg8n2+PqYQXf1UdE07EtRpIPrmnZ0r5QYDMx5+NGDHR6tRmDEqifs+O+E21gNHrFdcJT/FxcV8MP9dTCYTBoOBCROvokmTpnLBDUmNc94P9KWkpDg/QU9OTiY+Pp7s7GwZAEokEomX2FTB9sNFZORZSYiyf9lW0aCXEAI15xTWo7uxOgb2jvyLKM6v0I8SFYeuWSdKkpoR06kv+oatUTTuV08LNJe2i2TmyHDe2mRy+bIvyaDjjn7BXNouMiB+Bl4yiMOHD3HpkGFVejMtkUjsyDhPIvEdgyGaBg0asnHjBgYPvpQtv/xMXFyc81pauvRbNm/exBMznyI8PMK5YEC3bt2Z/8HHTjtvvPE6TRo3ca666256Z99+/SvsS9NmzbDZrOzds4fExERWr1pF9+4XERQUhKqqLFy4gNzTp5k+4xH0er2zL46pq47BprlzZjNq9JgLatXdQFE6zouP0FI/zHOeOqGq2DKOYTnyL5bDu9Ae+Iuc9INgtq+IqwFs7jbUaNGmNLWnXWnU1v5vanMUvZsRwGpiSPsoXrsWnl92ymVhjiSDjumjkhjcNgKjsWoDfaGhoQwdNpwjhw8z5vKxhIaGyQU3JOeEWj/Qt3HjRl5++WW2b99OWloa33zzDWPHjnVpM3fuXF5++WXS09Pp1KkTb7zxBj169Chna/v27dhsNho0aFBDvZdIJJK6zZqdeW4DohmjkhjSPgphMWNLP4Tl6G40+/4iL/s4thP7EUV5ldrWJjZE17wL+uad0Dfvgia+HkIIio1GlPgEth0u8XpwMRD0bx7E2F71+OPoWb9dGoaQlZlR+cYeyM7OYuuWnxk1+nLAnnvojjvvqpYvEyWSuoiM8ySS6mXeW2+yfftvnD6dw9NPzSQ0NJQ35rzFpGuu5YsvPuObr78iLCzUuapuVmYmH304n6SkZJ543P5FlU6n4777H6jQj6fpnU2aNiM1NZXnnnuGQwcPAHDvPXeTkpLKU08/i0aj4e6p9/Lm3DksXPgFsbFxTL3nXgB27/6XDT+up169es7VehMSE7nxxpur52BdgLiL8xIiFB4eE8aQ9lGo2elYj/yD5fA/aPbv4HT6QURxgbOtp2hGm9So1KBeW3QNWqEEhbi0samC7QcLazTWG9I+ikFtI92+wPZnQE4IwbZft5KckkKjRo0B6NmzN7169QlwzyUS36j1A32FhYV06tSJm2++mfHjx5er//LLL7n//vuZN28ePXv2ZNasWQwbNow9Z94KOcjOzuaGG27g3Xff9ejLZDJhMp1d0TEvz/6HqhDC5cJXVbVcmb/YVMHvh4vJyLeSEKmja+NQjze4QPj11Ya37Str56nel3Jvy2oCqUXt0CJQPn21Y7Ha+OOYGevJXBKj9B6v20Br4c6vgqg2Ldbuyuf+z08igGBhoqEtjYZqGg3T0zj9zkmORRoJPX0MVPu7Ww3gaSKuEhVnD/Qat0PXqA3ahm3QhLsm6Hb0eeM+E/M+OOj6ZV2UjumjEst9WRfo60JB0L1xaKXbVXYNCCH4/fffWLH8O8xmM6n16tGt20Uu+1pdnKvrQt6jylPTWtTFrwZqMs4Dz7GeqqrOn0CexxarjR1Hi5xxXueGwX6fx3U55vHHTiC08KfufNPittvvcGsnMTGJZ5553mXqpqqqxMTGsmjxN+XaZ2RklPM7Zcrdzvq4uDjefOsdZ7vExCTmvf2es3769IfJyLAvvOHw6bDXokVLHn7ksXJ1LVu2Ys7ceeXKS/fFcUyemPmUc0GIQFDZ32jVcV1483dhIK+L1X/n8cCCNOc01lj1NK1th2idcRDzO4dI1x9FX3zauZ0G3Oa308SmoGvUFk3D1hREpRDXoSfa8CiXNgL714AO1u7K54VlRq9ivcrwVQsFXGI9EKiq8NlOXl4uy75bwuHDh4iJiWXKXfc4p7mXjvPkPar67VwozwtfbNT6gb7hw4czfLjnPAuvvvoq//nPf7jpppsAmDdvHsuXL2f+/PlMnz4dsAd1Y8eOZfr06fTp43l0/fnnn+fJJ58sV56bm4vRaHR5wOTm5iKEcHk4+sqm/Wbmbigio+DsjSAhQmHKgDD6Ny//GXMg/Ppqw9v2lbXzVO9LubdlNYHUonZoYbHa2LIvD9MeE/ERWjqk6vx6E+hL3zftNzNnQxGZBQIoBDxft4HUwpPfO/uH0jGhKDBaCBXysyE7DZFxnJMbD/BiyUka2tJIEW5y7XjIfS0iYhHJjSGpKaJeC0S9lhAZh6X0V2yFJigsn69kw14TT60sKld+Ks/KfZ+fZObIcJfjXBuvi6KiQn5cv46DZ75cSExMIjIyqsbyswTqWpT3qKpT01rk51c8Pb42UpNxHniO9TIyMigpKQnoebzqr1w+3J575r5tJz5C4cZugmEdfT+P63LM44+dQGjhT53Uwv/21aFFdk4uO45ZyCm2r4jaLllDQX5etWrhzd9ogdZi4z4zb20qrvTvQr+1EAIKT0NuBiLnFMVpRzH+mcUzpkyS1CySRSaRokz85ebtrYiIQdRriUhtgZrcjNyIRKKS67tooeYXoSks8di3TfvNzFxeWK7cU6xXGefiujh48AA/rFtNcXExGo2Wdu06cPr0abczNuQ9qvrtXCjPC1/ivFo/0FcRZrOZ7du3M2PGDGeZRqPh0ksv5ZdffgHso+k33ngjgwYN4vrrr6/Q3owZM7j//vudv+fl5dGgQQMMBoNLQlJVVVEUxeXtkq+s3ZXPk8tzyr0VySwQPLm8kFevMbj9cqWqfh02YuPi2XHUVOmXhN76rKydqqqoAo4URpBVoDp9Kgi327mz521ZWXz5atJbAqmFtzYCqYW3x9xTubsyi9XGnyesWItDK/zKLVCcfRMIYM+nUZU3gd4cW1+v20BpUZHfp1cW8fiIMMa38O4epRbloWaeQM08iS3rJGrmSdSsM//PTgPr2fVuR1Vy3KxoscU3IrJJSzT1WlAQkUBs2+7oDHGVbOkemyp4+4ODHusVYN5mE2N71XOeW7Xtujhy5DA/rl9LQUEBWq2WSwZdSsuWrUlKSqrRwaWq3p/8sVPbtPDneRFoalqLkJAQj3V1kUDHeeA51ktISCAqKipg5/Hqv/N4ZUMuZb9/ySoQvLIBUlLCGNrh7Bcv3pyzvpxPgYx9AuW3Ou8p7mJMx1Q8j8/DKtxTzsX9pKI+V7edio6xt3b9qVv9dx7PL8sjs+DsoFBSlI47+p2NewKthbexXiC12HzAwtMri7yKLz35FTYr6mkjanY6anY6tqw0dCcPE1R82l6WcwosZ79m1gMjKunbaSWCoEZtiG3dHm2D1pyOSCShaWuX64JSX2l6+3fZWz7GepVRk9eFyWRi5YplbN++DYC4uHiuuvoaUlJS/bIr71GBsROIZ3dd0MKXOK9OD/RlZmZis9lISkpyKU9KSmL37t0A/PTTT3z55Zd07NiRJUuWAPDJJ5/QoUOHcvaCg4Odn9tWJzZV8MIyo9tPnwX2G9yLy41c0iaiWgZKNu03M+/DwwH5VNpb1u4qOPOgznXxOW1EAp0SK9iwyn4D91m4r1THAKO3fncct2BNy6+RATcoe5ztwVh1HufSU0pLY8yzcv/nJ3n1mtSA+z1X1603ft/cWMyYHiq64nzU0xnYsk+hHD9Isa0YcjPtgd/pDNScdJe8Kt6STxhHtSkc1aZyRJPCUW0KRzSppGkSeG58A0Z0sv9xnJ+RgSYyxu99/f1wscu16m5/03Ot/H64mIuahvntp7r4cf061q9fB9hzCE2ceBXJySkuqwlKJBLvCXScBzUT69lUwUsr3F/3jnv5SyszGNwuslqez+cq9jl3ft3HmNNHJTKoTfUtymJTBdsOFVU48HW++K3oGFff3w75LtNKHRjzrDy5wkpUVIHLYHkg8CXWC9QRt6mCuRuLK/U5sJkO5fQprFknUY7so9hSiMg5hZqdZh/IO51hn5lRCg2eU6qUxooWoyaWNE0Ce7WN2a1tym5tE9I18bw4NNUZ5xGAeKYux3oFBfm8+848srOzUBSF3n360alTZ5KSks911ySSctTpgT5v6Nevn/3G5ANz585l7ty52Gz2vFOBnrq747jFqxvcuh0n6Vxf7ywPxGef52JaXEWfZ//fgjQeGIBfU1iq4tefz8K93V+H70B/8u9Ne2+mlfry6bGn8tJlPx20VttxdodNFTy3NNdjQATw3NI02sUV+fQmsLJj6891W5ndSo+5amPXoTxCck7SSeQTo+YRLfKIVvOIEXlEnymLyztN7v/loLWZnTZ0gKmcR88IfQjEJCFikiE6iWPaFF79M5aj2hSyFQN4WDxCZ8vHaPR+ultFHDhhrrwRcOBENo0i7AOWgbhHVVTvS3l0dAyKotChYyf69u2PTqfHaDSe99NFfW1fE1r4+ryoLmpai7o4dbeq+BPnQflYL5BTdyt7XgCc8uJ54c/0n+qIfQLlt29TXbXEPRX5fXx4WLn0FpXZ90aLDXtNzN1QRFbR2YGvitLvBIpA+fXl3uTLOVWRXV/qvIn1XliWToeEYhREwO7tvsR6HVO1Vfdrs/D3/hzCcjNoJwqJEEVEikIMIp9k1T6lNknNJCk3i7z/O3t/9z3GCwZDIiI6AQyJqFEJ7CkK581d8ZzSxJOlRKMq7vehojjPn3uUP7FeZdTUs1YIgcEQjdlsZsjQy0hNrUdubi4ajdHv54U/dYHSojqoyzFoXdDigpm6Gx8fj1ar5dSpUy7lp06dIjnZ/5H1KVOmMGXKFPLy8jAYDAGfumtNywMqv3FZtZEkJlY8rcMXzsW0OG8+z/5ou8I1gxLQ67QV2vNlKlZ1fBbuzf5C9X3yX1n7qvr155Pk2Lh43vrosMc+V/U4u2PbwSIyCk5X2CajQHC8KNLrN4HeaOHPdeuwGx8Xi1JShCjKQxTnI4ryUYvyUQtz0WakE6ZRobjgTHkean42+rxslKI8uqg2PvZqLypBq0MTnYAmrh7a+FQ0cfXQxKeijUtFE18PJSLaJa9IjCpIO36QHA8Br4J99d3BnVMrnR7lLc0KinAMUFfYrl4siYl2bQPxqX5F9RWVCyGwWi3Ur29f4TM+Pp6IyEiaN28RsOeFP9TlaRMV1Vdl2oQ/+xMIalqL823qbnXFeVA+1gvk1N2qPC+qMv2numKfQPkd0yMloPcUb/y+/bOZj693jeMrs1+ZFj/8W8jTK31LvxMI1u7KD5hfb89zX8+pQE2L8yXW69Y4JGD3dt+u3Qh7jBdtQCkpRBTnoRbl22O9ovwzP46yUuXF9lhPFOWDuYQuwCdV6jUo4QY0scloYlPQxCajjU2GmCRylRDimrZGGxnjEuOpqor1lJGMtEIy86xuB1S9ifP8maLoT6xXGdX5rM3MzCA8PILQUPviHVdPugZF0RAaGhqQ54U/dYHSojqoyzFoXdDigpm6GxQURLdu3Vi3bh1jx44F7Ad23bp13HXXXQHzoygKGo3GRRh3Zd6SaNBX3uhMu7L2q+L3t8OFXr2l+uNoCT2anp3q4K1Pd+288ZlRINhx1ETP5hGV2vO2zN999QV3fm2q4IXllX/yP7hdFBqNxmc9PbUPlF9fyhVFYcdRU7Uf57JkFti8bufLdeLYRwUBFjPCYkKYS8BiQpQU0TAvm16Wo4SJEkJFif1fSsr8bqLj94K81WaEqQhRUoSuMJdcc7FHv1rcv5X1ZVi0gFCCYhMJT0pGE52AYkigUBuKoUFTdLHJaAwJ9oE8H46HRgMzRidx32cnUHDNMuXo2/RRSS4D9FW5PwF0bxpOUpTO4znlCDq7Nw1HU+qP1Krco7ypd1eenZXF118tJCsrk7vuvpf4+ATA/lVfIJ8X/hIon4G6R/naLlD3qAtNi5rcr5qgpuI8wOXYVvU89jfO8+acrahv1Rn7BMLvjqMmGkUE7p7ijd9TuVZ2ptlITXFvw9d7ikDxOt4KeBqPAPv15jz355yqyK63db7Get5es8JmRVhMYDbZ4zyLyR7nmUsQFhONM3MZaD5JEBaChZlgYTrzlZ39S7tIUUiEKKT1VxZyvyxAV5hHrtW7L9T8RUUhU4khKjUVQ0o9lNgkCvURGBq1QBefijYmGSWk/ICYqqrkGY3oomLdHhedVsP0UYnc//nJcnGeA2/iPF/uUXAm1jPoMOZWPMBYNtarjEA/axVF4ddft7ByxTLatm3HlVddA0BERKTb9lWJe/ypC4QW1UVdjkFruxa+bO/TQN/p06f55ptv2LRpE0eOHKGoqIiEhAS6dOnCsGHDKl3pzB8KCgrYv3+/8/dDhw6xY8cOYmNjadiwIffffz+TJ0+me/fu9OjRg1mzZlFYWOhcnS0QCFHxksm+0qVhCElROoyVvEHp0jAkoH6NuZbKG51p5/DhrU9P7bz2mWepdF+9LfPJb67Fr+Ppye9vB4s4lVt5QPTbwUK6NQ7xSc+KtAiEX0/2KzruxjzfjrMQwp4/xGaz/6vaEKr9X2w2hGq1LwZhsyBsVrBaETYL2OzlwmahUVohl5hPocOKXljRYkOPFZ2w2cuwoRNWWm4PJX+3sAdwpYM5i+nsQJ7z/yVoTSVkO3y5IRV40ZudPeqaD8WvMF8XhAiLQmeIQ4mIYd1RPWnWCHKUKHI0UZxWojitRJKjiSJXiSIqMphVDzZzBmOqas+Xpyv15kiA/Vj7wOC2Ebx6TWr5nEsGHQ+NTGRw2wif7xUVoQDTRiTwfwvSPA4uPjQyEQWBqgqf/FbWztvzXwjB9u3bWLliGWazmeDgYDIzM4mNjfPpHlWdBMqnr3ZqWouKyi9ULWpy3wJFbYjzwH7sHD9VPY8dcV5FAyRl4zxvztnK+lZdsU/A/OZZaBgeuHuKt36zClS3Nvy5p/x2qNDreCuQ+cV8ifO88evtee7rOWWz2RBWC7aSIlRHvGezgmrFZrEgMjOwWPJRUO3lNivCZkO1WiA7C9OpCBSh0uhkIYPMRrTY0AkbWmxnBt8sBGE+MwhnofUvQeT/bkVTkEeeBrCYXeO+MgN6qBUPICYD5dfkdkPW2QFWn9EHo4RFogmNRAmLRIRGsu4QZFrDyVfCyNeEk69EYFRiSdfEk6mJIT46hO8faOr8sq50jOcpvqtIY0fdoDbhbuO8hAiF6aOSK43zfL1HceaYTR/pfoDRU6xXGYF+1ubm5rJ06Tfs3WPPBZufn4/JZEKv17ttX5XnhT91gdKiOqjLMWhd0MIXG14N9J08eZLHH3+czz77jNTUVHr06EHnzp0JDQ0lOzub9evX88orr9CoUSOeeOIJrrrqKr87X5bffvuNSy65xPm7Y6W0yZMn8+GHH3LVVVeRkZHB448/Tnp6Op07d+b7778vl7jZF6o7Rx/Anf2DmbncSivrQVLV0olN7Te0K+qHkP3jHy7bCFWluKiIzLCwM59fi7KblbNz9ldBYraNoWaT/YulCtom/RtMxintWZ/FxWSGhqIoGrd27f8ISoqKyAwLPftpuIDkHBujTPbl1V0fhq52UnYGkXHy7OkoVJUSp1+HK2EvCwk50xcQonQ7xdmf1NM2xptMbh7AZ/0qQpD6ZzAZR7Qu++KWMnVCqJhKSs70RXGatWVYmVRS9s2eQEGgOdNIg4pYpScrVsFcUkJWUJB9H4U440ec/X+p34VQsTraIxwjNwAEn7YxrchsX8VYlPenAAoq4Qu1ZIeDarGQXeoNHUIFIVDNZrJ1urN9cAzCWcxkaxQUoYKqIlQbitVKZ5vgs3wrWlQ0QrX/e+ZHi4pW2P8f9J5KllDt21eRZGCmNw1/8y1/SaDTWAt9CASFYAsKRRMWBaER9p/gcAiNQIREIELCKbRBWFwiSlgUhNjbqLpgcs9MJ9NoNKj7zbzlJj+Og8ndBFmZGQG9RznomACfTI7g75NWsgsFseEKHVJ1aDXFGI1nv1QMlM+OCSoPDIAPtytnck3aiT+Tg6hjgn9+K2vnqb50uamkhHU/rOHQwQMAJCYlM2zYCKKjozEajW5tBFILbwmUT1/t1KQWlZVfqFrUxRx95yLOg+rN0Qdwe98gnlppRSes9LdsL1c/oX4w2Rt2OH8Xqjgb4535isVeVngm7tMgROk4sLzPhGwrl5hLXMrcPdsS9oaQkelp38rHQvb4s/hMjKcp1zYx28aQUn7L+nTEnUm7gyk+bDobs7nzWfq/Qj0TW4aVii3PNkjOsTLSdPZJ7/Bz1rJAARrvg8y8EGc86fBhjyGLyAwJdakTQsXkjDcdWtjjPlu+jqtKLPZ4y8WPOBNr2f8vVunJiNeAONMvoZ7xK87uQ9m4zxHbUTYGBP1pK/cWWc7Edmd92WMugSLs/4Yt0pAZqZzxp561o9pKxZZnYjmLhWytglK63GUblU5mGx/l2c76QaAVNrSo6LCiw4ZWqIS8byNLtaEIFT1wNoOgK3rA011Kx9kJncnAEx7aufAXWPB+0YlAIfTBEBJhj/HCoyA0EkLCISQCERrhjOfO/h5+tkznmktRVVXy/8rljQ2e/d3RL5iszAxn+0DmIuuYoHGJ82JCoUF4IbExRRiNJRXaK1vmbd86JsATI8PL5TP3FOtVRiCftTt3/sXWLb9QUlKMVquld59+dO7clZycHL/9equFt3WB1CLQ1OUYtC5o4UucpwhR0eiGnaSkJCZPnsyNN95I27Zt3bYpLi5myZIlzJ49mwkTJvDAAw943+NaiiNvy549e2jatKmLeBmllhL3l7W78sn56FkGF64PVJclEom36PQo+mDQBaHog7FpdOhCQlGCQuxvW/VBZ/4NRgkJQwkOQwkOZW+OluX/Wkkv0VOshFCkhBAeEc71g+rRt30CSnAoBIWinLmxV3Sv8FTvrtzdaobJBh0PDk+gU2JxudwQgbhH+UKgfDrsxMbFs+OoqdJVq731W1UtcnKy+XbJVxQUFKDVahk0eAgtWrQiKSmpwuN+PmjhS36UmtDCm/ILVYu8vDxiYmLIzc0lKiqwq1GerzhivZycHGeOvkCdx1//coKPN+bw4fH/VOcuSCSSsuj0KLozcVxQ8Nl4Th8MQSFn/+8sC0bRBTnb7smE5btMZJRoKVDCKFDCCI6K4qahjRnQJQVFpw/4/f1PYygvrcgoF+c9NNJ1ZeNA3KP8qfPmuerrMbGpgt8PF1ca61VGILQwmUysWP4dv//+GwBJyclcccXVFa6oez5pESjqcgxaF7TwJc7z6ou+f/75h7i4uArbhIaGMmnSJCZNmkRWVpb3va0DVFeen6EdDOR1isL8cyB6KZHUABqtPXmbokEoGjRaHRahkGcCFQ02NKjKmX/RYkNDckwQEaF6FK0WlDPba7Uojv9rNKDR2fPH6YLs7XR6FK0etLoz/9eBVoei04NWj6LTs/uUlRW7isgu1mBVtFjREREexPge8XRpFmXfxhnMBbkGefogFI3r4i9Go5E4N8m6y9IFaG+1sW7HSXsyZoOebo3DPAYmgcpFNrSDgUvaRJTzqyAwGkuq5R7lK4HMyaHXacvl7ayq36pocfz4MQoKCkhMTOLKqyaRlJTs/NK7suNe17XwxY7M0VcemaOvblD62AbqPL64RTBjOzUm76GAd1ciqR4UDSiKPd5TFBSNFqtQKLIIbGe+6VPRYEWLTdFiRUuCIZiI8CAUjQ6h1WK1CfQhIWfiOK09HtNoQaOlxGIlJCwcjU5vL9PqULRahEZLUbGJ8Kgoe6ynsZfvPmVhxd+FnDZpsCg6TAQRFhHCZR3CGdClPprgUIQ2iKy8fOJT6qENDi0X4/lDV6CTKth+uIiMPCsJUTq3sV4g7+9DO0QxpIOhUp+++K3tucg0GryO9SqjqloIobJv3x4A+vW7mCFDL0Onq3yo5HzRIpDU5Ri0tmvhy/ZeDfRVNshX1fa1nerM8xPcdTC6lCaeG5Sa2iCEoKCgkIiIiLNTGFymPpR5EJSbO6EgVMGOQ6fZeMBGfsnZ/keGahnWPpLWqa4ruQghyC8oIDIy0oNPuyMhBPn5+URGRaJxM51k54kSVvyZS36pGSVRYVqGd4ikQZSFyMhIlxNXCJW8/AKiIiOdCwgIIcjLzy9flpdHVJTBbf92Hi9m6R/55BafzclhCNMypouB9vVD3O6L6yFzP3ijOvoSFYWmjN+/jhYzf5Pr592qcnZCxy0D4ujcONyeAyIvH0N0NJozAZV9LobG3g/F8aNBUUAVcPp0LtGxZ5LqKpoz7e1tthwo4oXlGYCCzTl55KzfR8cm07dVJEIVZGVnExcfb/ercUzshaysLOLiE9BotfZ912jtSYCzsohPTEKrPZsDrvSbib88fG320MhEulXDynMAnYC2Vhs//JlmH/iK0nv1JtDxCbOowjWtIOhUT0dCQsSZ89Z9HpHK7PqS78Gz3wszF5mv7f3Rwmaz2ZOvC8GAAZcQGhJCj5690ev1Xh93qYXv7Xy9LqQWru0k/qGqgcvRV7pe0esJm3CvS3lhYSHh4eHlvgooW+4si4hAoyioQlBYUOD83RN70k2s21VAfok99hGKQlSIhsFtI2mVElymdSVfz5y5BxYUFLjGn2623ZNewuqd+eSXqM6JrVGhGoa0s8eXQj1jp8KY8iwC+zSlyFJxnzvf/5woYcVfbmLMjlG0SQ0hv8ARO5Ya/DmzX45YTnHEVNhnrdrjzaiz8SaCvDx7250nTHy4KQe1VD8E9hhORWOP8xqF4grUDNgAALHBSURBVIzlzvyrlIvxKNfG+cOZ9s5yDVsPFvHq95nOybtCUc4kS7FP4p0+Ook+raLOxo7OWPHswJ2jXBU4YzuNVnu2vQdW/53H88vSXVJqOOK8zm6+Novx8EVMYUYGYR7q8jMyCC5T10FVSRhg5FhRBFkFKgmROjo3DCY7KxNNqa9lBBkQFoVw6BWAe6ECdG8cWqrENdarjvu7RqOp0Kcvfitq50+dN8/Vc/GcrYpfm+3sYi4hIaFMmHAluXl5dO7cxXluBcKv1KL67VwoWvhiw+dVdz/66CPi4+MZOXIkANOmTeOdd96hbdu2fPHFFzRq1MhXk7WOmsjR5ySuif3HCxx+bWfydvmDqqrEpuRy3bAodqWrZXJuKeR58KlW4rOydvWaqkxon8uxwnByinH6VBDkuNnOnT1vy0rToBHc0Vu4yS+meMwf4g0V+W2YCoPjzeXyTiScyTvRuHkQpx029JUfWxefliDUEPftW7aK5CZtDHM2FLkEYg6/rZsGkWUBVRXkqlqESaW0GVVVyTXZEAXF5bXIy7d/wefhGuiYAB9dH8aWfXmYlFDiI7Ru87gFGlVVaRxRiMGgQ6MxkZVZ4LcdmYusagTKZ23QAmDbtq0cOXyIceOvoKCgACEELVq2duZokVr4315eF9Vnpy7m6DtXVHeOPpf69oPLlZeN5dyVly3zNg5MaQ+TBrmPfcrGed7gtd+2cN1Az359jWO9jUFTm6mM71A+xtRqFHIrsOHJvsd4M8Je1rCBhgGxJuZuKCKr6Ky9snFeoGnRLo6r9Yke/bZqHkRWub8BHQWui1CoqkpuYTEiO8crLTomqMwdK9wcY+/z5vpTp6oq+Xm5NDJAk0h7eVZmnsxFdgHkIvMFf/zm5GSzetVKOnXqQus29tRk4RGRWG2qy9/9gfArtah+OxeKFr7EeT4P9D333HO89dZbAPzyyy/MnTuX1157jWXLlnHffffx9ddf+2qy1jFlyhSmTJnizNtiMBhILDWlT1VVFEU5J3Peq+q3tI16qd5dNN74rKydo751q/Lz2t1t567c2zJ3pHhOr+AXlfmdkAhje1Wcd8JXPb1pPyERxvSo+Cs3X465p3JPZf20mhrPfxWIa7E6tPCmXXVqUdP3qPNFC41G4euvFnH8+DEAMjONJCenSi3kdeEXNa1FSEiIxzqJK2VjvYSEBGeOvtp6Hvt6PgUq9gmU3+q+p5SNMSuzURUtJsSr9G3m+qWZv/nFfCFQfv3Vwt0x9tZudWlRl/8u88dOIO5RF7IWQgh+/XULq75fgcViYdu2rfTt1x+tViu1CADyuqheLXyJ83we6Dt27BjNmzcHYMmSJUyYMIHbbruNvn37MnDgQF/N1Qm8mXd9LvtS3TYCMd+9ovqqzHf3Z38CRWV+NZrK805UhxZ6HXRpEERiouc34OeTFoHyKa+LqlOXtRBCsGvX32zetAGLxUJISAhjxoyjQ8dObnPxebIntZDXRVlqUouafg6eT5Q+trX5PK6tMU912QmEFv7UeaOFTquhZ7OIGtciUH7PJy3kdeF7u7qoxfbtv/HF558hhMBms3H52HFccskgvv5qMevWrSUjw8iD06bTs2cv5zb79+1j/vz3nF9s169fj5ycbAAsVhuHDx/hoWkPoNNpuXrStaSm1qu0/45+qELFbDIzfsIEBg26FICvFi/ixx9/IC0tzaUviqJw8OABPvxgPiUlJSiKwo033Uy7du1RFIUFX3zO1l+3oNfp0em0TLrmOjp16lxrtagIeV1Unxa+bO/zQF9ERARZWVk0bNiQ1atXc//99wP20cXi4uqboncuqQ15fgLl11cb3ravrJ2nel/KvS2rCaQWtUOLQPmUWlSduqxFQUEBS75ZzJ49uwFo0qQp4ydcSXR0tNRCXhdVoqa1qOln4fmEqlZPjr5Ansd1Oebxx04gtPCnTmrhf3upRfXbuVC1EELw+uuv8eTMp2nUuDFGo5F777mLHj160r5DB1q1bs3CLxe4+BdC8OKLzzPlrrvR6bR8ueBztm3bRsuWLbhs+EiCgoJp374DwcHBHD58iMcfe5Rnnn2+wv6X7keDhg3Zvftfnnn6SXr27E1oaCjtO3SgT9++vDl3jrMvjp+XX3qBKXdNpWPHTpw8eYKnnprJrFmzEULQqnVrJky84mxfHn+UefPerZVaVIS8LqpXC19s+DzQN2TIEG699Va6dOnC3r17GTFiBAC7du2icePGvpqrldRojj4fCIRfX214276ydp7qfSn3tqwmkFrUDi0C5VNqUXXqshbLvvuWQ4cOoNFo6NW7L127dsdsNmM0GqUW8rqoEjWthczR5z01mqMvQOdxXY55/LETCC38qZNa+N/+XGths9mY//67pKenodcHERkZyVVXX0NiYiL5+Xl89NEHZGZkotPpuOrqSbRo0dKlH3v27OaN2bMYP2Gi8wsto/EUn3/2KUXFRVgtFtq178C4cRMAPO6PxWLx2A+gTF+0jBg52rkAhDd9yc/PRxUq7c/0xZd7UE1pEejrQggBQnD8xHFCw8I4cfw4YWHh5OTkEBVlIDc3F7PZTO5p+9/tYH+Rm5eXS1BQEJ99+hEAQUFBdOzYhWbNWgB2DQFCQkIRQuXkyRPodDrmzZtL61ZtuHTIUDIyMpj12itMvec+EhOTnP0IDgkhIyODsLAwcnJyyM/Px2CIBuzngKMvqqpy8uRJ8vLySE5OwWg0otPpCQ4K5scff6RJk6bUq1fftS+qyqFDh9DpdLVOi4qQ96jq1aJac/TNnTuXRx99lGPHjvHVV185V9jdvn07kyZN8tVcreRCydHn7UVT1fnuFdX7Uu5tWU0gtagdWgTKp9Si6tRlLS4fO47Fi76kX/8BtGnTVmrhpx15XZSnprWQOfq850LI0Rco6vI9xZ86qYX/7c+1FlarlZGjRtOlS1cURWHlyhUsWriAJ596hsWLvqR9uw5cedXV7N+/j5dfepG5b85Dp7P/OVxYWMjy5d/RtWs3IiMinYNyH37wPv369WfEyFGYzWamT3+QkydP0LlzF4/7YzabPfYDcOnL3r17efml5xk0aDBBQUGV9qVv3350v6gHBoOBhx9+iJMnT9C1a7dap0V1XBf3/9+DzHrtfwQHh1BYWMADDz5Eamqq029QUBCGaIPzeCUmJhITE0N2djb9+l1MZmYG+/cfICg4yNnGwbp1a0lKSqZx4yYkJibywAMPMX36g3Tq1JmPPv6AyTfeTIcOHcv0I5iCggL+74FppKamutjT6/XOvjj6FxMTw/79++jTpy/79+/DaDyF2WwiOjra5ZitW7eWpORkWrduTWZmZq3UwhPyHlW9WlRrjr7o6GjmzJlTrvzJJ5/01VSdwZt51+eyL9VtIxDz3Suqr8p8d3/2J1BILWqHFnU5D0RF9VKL6tPixInjHDlymH79LgYgISGR227/LxkZGVKLKtqR10V5alKLmn4Onk+UPra1+TyuyzGPP3YCoYU/dVIL/9ufSy1CQkLo3v0iZ7tWrVrx3dJv0Wg0/PLLz8yZOw+NRkPLlq2IjY3l33//oVOnzgDMn/8eV0y8ki1btzjtOWwXFxej0WiwWq3YbDZiY+0fusydO5vu3S5i7LjxpKen8dijjzDzyaeoV6++x34AZfrSEoMhmt27/6Vz5y5e9UVRFJe++HoPqgktAn1d2Gw2vv5qMQ9Om0G7du3Yv28fzz//LK/Nmk1ERASKojj9q6rKjz/+QJcu3Zg+/RE++eQjTueepmGDhrRp0xadTufSv7/++pPFi77k0cdmOvseHR3NvffczxNPPMrASwZx8cUDyvWjTZs2bPv1V+bOeYPXZs0mKirK5biU1W7aQzP47NNPWPLNVzRw9EWrczlmjr48/sSTaLXaWqlFZch7VB3N0bdx48YK6y+++GJfTUokEolEUu3YbDZ+WLeGDRvWI4SgXmo9mjRtBuAMECUSiUQikZwfLF++jIt69CA/Pw+r1UpMTIyzLiExkczMTAB++fknNIrCRT16smXrFhcbN918K88//wyrVq2ksLCQiROvpGnTpqiqyo033sIrr7xI8xYt+OjDD7hh8o3Uq1ffYz8At32JjYsjMyPDu7489wzff7+CouJirjjTlwuBQ4cOkp2dTbt27QBo3qIFcXFxHDp4kA4dOzrb5eae5u15czl58gQH9u/jP7fdyWOPz3TWT717Cg0aNHT+vmvXTubOmc2MGY9Sr14957RfgIOHDhIREUl2VhZCCBRFcemHqqo0atzY2Y9OnTtXuA+NGzfxoS/1UVWZd1fiPz4P9A10s7Ju6T+QHLlOJBKJRCKpLWRlZvLV4i85dSodgI4dO5OcknKOeyWRSCQSiaQ6+GrxItLT0pj55NOYzSaP7XJycli8eBFPPf2M2/pV36+kf7+LGT9hIrmnT/P4E4/SvHkLOnTsSEREBFOn3suTMx9n4CWD6N+//AcvpftRGd70pV+//vTt15/g4CBmznyc5s1bVDrAdD4QH59ATk42x48fo379BqSlpXHqVDqp9eoB9hx+eXl5rFyxjLCwMEJDw+jbtz+5p08TExsLwJo1qwkJCXZOwd21axezX3+Nh6Y/QuMmTVwG1g4eOMDSb5fwyv9e4825b7Dkm68ZN36CSz9SU+0Dg+ml+lEROTnZxMXFu/SlfYcOZGRk8M8/u3hj9ixnXySSquLzQF9OTo7L7xaLhT/++IPHHnuMZ599NmAdq01UtpJKTREIv77a8LZ9Ze081ftS7m1ZTSC1qB1aBMqn1KLq1FYthBD8tu1XVq5chsViISQkhNGjx9LxzFQdx3ZSi6rbkddFeWpaC/n2339UVa6664m6fE/xp05q4X/72qLF0m+XsGXLLzz+xJPo9Xr0ej1arZasrCznl3RG4yni4uLYv38fOTnZ/N/99wKQl5/Ptm2/cjr3NNdccx0rv1/B7NlzUVWVyKgounTpyt87/6Jd+/YIITh48AAREZFkZWVhs9lcPoAp2w9VVQkPj3Dpi6qqZGdlEutlX2bNegObzUZk5Nm+lP6irbZpESiioqK4/Y7/8sorL6FRNKhC5eZb/kNcXByffvox369cgdlsRqPRoNfreeGFl6lXvz6LFn7Jpk0bEALq1a/PAw9ORwiBEII3576BxWJhzpzX7U4EXHPt9YSFhfHqqy9z53+nYDAYuOvuqcyYPo1WrVvTunUbZz8URcFisXDzzbcSFxeHqqp8tXgRq1evIi8vl6NHj/Deu+/w4ksvI4Rg9erV/LR5o9u+vPXmHNe+AHfdNZXQ0LBap0VFyHtU9Wrhiw2fB/oMBkO5siFDhhAUFMT999/P9u3bfTVZ65Cr7vrevrJ2nup9Kfe2rCaQWtQOLQLlU2pRdWqrFqtXrWDPnt0AJCUlM+yykRgMBpepGRXZkVrI66Iq1LQWctVd75Gr7npPXb6n+FMntfC/fW3QYt26NWz79Vem3nMvhYWFFBYWAtC5S1eWfPM1I0eN5sjhw2RmZJKQkIhWq+W5519y+vz44w+pX78+gwZditFoJC42jo0bN9CnT19MJhN/7viDwYOHYDQa+feff/h2yTdncq99zGeffcLQoZcBeOxH2b7Yp4LmkJCQiF6vr7QvmzZtpG3bdpSUlLj0pTZqEWhatmzF9OmPuJT9888uDh86SJMmjdFqtfTtdzEdO3ZGURSMRiMDBl7CgIGXuOyH43iVnkZben8KCgp49DF7naPt40885fzd0Q9H+9JxZf+LB9D/TD4/ByUlJnJzcxkwYCCXXDKoXF9yc3N55NEn3Gpx+vTpWqmFJ+Q9qnq1qNZVdz2RlJTEnj17AmXunCJX3fW9fWXtPNX7Uu5tWU0gtagdWgTKp9Si6tRWLdq178D+/fu49NJhNG/R0uVe7o0dqYW8LqpCTWshV931HrnqrvfU5XuKP3VSC//bn2stcnKy+fqrxSQlJTFnzmwA9Do9z7/wErfeehtvzJ7F00/NRKfTce9995PiJoVHSEiIy0q39957P++//y6bN23EarXQvXsPho8YSWFhIYsWf8mUu6bSokUL/u+BB5kxfRrdu19EQkKCx34AZfqi5aabbyElJaXccXHXl/fef4cNP/6AAC460xd3eYbPtRY1dV3ExsYSGxuL1WrlyqsmkZzsf1qW8/W6kPco79rVBS2qddXdv/76y+V3IQRpaWm88MILdD5P8wN4s5LKuexLddsIxAo2FdVXZQUbf/YnUEgtaocWgfIptag6tUELk8lETna2M/9e167dadq0GQZDtPPLbKlF9dmR10V5alKLmn4Onk+UPra1+TyuyzGPP3YCoYU/dVIL/9ufSy0SEhL56utv3fYrNjaWJ2Y+Vel+Tp16r8vvzZo357nnXyzXLiIigpkzn3a+QDQYonnzrXec9Z76UbYvjq+63O27u748++wLGI1Gjy8uS3O+XhfHjx8jJSUVrVZLUFAQ111/I4WFhSQnlx8s9ZXz8bqQ9yjv29V2LXzZ3ueBvs6d7Z/CCiFcynv16sX8+fN9NSeRSCQSSUA4evQIixYuwGK1MHXq/YSFhaEoCjExsaiqzF0mkUgkEolEUlexWq2sW7eGTRt/ZODAQVw6ZBhgz99XUlJyjnsnkdQufB7oO3TokMvv9rcnCXK6iEQikUjOCTabjXVrV7Nhw3qEEBgM0eTmniYsLOxcd00ikUgkEolEUkWMp06xcNEXpJ08CdhzlQkh3E5blkgkfgz0NWrUqDr6IZFIJBKJz2RmZrB48QKMp04B0KlzF0aPHktoaOg57plEIpFIJBKJpCqoqsrWLb/w/ffLsVqthIWFMXbsBNq173CuuyaR1Gq8GuhbsGABV199tVcGjx07xtGjR+nbt2+VOlabqGzJ5JoiEH59teFt+8raear3pdzbsppAalE7tAiUT6lF1alpLYQQbNu2le9XLsdisRASEsKYy8fRoUMnpx1f7Eotqm5HXhflqWkt5BR1/1FV1flTW8/juhzz+GMnEFr4Uye18L+91KL67VxIWuTl5fHN14vYv38fAC1atGTc+IlERkZVy/NdalF1pBbVq4UvNrwa6Hvrrbd48sknuemmmxg9ejRt2rRxqc/NzeWnn37i008/Zc2aNbz//vu+9biWMXfuXObOnYvNZgPs++dIkgr2A1xXl6v21Ya37Str56nel3Jvy2oCqUXt0CJQPqUWVedcaLFn979YLBaSk1MYdtkIoqIMGI1Gv+xKLapuR14X5alpLfLz8/32caFRNtbLyMigpKSkVp/HdTnm8cdOILTwp05q4X97qUX127mQtMjJyeHw4UNotVr69RtAh46dKC4uobjYNR+f1EJeFxeKFr7EeV4N9G3YsIGlS5fyxhtvMGPGDMLDw0lKSiIkJIScnBzS09OJj4/nxhtvZOfOnSQlJfnd+drAlClTmDJlCnl5eRgMBgwGg8vKRqpad5er9tWGt+0ra+ep3pdyb8tqAqlF7dAiUD6lFlWnprSw2WxotVoAJl5xFX/99SeNGjWpdPU5qUX125HXRXlqWguZL9l7ysZ6CQkJREVF1erzuC7HPP7YCYQW/tRJLfxvL7Wofjvnuxal47zExEQmXnE1iYmJJCQk+rWvviC1qDpSi+rVwpc4z+scfWPGjGHMmDFkZmayefNmjhw5QnFxMfHx8XTp0oUuXbrU6ElUk3izZPK57Et12wjEUtUV1VdlqWp/9idQSC1qhxaB8im1qDrVqYXJVMLyZd9RVFTEtdfdgKIohIdH0LNnb+cX11KLyvtc3XbkdVGemtTifI3DaoLSx7Y2n8d1Oebxx04gtPCnTmrhf3upRfXbOV+1OHToIF8tXsiEiVfSpElTADp06OjVtlILeV1cCFr4sr3Pi3HEx8czduxYXzeTSCQSicQvjhw5zOJFC8jOzkZRFE6cOE79+g3OdbckEolEIpFIJFXEarWydu1qNm/agBCCH9at4ZZbbz/X3ZJI6jQ+D/RJJBKJRFIT2Gw2fli3hg0b1iOEIDo6mokTr5KDfBKJRCKRSCTnAadOpbNo4QLS0k4C0K3bRYwcNfoc90oiqfvIgT6JRCKR1DoyMowsXvwlJ0+cAKBzl66MHn05ISGh57hnEolEIpFIJJKqoKoqW375mVWrVmC1WgkLC2fc+Am0bdv+XHdNIjkvkAN9EolEIqlVCCFYsOAzMoxGQkNDuXzsBK9ztEgkEolEIpFIajd79+5h+fKlALRs2YrxE64kMjLyHPdKIjl/kAN9XiCEQFVV5++qqpYrqwkC4ddXG962r6ydp3pfyr0tqwmkFrVDi0D5lFpUnUBqATB61OVs3LSBceMmEBVl8PkY+tpOalF1O1KL8tS0FjX9LDyfUFXV+VNbz+O6HPP4YycQWvhTJ7Xwv73UovrtnC9atGjRko6dOtOoYWMu6tETRVH8PpZSC3ldXCha+GLD54G+9evXc8kll/i6WZ1i7ty5zJ07F5vNBkBubq5zVUewH+Dc3FyEEDW6ik0g/Ppqw9v2lbXzVO9LubdlNYHUonZoESifUouqU1WfBw7sw2w206pVG3JzczEYDAwfPoqSEhMlJcYq+5VaVL8dqUV5alqL/Px8v31caJSN9TIyMigpKanV53Fdjnn8sRMILfypk1r4315qUf126qoWJpOJrVt/pkeP3oSEhAAwYMAgwH7/rQpSC3ldXCha+BLn+TzQd9lll1G/fn1uuukmJk+eTIMG519S9ClTpjBlyhTy8vIwGAwYDAYSExNdxFMUhYSEhBq/cKrq11cb3ravrJ2nel/KvS2rCaQWtUOLQPmUWlQdf32aTCZWLF/K779vR6/X07ZtO6Kjo6UWVUBeFxeuFo4/niSVUzbWS0hIICoqqlafx3U55vHHTiC08KdOauF/e6lF9dupi1ocOnSQrxYvJDf3NEIVXHHl1V4dG2+RWsjr4kLRwpc4z+eBvhMnTvDJJ5/w0Ucf8eSTTzJo0CBuueUWxo4dS1BQkK/m6gSKoqDRaFyEcVd2rvpS3Ta8bV9ZO0/1vpRLLaQWZQmUT6lF1fHV55Ejh1m0cAE5OdkoikLv3n2JiYklOztbalFF5HVxYWpR08/B84nSx7Y2n8d1Oebxx04gtPCnTmrhf3upRfXbqStaWK1W1q5ZxebNGxFCEBsbS6/efapFJ6mFvC4uBC182d5nT/Hx8dx3333s2LGDrVu30rJlS/773/+SmprK1KlT+fPPP301KZFIJJILCKvVyurV3/PuO2+Rk5NNdHQMt9x6O8MuG4FOJ1PHSiQSiUQikdRlTqWn89Zbb7Bp0waEEHTrfhF33X0vjRo1Ptddk0guCKr0F1XXrl1JTk4mLi6OF154gfnz5/Pmm2/Su3dv5s2bR7t27QLVT4lEIpGcB9hsNt595y2OHz8GQJeu3Rg1agwhIaHnuGcSiUQikUgkkqqyd89uvvjiU6xWK2Fh4YwbP4G2bduf625JJBcUfn07aLFYWLx4MSNGjKBRo0asWrWKOXPmcOrUKfbv30+jRo244oorAt1XiUQikdRxtFotLVq2IjQ0jEnXXM/EiVfJQT6JRCKRSCSS84T6DRoSFhZOy1atmXrP/XKQTyI5B/j8Rd/dd9/NF198gRCC66+/npdeeon27c9evOHh4bzyyiukpqYGtKMSiUQiqZvk5eVisViIi4sH4JJLBtOjRy+ioqLOcc8kEkllmEwmgoODz3U3JBKJRFKLOXzoEA0aNgQgLCyMO+6cQlSUAUVRznHPJJILE58H+v755x/eeOMNxo8f7zHwi4+PZ/369VXunEQikUjqNjv//oslS74mOjqaO+68C51Oh1arlYN8EkktZeXKlSxYsIBNmzZx7NgxVFUlPDycLl26MHToUG666Sb5MlcikUgkAJhMJSxauIC//trBuPETqV/fPthn+H/27ju8iSNxH/gruRcsDO4Um96xwXQI1WAMMb0loedyJU7lkku4u/R6/C79nHBJLp0LYNNr6N0JoZgSwDTT3TC4d+3+/uCLDiPL1q5WWpX38zw8D54Zz4z0auXxWLura6zuxIhcnOSNvu3btzfcqbs7hgwZImtCRETk+CoqKrB+/RocPXIYANA4sDFKS0uh0+lUnhkR1WXVqlV44YUXUFxcjDFjxuCFF15AREQEfHx8cOvWLZw8eRLbtm3DG2+8gblz5+KNN95AcHCw2tMmIiKVXLx4ASnLl6KkpBharRYlJSVqT4mI/o/kjb533nkHoaGhmD9/fq3yr776Cnl5eXjhhRcUmxwRETmeS5mZSElZioKC29BoNBg8eCiGjxjJO+oS2bFFixbhgw8+QEJCArRa40s4T5s2DQBw/fp1fPLJJ/jhhx/w7LPP2nqaRESkspqaGmzdshn79++FKIpo0qQppk6bgebNWyA3N1ft6RERZGz0/fvf/8Z///tfo/IuXbpgxowZ3OgjInJRer0eW37ahH379kAURQQGBmLK1BmIimql9tSIqAFpaWlmtWvWrBneffddK8+GiIjsUXZ2NlJTlyInOxsA0KVLN0ycNAU+Pj4QBEHl2RHRXZI3+rKzsxEeHm5UHhwcjKysLEUmZW9EUaz1xiUIglGZLSgxrtQ+zG3fUDtT9VLKzS2zBWZhH1koNSazsNzdsS5dyoQoiujRIxZjxibC29tb0jyYheV4XLhuFtZ6bHq9HidOnEBkZCQCAwOtMobaBEEw/LPX17Ejr3nk9KNEFnLqmIX89szC+v2onUVZaQlyc3Lg5+eHceMnoWnTIHh4eEh6/1Saq2bB40J6e0fPQkofkjf6WrRogf3796NVq9qf0Ni/f7/TXJw5OTkZycnJ0Ov1AIDCwkLk5uYaTmURBAGFhYUQRbHO01usRYlxpfZhbvuG2pmql1JubpktMAv7yEKpMZmFfHd/cGk0GhQVFWHI0BHIz89D27btUVRUhKKiIkn9MQvL8bhw3SyKi4tlj3GvZ555Bt26dcOjjz4KvV6PIUOG4MCBA/D19cX69esxdOhQRcZR0/1rvby8PFRUVNj169iR1zxy+lEiCzl1zEJ+e2Zh/X7UyEKv10Oj0aCwsBA6nQ4j4uIRGRkJb28fFBQUMAseF2Y9Hmv24ypZSFnnSd7oe+yxx/DMM8+guroaw4cPB3DnBh1/+ctf8Oc//1lqd3YpKSkJSUlJKCoqgk6ng06nQ0hISK3wNBoNgoODbX7gWDqu1D7Mbd9QO1P1UsrNLbMFZmEfWSg1JrOQp6ioEKtWpiIkNBTx8WMMY3bo0EF2n8zCcjwuXDcLb29v2WPcKzU1FTNnzgQArFu3DpmZmThz5gy+//57/O1vf8P+/fsVGUdN96/1goODERAQYNevY0de88jpR4ks5NQxC/ntmYX1+7F1FsePpeOnnzZh7rxH0bhxYwQHByMsLKzOtsxCejseF8r04ypZSFnnSd7oe/7555Gfn4/HH38cVVVVhgFfeOEFLFy4UGp3DkGj0UCr1dYKpq4yteZi7T7Mbd9QO1P1UsqZBbO4n1JjMgtpTpw4jjWrV6C8vByXL1/CoEFDmIWEMmtjFq6ZhVKP6+bNm4Zf5DZu3IipU6eiffv2mD9/Pj766CNFxrA39z639vw6duQ1j5x+lMhCTh2zkN+eWVi/H1tkUVlZgfXr1uD48WMAgLQD+9Gv/0BmIbM9jwvr9+MKWUj5fskbfRqNBv/4xz/w0ksv4fTp0/Dx8UG7du3g5eUltSsiInIwFRXlWLduDdKPHgEARDRrhmlTH0KjRo1QXl6u8uyISAmhoaE4deoUwsPDsXnzZnz22WcAgLKyMri5uak8OyIisqarV69gx/atKCoqhFarxdBhIzB48FDk5+erPTUiMpPkjb67/P390bt3byXnQkREdiwz8yJSU5ahoOA2NBoNhgwZhmHD4+Du7g5B4J3WiJzFvHnzMG3aNISHh0Oj0SAuLg4A8Msvv6Bjx44qz46IiKyhuroaW37ahAMH9gEAmjZtiqnTHkKLFi25ziNyMJI3+kpLS/Huu+9i+/btyM3NNTroL168qNjkiIjIPlRWVmDJD9+ivLwcgYFNMHXaDERGRqk9LSKygldffRVdu3bF1atXMXXqVMNZG25ubnjxxRdVnh0REVnDwYM/Gzb5evfuizFjE+Hp6anyrIhIDskbfb/73e+we/duzJo1y/CXXiIicm5eXt548MHxuHDxPB58cBy8vJS56D8R2acpU6YYlc2ZM0eFmRARkS306zcA58+dRfv2HdG3X3+bX9uNiJQjeaNv06ZN2LBhAwYOHGiN+RARkR0QBAE/px1AcHAw2rW/cxfdmB49EdOjp8ozIyJre/311+utf/nll200EyIispaCggLs27cHY8Y8CHd3d7i5uWHW7HnIzc1Ve2pEZCHJG32BgYFo0qSJNeZCRER2oLCwECtWLMeF8+fQqFEjPP3Mc/Dx8VF7WkRkI6tWrar1dXV1NTIzM+Hu7o42bdpwo4+IyIGJooiMM6exZ89OVFRUwNfXF3Fxo9SeFhEpSPJG3xtvvIGXX34Z3377LXx9fa0xJyIiUsmJ48ewZs1KlJeXw8PDA8OGxcHbm6fpErmSo0ePGpUVFRVh7ty5mDhxogozIiIiJZSXl2H16pU4eeI4AKBFi5bowbM1iJyO5I2+9957DxcuXEBoaCiioqLg4eFRq/7IkSOKTY6IiGyjvLwc69etQXr6nffwZs2aY+q0GQgODlF5ZkRkDwICAvDaa68hMTERs2bNUns6REQk0fnz57AidTmKigqh0WgwbHgchg4dDjc3N7WnRkQKk7zRN2HCBCtMg4iI1FJWVorkf32EgoICaDQaDB06HMOGx3HhR0S1FBYWorCwUO1pEBGRRD+nHcC6dasBAE2DgjBixCh069adN9wgclKSN/peeeUVa8yDiIhU4uvrh6io1rhy5TKmTpuBli0j1Z4SEano448/rvW1KIrIysrC999/j4SEBJVmRUREcrVr3x6enp6I6RGL+PgEFBQUqD0lIrIiyRt9wJ079KSmpuLChQt4/vnn0aRJExw5cgShoaFo1qyZ0nMkIiKFZWdnwd/PH/6NGgEAEsdNgEYDeHnxenxEru6DDz6o9bVWq0VwcDDmzJmDhQsXqjQrIiIylyAIuHb1CkJC7lyCpWnTIDy74HkEBOggCILKsyMia5O80Xf8+HHExcVBp9Ph0qVLeOyxx9CkSROsXLkSV65cwXfffWeNeRIRkQIEQcCBA/uw5adNaNeuPWbOmguNRsMbbhCRQWZmptpTICIimW7fvoWU5Utx+fIlBAYGok3bdgCAgACdyjMjIluRfFL+ggULMHfuXJw7d67WL4ZjxozBnj17FJ0cEREpp6CgAF9/9SU2bVwPvV4PURRRXV2t9rSIiIiIyEKiKOLokcP45OMPcPnyJXh4eKCktETtaRGRCiRv9P3666/4wx/+YFTerFkzZGdnKzIppU2cOBGBgYGYMmWK2lMhIlLFsWNH8cnHH+DixfPw8PDA+PGTMGv2PHh6eqo9NSKyA3/84x9x7do1s9ouW7YMS5YssfKMzMd1HhG5urKyUiz98Qekpi5DZWUlWrRoiYcemoVu3aLVnhoRqUDyqbteXl4oKioyKj979iyCg4MVmZTSnn76acyfPx/ffvut2lMhIrKpiopyrFm9CsePpwMAmjVvgWnTZiAoyD7fr4lIHcHBwejSpQsGDhyIxMRE9OrVCxEREfD29sbt27dx6tQp7Nu3D0uXLkVERAQ+//xztadswHUeEbmy8+fPITVlKYqLi6HVajF8+EgMemAw8vPz1Z4aEalE8kbfuHHj8Prrr2P58uUAAI1GgytXruCFF17A5MmTFZ+gEoYOHYpdu3apPQ0iIpvTaDS4du0KtFothg4djqHDRsDNzU3taRGRnXnjjTfwxBNP4Msvv8Snn36KU6dO1apv1KgR4uLi8Pnnn2P06NEqzbJuXOcRkSsrKipEcXExgoKCMXXaDDRv3oI33CBycZJP3X3vvfdQUlKCkJAQlJeXY8iQIWjbti0aNWqEt956S/EJ7tmzB4mJiYiIiIBGo8Hq1auN2iQnJyMqKgre3t7o27cvDh48qPg8iIgcRU1NDURRBHDnLrrTpj+M3//hcYyIG8VNPiIyKTQ0FH/7299w4sQJ3Lx5E0eOHMH+/fuRkZGB27dvIzU1VfFNPq7ziIiku/cayz16xGLCxMlIeuJpNG/eQsVZEZG9kPyJPp1Oh61bt2L//v04duwYSkpK0LNnT8TFxVljfigtLUV0dDTmz5+PSZMmGdUvW7YMCxYswOLFi9G3b198+OGHiI+PR0ZGhuF24uaqrKxEZWWl4eu7pyiLoljrryKCIBiV2YIS40rtw9z2DbUzVS+l3NwyW2AW9pGFUmM6UxbZ2VlITVmGDh06YUTcSABAs2bNDW2thVnwuGAWxmydhZKPLTAwEIGBgYr1Z4ot13mA6bWeIAiGf/b6OnbkNY+cfpTIQk4ds5DfnllYv5+amhr8+usvOHP6FP70+JPw9fUFAMTG9jb011C/zEKZfnhcGGMW1s1CSh+SN/q+++47TJ8+HQMHDsTAgQMN5VVVVVi6dClmz54ttct6JSQkICEhwWT9+++/j8ceewzz5s0DACxevBgbNmzAV199hRdffFHSWO+88w5ee+01o/LCwkLk5uZCq73zAUhBEFBYWAhRFA1ltqDEuFL7MLd9Q+1M1UspN7fMFpiFfWSh1JjOkIUgCDh27CjSDuyHIOhRUlKMzl26wMPDNjfbYBY8LpiFMVtnUVxcLHsMtdhynQeYXuvl5eWhoqLCrl/HjrzmkdOPElnIqWMW8tszC+v2U1hYiK1bNiEr6wYAYO+e3ejRM1Zyv8xCmX54XBhjFtbNQso6T/JG37x58zB69Gijv6IWFxdj3rx5im/01aeqqgqHDx/GwoULDWVarRZxcXFIS0uT3N/ChQuxYMECw9dFRUVo0aIFdDodQkJCaoWn0WgQHBxs8wPH0nGl9mFu+4bamaqXUm5umS0wC/vIQqkxHT2LkpJibNywFpcuZQIA2rfviEEPDEZ4eASzMLMdjwvL+2EWxmydhbe3t+wx7JHS6zzA9FovODgYAQEBdv06duQ1j5x+lMhCTh2zkN+eWVinH1EUkX70CDZsWIvKykp4eHhg7Nhx6BnbCxqNRnK/zEKZfnhcGGMW1s1CyjpP8kafKIp1vqFcu3YNOp1OancWuXnzJvR6PUJDQ2uVh4aG4syZM4av4+LicOzYMZSWlqJ58+ZISUlB//79jfrz8vKCl5eX1edNRKSU48fTsW7tGlRV3Vn4JYx5ED179sLNmzfVnhoRkUWUXucBXOsRkWMpKyvFmjWrcOq3kwCAli0jMWTocLRt287kJh8RkdkbfT169IBGo4FGo8GIESPg7v6/b9Xr9cjMzLS7u7DdtW3bNkntk5OTkZycDL1eD4Cn7trrKSxyHo9SmIV9ZOHIHw+vr97c8oKC21i5IgWiKCI0NAyj4hPQuHEg8vLymIXEdjwuLO+HWRizdRaOeOquEqSu8wDjtR5P3TXNkd9T5NQxC/ntmYXy/ezatR2nfjsJrVaLPn37o0ePWBQXF9f63VRqv8xCmX54XBhjFtbNwiqn7k6YMAEAkJ6ejvj4ePj7+xvqPD09ERUVhcmTJ5s/SwUEBQXBzc0NOTk5tcpzcnIQFhYmu9+kpCQkJSWhqKgIOp2Op+7a6Sksch6PUpiFfWThyB8Pr6/e3PKQkBAMHz4SxSXFGD16DDw8PGQ9HiW4ehZSyqyNWbhuFkqduvvKK69g/vz5iIyMVKQ/uay1zgOM13o8ddc0R35PkVPHLOS3ZxbK95OYOB5lpWUYOSoezZo1hyAIcHNzYxZmPh5r9sPjwhizsG4WVjl195VXXgEAREVFYfr06XZxHRhPT0/ExsZi+/btho1IQRCwfft2PPHEE4qNo9FooNVqawVTV5ktKDGu1D7Mbd9QO1P1UsqZBbO4n1JjOkIW1dXV2L9/DwYOfADh4REAgKHDhiM3NxceHh7MgseF4mMyC8vZMgulHteaNWvw1ltvYciQIXj00UcxefJkVU53tdU6D0Ct59aeX8eOvOaR048SWcipYxby2zMLy/q5fv0ajh07ioSEB6HRaODn54/5jz4ma1xmYf1+mIUxZmG9LKR8v+Rr9M2ZM0fqt1ikpKQE58+fN3ydmZmJ9PR0NGnSBC1btsSCBQswZ84c9OrVC3369MGHH36I0tJSw93ZlCCK9d8y2VaUGFdqH+a2b6idqXop5eaW2QKzsI8slBrTEbLIyrqB1JRlyM3NQdaNG/jT409Cq9UyCx4XRpiF62ah1GNLT0/H0aNH8fXXX+Ppp59GUlISZsyYgfnz56N3796KjHGXPazzgDvP3d1/9vo6duQ1j5x+lMhCTh2zkN+eWcjvp6amBvv37cWOHVshCAJCQ8PQo4fxHXWZhTEeF8zCVbKQ0ofkjT69Xo8PPvgAy5cvx5UrV1BVVVWr/tatW1K7rNehQ4cwbNgww9d375Q2Z84cfPPNN5g+fTry8vLw8ssvIzs7GzExMdi8ebPRhZul4DX6pLdvqJ2peinl5pbZArOwjyyUGtOeswCA9KOHkZZ2AIKgh7e3N2JjextutsEseFyY+1is3Q+zMGbrLJS8Rl+PHj3Qo0cPvPfee1i3bh2+/vprDBw4EB07dsSjjz6KuXPnKnITNjXWeQCv0SeFI7+nyKljFvLbMwt5/Vy/dg0rUpchK+sGAKBNm3Zo0iQIubm5ssdlFtbvh1kYYxbWzcIq1+i767XXXsOXX36JP//5z/j73/+Ov/3tb7h06RJWr16Nl19+WWp3DRo6dKjhl11TnnjiCUVP4eA1+qS3b6idqXop5eaW2QKzsI8slBrTXrPw8HDHqpWpuHQpEwDQoWMnDBw4GJGRkcxCYnseF9bvh1kYs3UW1risiiiKqK6uRlVVFURRRGBgIP71r3/hpZdewhdffIHp06db1L8a6zyA1+iTwpHfU+TUMQv57ZmFNKIo4sjhQ9iwYS2qq6vh5eWFsWPHIaZHT2g0dd9Rl1kY43HBLFwlC6tco++uJUuW4IsvvsDYsWPx6quv4qGHHkKbNm3QvXt3/Pzzz3jqqaekdmn3zDnvWs25WLsPJc53r6/ekvPd5TwepTAL+8jCka8DUV/9rVv5WJG6DJWVlfD09MSYsYno2bMX8vLymIXM9jwurN8PszBmyyyUfFyHDx/G119/jR9//BFeXl6YPXs2kpOT0bZtWwDAJ598gqeeesrijT57ce9za8+vY0de88jpR4ks5NQxC/ntmYX51q9bg7S0/QCAyMgoTJk6A02aNFFsXGZh/X6YhTFm4aDX6MvOzka3bt0AAP7+/igsLAQAPPjgg3jppZekducQGjrv2laUGFdqH+a2b6idqXop5eaW2QKzsI8slBrTHrNo3DgQoaFhEEURk6dMQ9OmQczCgvY8LqzfD7MwZusslHps3bp1w5kzZzBq1Cj85z//QWJiItzc3Gq1eeihh/D0008rMp49EAReo88UR35PkVPHLOS3ZxbSdO7SFb/++gt69+6LUfEJcHd3t/i9x5x2zEKZfpiFMWZh3Syk9CF5o6958+bIyspCy5Yt0aZNG2zZsgU9e/bEr7/+qsod2ayB1+iT3r6hdqbqpZSbW2YLzMI+slBqTHvJ4vr1awgNDYVW64aioiLEjRwNHx8f6PUCcnNzmYUF7XlcWL8fZmHM1lkodY2+adOmYf78+WjWrJnJNkFBQTb/BUJJvEaf+Rz5PUVOHbOQ355Z1K+6uhq5uTlo1qw5AMDX1w+zZs9DdXUNbt68ySxk4nHBLFwlC6teo2/ixInYvn07+vbtiyeffBIzZ87Ef/7zH1y5cgXPPvus1O7sEq/RJ719Q+1M1UspN7fMFpiFfWSh1JhqZ9G4cWNs37YFaWn70a//ACQkPMgseFzIxixcNwulrtHnrGdo3IvX6DOfI7+nyKljFvLbMwvTrl+/htSUZSgsLMCfHn8KwcHBAO780SQvL49ZWIDHBbNwlSyseo2+d9991/D/6dOno2XLlkhLS0O7du2QmJgotTuHYM5512rOxdp9KHG+e331lpzvLufxKIVZ2EcWjnwdCAC4eTMPy5YtQd7/3VlNFERoNBpmwePCIszCNbNQ6nFNnjwZffr0wQsvvFCrfNGiRfj111+RkpKiyDj25N7n1p5fx4685pHTjxJZyKljFvLbM4va9Ho99uzZhR3bt0IQBDQKCEBFRblFP5uYhTEeF8zCFbKQ8v2SN/ru179/f/Tv39/SboiIXIogCNi7dze2bf0JgiDA398fkyZPRYcOnSAIjns6HBE5vj179uDVV181Kk9ISMB7771n+wkRETmgW7fykbJ8Ka5cuQwA6Nq1O8ZPmARfX1+VZ0ZEzs6sjb61a9ea3eG4ceNkT8ZeNXSBRVtRYlypfZjbvqF2puqllJtbZgvMwj6yUGpMW2dRWFCA1NRluHQpEwDQsVNnTJgwCX5+/hAE0xeDZxby2/O4sH4/zMKYrbNQ6rGVlJTA09PTqNzDwwNFRUWKjGFv7r732vPr2JHXPHL6USILOXXMQn57ZvE/R44cwob1a1FVVQUvLy88+OB4RMf0gEajsehnE7MwxuOCWbhKFlL6MGujb8KECWZ1ptFoDBc1dmS8GYf09g21M1UvpdzcMltgFvaRhVJj2jqL4uJiZGXdgIeHB2J79UHPnr1QWlqG0tKyer+PWchvz+PC+v0wC2O2zkKpm3F069YNy5Ytw8svv1yrfOnSpejcubMiY6iNN+MwnyO/p8ipYxby2zOL/7l27RqqqqoQEdEMI0clICAgAHl5eZL7kdueWVi/H2ZhjFlYNwvFb8Zh651gtfFmHNLbN9TOVL2UcnPLbIFZ2EcWSo1piyz0ej08PDwAACEhIZgx4xE0DmwCvV7PLCzoh8eFMWbhulkoeTOOSZMm4cKFCxg+fDgAYPv27fjxxx+d5vp8vBmH+Rz5PUVOHbOQ397Vs6iurjas9caOTUTzZs0R06OnRe8lctu7eha26IdZGGMW1s3CqjfjcEXmXGBRzblYuw8lLmxZX70lF7aU83iUwizsIwulxrRmFhcvXsCqlamYMHESOnToBABo36EjBEEwfFqYWcjvh8eFMWbhmlko9bgSExOxevVqvP3220hNTYWPjw+6d++Obdu2YciQIYqMYW/ufW7t+XXsyGseOf0okYWcOmYhv70rZlFTU4NNmzbg6tXL+OMfn4C7uzu0Wi169e4jqR9mIR+PC2bhCllI+X5u9BERWUl1dTX27N6JY8eOAgD27tmN9u07QqPRqDwzIqL6jR07FmPHjlV7GkREdu3atatYkboMN2/eBACcP38WHTs6xyUOiMhxcaPPDA1dYNFWlBhXah/mtm+onal6KeXmltkCs7CPLJQa0xpZ3LhxHakpy5CXlwsA6Nu3P0bFJ0AURYiiWG8/zILHhSWYhetmofRjq6qqQm5urlG/LVu2VHQceyAIvBmHKY78niKnjlnIb+9KWVRXV+PgL2n49ddfIAgCAgICMGnSVLRp207SXJiF5XhcMAtXyUJKH9zoqwNvxiG9fUPtTNVLKTe3zBaYhX1kodSYSmYhCAKOHDmEX34+AEEQ4O3jg7i4eLRq1RoFBQVm9cMseFxYglm4bhZK3Yzj3LlzmD9/Pg4cOFCrXBRFaDTOeeM13ozDNEd+T5FTxyzkt3eVLAoLCrBlyyZkZ2cBANq164Chw4bD29sHubm5kvpiFpbjccEsXCULxW/G4Wp4Mw7p7RtqZ6peSrm5ZbbALOwjC6XGVDKLCxfOI+3APgBAp85dMGDAA2jZsiWzsFI/PC6MMQvXzUKpm3HMnTsX7u7uWL9+PcLDw53ycgO8GYf5HPk9RU4ds5Df3lWy2LRpPbKzs+Dp6YXExPGIjukh+32SWViOxwWzcJUsFL8ZR1FRkdkdBgQEmN3WUZhzgUU152LtPpS4sGV99ZZc2FLO41EKs7CPLJQaU6ks2rVrj379BqBZs+aIjumBvLw8ZmHlfnhcGGMWrpmFUo8rPT0dhw8fRseOHRXpzxHc+9za8+vYkdc8cvpRIgs5dcxCfntXyGL8+IlYv24N+g8YhDZt2jILHhfMwow5W7sfV8hCyvebtdHXuHFjs/9K4QyncxARmausrBSbN2/EyJGj0ahRIwBA4rgJAO78RYeIyNF07tzZcGF5IiJXd+bMaWRl3cCwYSMAAE2aNMXMWXMln6ZLRGQrZm307dy50/D/S5cu4cUXX8TcuXPRv39/AEBaWhq+/fZbvPPOO9aZJRGRHTp37ixWrUxBcXExykrLMHPWHLWnRERksX/84x/4y1/+grfffhvdunWDh4dHrXpnPHuDiOh+VVVV2LhxHX49+AsAoFVUa0S1aqXyrIiIGmbWRt+QIUMM/3/99dfx/vvv46GHHjKUjRs3Dt26dcPnn3+OOXP4iy4RObeqqirs3rUDx4+nAwCCg0MwbPgIdSdFRKSQuLg4AMCIEbXf15zpZhxERPW5evUKUlOWGj7dPHDgA2jWvLnKsyIiMo/km3GkpaVh8eLFRuW9evXC7373O0UmZW8aumWyrSgxrtQ+zG3fUDtT9VLKzS2zBWZhH1koNaaUfm7cuI6U5Utx82YeAKBvv/4YNSoBnp6eZj+nDdUzCx4XlmAWrpuFUo/t3jM5XIUgCIZ/9vo6duQ1j5x+lMhCTh2zkN/eGbLQ6/XYs3sndu3aAUEQEBAQgEmTp6FNm7aG8ZQcl1lYjlkwC1fJQkofkjf6WrRogS+++AKLFi2qVf7ll1+iRYsWUruzS8nJyUhOTjb8xbqwsBC5ubmGix8KguPerlpqH+a2b6idqXop5eaW2QKzsI8slBrT3H4uXcrEhvVrIAgCfHx8EBcXj6hWrVFQUCCrX2ZheT88LowxC9fNori4WPYY97r3TA5ndf9aLy8vDxUVFXb9OnbkNY+cfpTIQk4ds5Df3tGzEEURa9esxJUrlwEA7dp3wNChI+Dt7W10PT5mweOCWRhjFtbNQso6T/JG3wcffIDJkydj06ZN6Nu3LwDg4MGDOHfuHFasWCG1O7uUlJSEpKQkFBUVQafTQafTISQkpFZ4jnq7aql9mNu+oXam6qWUm1tmC8zCPrJQakxz+9HpdNi/bw/CwsLQf8ADaNmyJbNoYM7W7ofHhTFm4bpZeHt7yx7jfnv37sW///1vXLx4ESkpKWjWrBm+//57tGrVCoMGDVJsHLXcv9YLDg5GQECAXb+OHXnNI6cfJbKQU8cs5Ld3hix69+6L3NwcPJg4AdHRMbIeqxTMwnLMglm4ShZS1nmSN/rGjBmDs2fP4rPPPsOZM2cAAImJifjjH//oNJ/ou585t0xWcy7W7kOJW1XXV2/JrarlPB6lMAv7yEKpMevqRxRFnD17Bu3adYBWq4WPjw/++MckePv4IC8vj1ncx5pZKNGeWVi/H2ZhzJZZKPW4VqxYgVmzZuGRRx7BkSNHUFlZCeDOGQ5vv/02Nm7cqMg49uTe59aeX8eOvOaR048SWcipYxby2ztaFiUlJSgouI3mze/8HtujZyw6dOgIP3//Br+XWfC4YBbGmIX1spDy/ZI3+oA7p+++/fbbcr6ViMghlJaWYvWqFTh16iTGPjgOAwbc+QSLn78/BMG217sgIrKlN998E4sXL8bs2bOxdOlSQ/nAgQPx5ptvqjgzIiLlnD59CqtWpkCj1eKppxbAz88PGo3GrE0+IiJ7JmtLce/evZg5cyYGDBiA69evAwC+//577Nu3T9HJERGpISPjDD7++H2cOnUSbm5uELmxR0QuJCMjA4MHDzYq1+l0Jq9JSkTkKCorK7F61Qr88P03KC0thZ+vH8rLy9SeFhGRYiR/os8VT+cgItdQVVWFLT9twi+/pAEAgoNDMG36Q4iIaKbyzIiIbCcsLAznz59HVFRUrfJ9+/ahdevW6kyKiEgBV69eQcryH5Gfnw+NRoOBAx9A3Mh4eHh4qD01IiLFSN7o4+kcROSM8vJy8eN/v8fNm3kAgP79ByJ+9Bgu/IjI5Tz22GN4+umn8dVXX0Gj0eDGjRtIS0vDc889h5deeknt6RERSSaKInZs34pdu3ZAEATodI0xZco0tG7TVu2pEREpTvJGH0/nICJnJAgCbt3KR6OAAEyZPA1t27VXe0pERKp48cUXIQgCRowYgbKyMgwePBheXl547rnn8OSTT6o9PSIiyTQaDfJv5UMQBERHxyBx3ET4+PioPS0iIquQvNHniqdziKJY6+L7giAYldmCEuNK7cPc9g21M1UvpdzcMltgFvaRhaVjVlVVwdPTE4IgICQkFNOmPYRWrdvA19e33j6ZhTGlxuRxYTlm4bpZKPXYNBoN/va3v+H555/H+fPnUVJSgs6dO8PfiS9QLwiC4Z+9vo4dec0jpx8lspBTxyzkt7e3LERRRHV1NTw9PQEAY8eOQ8eOndG1azfD98rFLHhcMAtjzMK6WUjpQ/JGnyuczpGcnIzk5GTo9XoAd64/mJuba7idsSAIKCwshCiKNr1dtRLjSu3D3PYNtTNVL6Xc3DJbYBb2kYXcMUVRxKlTJ3Fg/15MnDQVTZo0RWFhIZoGBaOkpAQlJSWKjMssrN8PszDGLFw3i+LiYtlj3Gv+/Pn46KOP0KhRI3Tu3NlQXlpaiieffBJfffWVIuOo6f61Xl5eHioqKuz6dezIax45/SiRhZw6ZiG/vT1lUVZWiu3btkKjAcY+OB4ajQYAEBISitzc3IafJIWeE6X7ccQsrI1ZMAtXyULKOk/yRp8rnM6RlJSEpKQkFBUVQafTQafTISQkpFZ4Go0GwcHBNj9wLB1Xah/mtm+onal6KeXmltkCs7CPLOSMWVpagjWrV+L06VMAgHNnM5A4bgKzsJBSY/K4sByzcN0svL29ZY9xr2+//RbvvvsuGjVqVKu8vLwc3333nVNs9N2/1gsODkZAQIBdv44dec0jpx8lspBTxyzkt7eXLE6fPoU1q1egtLQU7u7uAESEhISa+/SYhVnwuGAWxpiFdbOQss6TtNGn1+uxf/9+JCUludTpHBqNBlqttlYwdZWpNRdr92Fu+4bamaqXUs4smMX9pIyZkXEaK1ekoKSkBG5ubogbGY9BgwZL7kdKe2Zh/X6YhTFm4ZpZWDpGUVERRFGEKIooLi6utaDU6/XYuHEjQkJCLBrDXt373Nrz69iR1zxy+lEiCzl1zEJ+ezWzqK6uxsYN63Do0EEAQFhYOKZNewihYWFmPUapmAWPC2ZhjFlYLwsp3y9po8/NzQ2jRo3C6dOn0bhx41qncxAR2auqqips2rgeBw/+DODOaRtTpz2EiIgIAHf+8kJE5OoaN24MjUYDjUaD9u2Nb0ik0Wjw2muvqTAzIqL6XblyGStSl+PWrXxoNBoMGjQYcSPj/+8TfURErkXyO1/Xrl1x8eJFtGrVyhrzISJS3NGjhw2bfAMGDsKoUQnw8PBQeVZERPZl586dEEURw4cPx4oVK9CkSRNDnaenJyIjIw1/ICEisheCIGDlihTcupUPna4xpkydjtat26g9LSIi1Uje6HvzzTfx3HPP4Y033kBsbCz8/Pxq1QcEBCg2OSIiJfTu3ReXMjMR26s32rZtp/Z0iIjs0pAhQwAAmZmZaNGihc1P9yEikkOr1WLipCk4dOggHnxwPHx8fNSeEhGRqiRv9I0ZMwYAMG7cOGg0GkO5KIrQaDSGu5cREaklPz8fO3Zsxfjxk+Dp6QmtVovpMx5We1pERA4hMjISAFBWVoYrV66gqqqqVn337t3VmBYREYA7v3ce/OVnuLm7ITa2NwAgMjIKrVq1VnlmRET2QfJG386dO60xDyIii4miiEOHDmLjhnWoqqqCv58/EsY8qPa0iIgcSl5eHubNm4dNmzbVWc8/6hKRWoqLi7Fu3WpcvpQJd3d3tG7FU3SJiO4neaPv7mkdRET2pKSkBKtWpeLM6VMAgFatWqNf/4Eqz4qIyPE888wzKCgowC+//IKhQ4di1apVyMnJwZtvvon33ntP7ekRkYs6deokVq1cgbKyUri7u2NUfAJ0jRvj5s2bak+NiMiuSN7o27NnT731gwcPlj0ZIiI5MjLOYNXKVJSWlsDNzQ0jR8Zj4KDBvL4UEZEMO3bswJo1a9CrVy9otVpERkZi5MiRCAgIwDvvvIOxY8eqPUUiciGVlZXYtHE9Dh/+FQAQFBSMGQ89gvDwCAiCoPLsiIjsj+SNvqFDhxqV3XutPp7OQUS2dPxYOnbv3gEACAkJxbTpDyE8nHeFJCKSq7S0FCEhIQCAwMBA5OXloX379ujWrRuOHDmi8uyIyJXoa2rw2WefIP/mTWg0GgwcNBjdu0UjNDRM7akREdktyRt9t2/frvV1dXU1jh49ipdeeglvvfWWYhOzJ6Io1vprkSAIRmW2oMS4Uvswt31D7UzVSyk3t8wWmIV9ZCEIAlq1bo2DB9PQo0cs4kbGw8PDQ/IcmIXllBqTWViOWbhuFko9tg4dOiAjIwNRUVGIjo7Gv//9b0RFRWHx4sUIDw9XZAx7IwiC4Z+9vo4dec0jpx8lspBTxyzkt7dGFlo3N3TrFo2jRw5j8pRpiIyMQl5eHrPgcdHgnK3dD7Mwxiysm4WUPiRv9Ol0OqOykSNHwtPTEwsWLMDhw4eldml3kpOTkZycbPh0YmFhIXJzcw2nAQqCgMLCQoiiaNNTA5UYV2of5rZvqJ2peinl5pbZArNQLwtBEHD58iW0atUagiCgpkaPhx+eDV8/P6M/REjpk1lYRqkxmYXlmIXrZlFcXCx7jHs9/fTTyMrKAgC88sorGD16NJYsWQJPT0988803ioyhtvvXenl5eaioqLDr17Ejr3nk9KNEFnLqmIX89kplcfv2LYiiiMaNA1FYWIhOnbqgffuO8PLyQm5uLrPgcWH247FmP8zCGLOwbhZS1nmSN/pMCQ0NRUZGhlLdqSopKQlJSUkoKiqCTqeDTqdDSEhIrfA0Gg2Cg4NtfuBYOq7UPsxt31A7U/VSys0tswVmoU4W+TdvYtXKFFy7dhUzZ85Bu/YdFBmTWVhOqTGZheWYhetm4e3tLXuMe82cOdPw/9jYWFy+fBlnzpxBy5YtERQUpMgYart/rRccHIyAgAC7fh078ppHTj9KZCGnjlnIb29pFkFBQTj060H89NNGNG0ahMd+/ydmIbM9jwvr98MsjDEL62YhZZ0neaPv+PHjtb4WRRFZWVl49913ERMTI7U7h6DRaKDVamsFU1eZWnOxdh/mtm+onal6KeXMwjWzEEURv/76CzZuWIfq6mp4e3tDEARotVrFxmQWlmMWzIJZGLNlFtZ6XL6+vujZs6dV+rYX9z639vw6duQ1j5x+lMhCTh2zkN9ebhZlZaVY8sMGnDt3FgDg7++PmppqZmFBex4X1u+HWRhjFtbLQsr3S97oi4mJgUajgSiKtcr79euHr776Smp3RET1KikpwaqVKThz5jQAoHXrNpg8ZRoaNw6EIPBOa0RESliwYIHZbd9//30rzoSIXM2pUyexamUqKioq4O7ujvjRY9Cv3wAAQElJqcqzIyJyPJI3+jIzM2t9rdVqERwcrNjpIkREd509m4HUlGUoLS2Bm5sbRo0ajQEDH7D5X6aIiJzd0aNHzWqn0WisPBMichXV1dVYu3YVjhw+BAAICw/H9GkPIyQ0FAD4B10iIpkkb/RFRkYalRUUFHCjj4gUV11djdLSEoSGhWHa1IcQ5qR3eyQiUtvOnTvVngIRuRg3Nzfk37wJjUaDnj174cHE8fD09FR7WkREDk/yRt8//vEPREVFYfr06QCAadOmITU1FeHh4di4cSOio6MVnyQRuY7Kygp4ed35w0GXLl0xfcbD6NSpCzw8PFSeGRERERFZoqamBlqtFu7u7tBqtZgydQYKCm7D19cP7u6K3SeSiMilST7/bfHixWjRogUAYOvWrdi6dSs2b96MhIQEPP/884pPkIhcg16vx7ZtW/D+e4tQVFRkKO/ePYabfEREREQO7tatfHzx+Wf46aeNhrImTZogKqqVirMiInI+kv9skp2dbdjoW79+PaZNm4ZRo0YhKioKffv2VXyCROT8bt7Mw/LlS3H92lUAwPFjRzHogSEqz4qIiIiILCWKIn5O24/NmzdCr9ejoOA2hg0bAV9fP7WnRkTklCRv9AUGBuLq1ato0aIFNm/ejDfffBPAnTdwvV6v+ASJyHmJooiDB3/Gpo3rUV1dDW9vH4wfPxHdo2PUnhoRERERWaioqBArVqTg/LmzAIC27dpj8uSp3OQjIrIiyRt9kyZNwsMPP4x27dohPz8fCQkJAO7cra1t27aKT5CInFNJcTFWrkxBRsYZAEDr1m0xeco0NG7cWN2JEREREZHFTp44jtWrV6K8vAzu7u4YMPABxMWNgpubm9pTIyJyapI3+j744ANERUXh6tWrWLRoEfz9/QEAWVlZePzxxxWfIBE5p717dyMj4wzc3d0xatRo9B8wCFqt5MuGEhEREZGdKSsrw6pVqaioqEB4RASmTJkBANBoNCrPjIjI+Une6PPw8MBzzz1nVP7ss88qMiEicg0j4kbh9u1bGD5iJMLCwtWeDhEREREpxNfXF+PGT0ROdjaGjxgJrVaL3NxctadFROQSZN/D/NSpU7hy5QqqqqpqlY8bN87iSRGR87ly5TIOH/oV4ydMglarhaenJx5+ZLba0yIiIiIiC9XU1GDbti1o3LgxQkJCAADR0T2A6Dv1giCoODsiItcieaPv4sWLmDhxIk6cOAGNRgNRFAH872PYvCEHEd1Lr9dj545t2LVrB0RRRESzZujbt7/a0yIiIiIiBeTm5GB5yo/IunEDvr6+iI7uAR8fH7WnRUTksiRfEOvpp59Gq1atkJubC19fX/z222/Ys2cPevXqhV27dllhipZZv349OnTogHbt2uHLL79UezpELiUvLxf//ven2LlzO0RRRExMT0TzjrpERKQgrvWI1CEIAg4c2Ifk5I8Mm3xDho6Al5eX2lMjInJpkj/Rl5aWhh07diAoKAharRZarRaDBg3CO++8g6eeegpHjx61xjxlqampwYIFC7Bz507odDrExsZi4sSJaNq0qdpTI3Jqoiji4C8/Y9Om9aiuroaPjw/Gj5+Ebt2j1Z4aERE5Ea71iNRRWFiIlSuW4/z5cwCAdu07YOLEySgvr1B5ZkREJPkTfXq9Ho0aNQIABAUF4caNGwCAyMhIZGRkKDs7Cx08eBBdunRBs2bN4O/vj4SEBGzZskXtaRE5vQ3r12Lt2lWorq5Gm7bt8ORTC7jJR0REiuNaj8j2ioqK8MnHH+D8+XPw8PBAYuIEzJkzH40aBag9NSIigoyNvq5du+LYsWMAgL59+2LRokXYv38/Xn/9dbRu3VrRye3ZsweJiYmIiIiARqPB6tWrjdokJycjKioK3t7e6Nu3Lw4ePGiou3HjBpo1a2b4ulmzZrh+/bqicyQiYz16xsLLywtjx47D3LmPQqfTqT0lIiKyQ1zrETmegIAAdOzYCRHNmiEp6Wn06z/AcL12IiJSn+SNvr///e+Guya9/vrryMzMxAMPPICNGzfi448/VnRypaWliI6ORnJycp31y5Ytw4IFC/DKK6/gyJEjiI6ORnx8PG/dTmRjlZUVOHf2f5/obdasOZ7/y18xYOAgaLWS32aIiMhFcK1H5BiuX7+G4uJiw9eJ4ybgj398AsH/d4ddIiKyH5Kv0RcfH2/4f9u2bXHmzBncunULgYGBiv8lJyEhAQkJCSbr33//fTz22GOYN28eAGDx4sXYsGEDvvrqK7z44ouIiIio9Vfd69evo0+fPib7q6ysRGVlpeHroqIiAHeuN3bvLeEFQTAqswUlxpXah7ntG2pnql5KublltsAs/ld2+fIlrEhdjqKiQkyZOgNBQUEAAC8vL6vnolT+zpKF3MejBGbBLJiFMVtnYeufhUqwl7WeIAiGf/b6OnbkNY+cfpTIQk4ds6itpqYG27Ztwf59e9C+fQfMnDUXGo0GHh4ehr7M6ZdZKNMPjwtjzIJZuEoWUvqQvNF31/nz53HhwgUMHjwYTZo0gSiKcruSpaqqCocPH8bChQsNZVqtFnFxcUhLSwMA9OnTBydPnsT169eh0+mwadMmvPTSSyb7fOedd/Daa68ZlRcWFiI3N9fwySRBEFBYWAhRFG36aSUlxpXah7ntG2pnql5KublltsAsBNy+fRtpaftx5PCvEEUR/o0aoaCgoNaxYm1K5e/oWdjDccEsmAWzMGbrLO79tI0zsOVaLy8vDxUVFXb9OnbkNY+cfpTIQk4ds/if/Pyb2PLTJty8mQcAcHNzR3ZWFtzc6/4VkllYvx8eF8aYBbNwlSykrPMkb/Tl5+dj2rRp2LlzJzQaDc6dO4fWrVvj0UcfRWBgIN577z2pXcpy8+ZN6PV6hIaG1ioPDQ3FmTNnAADu7u547733MGzYMAiCgL/85S/13oVt4cKFWLBggeHroqIitGjRAjqdDiEhIbXC02g0CA4OtvmBY+m4Uvswt31D7UzVSyk3t8wWXD2LnJxsbN26CXn/d+pUdEwPjBmTiOLiYptmoVT+jpyFvRwXzIJZMAtjts7C29tb9hj2yJZrveDgYAQEBNj169iR1zxy+lEiCzl1zOJO/S8/H8CWLZtRU1MDX19fDB06An379WcWZjwea/bD48IYs2AWrpKFlHWe5I2+Z599Fh4eHrhy5Qo6depkKJ8+fToWLFhgs40+c40bNw7jxo0zq62Xlxe8vLysPCMix/brr79g08b1qK6uho+PD8aNn4iuXbtDEASn+zQJERHZP671iJRTUlKC1JSluHDhPACgXfsOGD9+EioqKlSeGRERmUvyRt+WLVvw008/oXnz5rXK27Vrh8uXLys2sYYEBQXBzc0NOTk5tcpzcnIQFhZmUd/JyclITk6GXq8HwFN37fUUFjmPRymunEV+fj6qq6sRHh6BUfFjEBAQgNzcXFWycOSPh9dX74jHBbNgFszCmK2zcLY/tthyrcdTd01z5PcUOXWunkVVVRXy8vLg7u6OQYOGoGu37igvL2cW9+FxwSyYhTFmYd0srHrqbmlpKXx9fY3Kb926ZdO/kHp6eiI2Nhbbt2/HhAkTANx5Urdv344nnnjCor6TkpKQlJSEoqIi6HQ6nrprp6ewyHk8SnG1LKqrqwzHwKhRoxEREYGQkDDVjwtH/nh4ffWOeFwwC2bBLIzZOgtnO3XXlms9nrprmiO/p8ipc8UsKisr4eHhYejzkZmz4eHhieDgYEnjMgvr98MsjDELZuEqWUhZ50ke6YEHHsB3331n+Fqj0UAQBCxatAjDhg2T2l29SkpKkJ6ejvT0dABAZmYm0tPTceXKFQDAggUL8MUXX+Dbb7/F6dOn8ac//QmlpaWGO7MRkWUqKyuxetUKLF/2I6qqqgDcuRB6167dodEoe5dtIiJyPVzrEakrM/MiPvn4Axz8Jc1QFhHRzLDJR0REjkfyJ/oWLVqEESNG4NChQ6iqqsJf/vIX/Pbbb7h16xb279+v6OQOHTpUa/Pw7sWT58yZg2+++QbTp09HXl4eXn75ZWRnZyMmJgabN282umizVDx1V3r7htpJ+birqXJzy2zBFbK4ceM6tm7ZjKKiQgDAkSOH0bZtO5N9qJGFUmPaexb1lTMLZmHuY7F2P8zCmK2zcMRTd+1lrcdTd01z5PcUOXWuksWtW7ewf98eHD16GADw888HENWqjaTXs5R2zEKZfpiFMWbBLFwlC6ueutu1a1ecPXsW//rXv9CoUSOUlJRg0qRJSEpKQnh4uNTu6jV06FCIolhvmyeeeMLi0zfux1N3pbdvqJ2Uj7uaKje3zBacOYvAwEDs3r0Te/fsgiiKaNy4MYYPH4nomB52l4Ujfzy8vnpHPC6YBbNgFsZsnYUjnrprL2s9nrprmiO/p8ipc4UssrJuYOuWTbh5Mw8AEBvbGwljHjR5GSZmYYzHBbNgFsaYhXWzsOpddwFAp9Phb3/7m5xvdUgajQZarbZWMHWVqTUXa/dhbvuG2pmql1LOLKybxe3bt7ByxXLcuHEdANCjZyzGjElEUVGR3Wah1Jj2loUjHhfMglkwC2O2zMLWPwedyb3PrT2/jh15zSOnHyWykFPnrFkIgoC0A/uwZctm1NTUwNfXDxMnTUHnzl0UG5dZWL8fZmGMWTALV8hCyvfL2uirqKjA8ePHDXfavNe4cePkdGnXRFGs9TgFQTAqswUlxpXah7ntG2pnql5KublltuCsWRzYvxc3blyHj48vxk+YiC5dutl1FkqNaY9ZONpxwSyYBbMwZussbP2z0JkIgmD4Z6+vY0de88jpR4ks5NQ5cxa5uTnYvHkjBEFAZFQrTJs2AwEBOotf7+a0YxbK9MMsjDELZuEqWUjpQ/JG3+bNmzF79mzcvHnTqE6j0RiudeLIeI0+6e0bameqXkq5uWW24KxZ9OjZG25ubnhg8DD4+/sbNvPtNQulxrTHLBztuGAWzIJZGLN1Fo54jT618Bp95nPk9xQ5dc6dhQb9BwyCu7s7mjdvibKyclRUVCo2LrOwfj/MwhizYBaukoVVr9H35JNPYurUqXj55ZctvhCyveI1+qS3b6idqXop5eaW2YKzZHH69G+4dvUq4kePMZRHR0c7TBZKjWkPWTj6ccEsmAWzMGbrLBzxGn1q4TX6zOfI7yly6pwpi/LycmzauA4DBj6AsLA711If/X9rvry8PGZhAR4XzIJZGGMW1s3Cqtfoy8nJwYIFC5x2k68u5px3reZcrN2HEue711dvyfnuch6PUhw5i+qqKqxelYqjR48AANq1a4/Wbdo6ZBaOfB2I+uqZBbOwBLNwzSxs/XPQmdz73Nrz69iR1zxy+lEiCzl1zpDFxQvnkZq6HIWFBcjOycbjjz9l0fshszDG44JZMAtjzMJBr9E3ZcoU7Nq1C23atJH6rURkBy5dykTK8h9RVFQEjUaDwUOGIapVa7WnRUREREQWqqmpwdatm7F/316IooimTZti/PhJ/EMAEZELkbzR969//QtTp07F3r170a1bN3h4eNSqf+qppxSbnL1o6AKLtqLEuFL7MLd9Q+1M1UspN7fMFhwxi5qaGuzYsQ379u6GKIoIDAzE5CnTERkZVe/32XMWSo3J48JyzIJZMAtjts7C1j8LnYkg8GYcpjjye4qcOkfOIjs7G6mpS5GTnQ0A6NWrD0YnjIWXl5dF74fMwhiPC2bBLIwxC+tmIaUPyRt9P/74I7Zs2QJvb2/s2rULGo3GUKfRaJxio48345DevqF2puqllJtbZguOmMWa1Stw5cplAECbNu0wfEQcvL19kJubW+/32XMWSo3J48JyzIJZMAtjts6CN+MwH2/GYT5Hfk+RU+eoWeTkZCM1ZRkEQQ8fHx+MGDEKrVq3QWFhoaR+pI5rbjtXysKa/TALY8yCWbhKFla9Gcff/vY3vPbaa3jxxRed9iPgvBmH9PYNtTNVL6Xc3DJbcMQsBj0wGKtXrUDiuAkIDg51iiyUGpPHheWYBbNgFsZsnQVvxmE+3ozDfI78niKnzlGzCAoKwi8/H4C3jw8mTJgMf39/Wf1Y0p5ZWL8fZmGMWTALV8nCqjfjqKqqwvTp0512k68u5lxgUc25WLsPJS5sWV+9JRe2lPN4lGLvWZSUFOP2rVto1frO9TS7dOmGNm3awdPT0/AJVWfIQqkxeVxYjlkwC2ZhzJZZuNLaTGn3Prf2/Dp25DWPnH6UyEJOnaNkcerUSbRr1wEeHh7QarWYPWc+vLy8oNFoJPUjdVy57Zw5C1v2wyyMMQtm4QpZSPl+ySPNmTMHy5Ytk/ptRGRD586dxb8++RBLlnyPoqL/nbLBT3sQERERObby8jIsW/pfLPnhO/y0eaOh3Nvb26xNPiIicm6SP9Gn1+uxaNEi/PTTT+jevbvRzTjef/99xSZnLxq6wKKtKDGu1D7Mbd9QO1P1UsrNLbMFe82ioqIC69etwbFjRwEAERHNUFVZZdZz5ohZKDUmjwvLMQtmwSyM2ToLW/8sdCaCwJtxmOLI7yly6uw9i6tXLmPHjm0oKiqEVquFj48P9Hq9pA0+ZmE5HhfMglkYYxbWzUJKH5I3+k6cOIEePXoAAE6ePFmrzln+gsSbcUhv31A7U/VSys0tswV7zOL69WvYumUziouLoNFo0DO2N/r27Y8avd5ww436+nHELJQak8eF5ZgFs2AWxmydBW/GYT7ejMN8jvyeIqfOXrOoqanBgf17DX/M1ekaY1R8AsLCwpGXlyepL2ZhOR4XzIJZGGMW1s3Cqjfj2Llzp9RvcTi8GYf09g21M1UvpdzcMluwpyxEUcTWLZuxb98eiKKIwMBADB8+Et2jY5w+C6XG5HFhOWbBLJiFMVtnwcszmI834zCfI7+nyKmzxyzy8nKxYsUy5ObkAAB69eqDhDEPwtPTU1Z/zMJyPC6YBbMwxiysm4VVb8bhisy5wKKac7F2H0pc2LK+eksubCnn8SjFnrKorKqCKIqIje2NhDEPorCw0GWyUGpMHheWYxbMglkYs2UWtv456EzufW7t+XXsyGseOf0okYWcOnvLwtvbG0WFRfDz88ew4XHo27cfs+BxwSzMmLO1+2EWxpiFfdyMgxt9RA5IEARUVVXC29sHAJCQMBYdOnREx46dIAi8RhMRERGRIysvL4ePz511nk7XGDNnzkbToGCUlZWpPDMiIrJ3/NMvkYMpKirEN9/8B0t++M6wqefp6YmOHTupPDMiIiIisoQoikhPP4J//r93cObMKUN5q9Zt4O/vr+LMiIjIUfATfUQO5NzZDOzevQPl5eXw8PBATk42wsMj1J4WEREREVmovLwMa1avwokTxwAAv/56EB07dlZ5VkRE5Gi40WeGhm6ZbCtKjCu1D3PbN9TOVL2UcnPLbMHWWVRUVGDdutU4fiwdANCsWXNMnjIdwcHBZj+nDdU7YhZKjcnjwnLMglkwC2O2zoKXbpBPEATDP3t9HTvymkdOP0pkIadOrSwuXDiPlSuWo6ioCFqtFkOHjcDgwUOt8j7GLCzHLJgFszDGLKybhZQ+uNFXh+TkZCQnJ0Ov1wMACgsLkZuba7j4oSA47u2qpfZhbvuG2pmql1Jubpkt2DKL69euYuvWzSguLoZGo0Fsrz7o06cfRFFEbm6u5H6dKQulxuRxYTlmwSyYhTFbZ1FcXCx7DFdz/1ovLy8PFRUVdv06duQ1j5x+lMhCTp2ts6ipqUHagX1ITz8CAGjcOBCj4hMQGhqG/Px8Wc9JQ5iF5ZgFs2AWxpiFdbOQss7jRl8dkpKSkJSUhKKiIuh0Ouh0OoSEhNQKz1FvVy21D3PbN9TOVL2UcnPLbMFWWQiCgOXL/ovi4mI0adIEw4aPRPfu0cyigTlbux8eF8aYBbNgFsZsnYW3t7fsMVzN/Wu94OBgBAQE2PXr2JHXPHL6USILOXW2zuLs2QzDJl+fPv0QP3oMPD09JT9WKZiF5ZgFs2AWxpiFdbOQss7jRp8ZzLllsppzsXYfStyqur56S25VLefxKMUWWWi1WkyZOgMHD6YhPn4MCgsLmcV9lBqTx4XlmAWzYBbGbJmFrX8OOpN7n1t7fh078ppHTj9KZCGnzpZZdOzYCYMeGIxWrdqYdWM1ZsHjglkYYxbMwhWykPL9XBES2RFBELB/3x7s27fHUBYREYEJEybDy8tLxZkRERERkaVu376NJT98h6KiIkNZQsKDZm3yERERmYOf6COyEwUFBViRuhwXL56Hm5sbOnbshKCgYLWnRUREREQWEkUR6elHsG7tGlRWVkCr1eChh2epPS0iInJC3OgjsgPHj6VjzZpVqKgoh4eHB8aMSUTTpkFqT4uIiIiILFRWVoY1q1fi5MnjAICWLSMxKn6MyrMiIiJnxY0+IhVVVlYgZflSHD+eDgBo1rwFpk2bwU/yERERETmB8+fOInXFchQXFUGr1WL48JEYPGQo3Nzc1J4aERE5KW70EamkpqYGy5f9FwUFBdBqtRgydDiGDRvBhR8RERGREzhx/BiWLl0CAAgKCsbUaTPQvHkLlWdFRETOjht9ZhBFEYIgGL4WBMGozBaUGFdqH+a2b6idqXop5eaW2YIS42q1WnTtGo1Tp05gytQZaNGipaFvS8Z0pSyUGpPHheWYBbNgFsZsnYWtfxY6E0EQDP/s9XXsyGseOf0okYWcOiWzaNuuPZo2DUKbtm0RHz8Gnp6eNl3HK9WPM2ShNGbBLJiFMWZh3Syk9MGNvjokJycjOTkZer0eAFBYWIjc3FzD7YwFQUBhYSFEUbTp7aqVGFdqH+a2b6idqXop5eaW2YLccW/ezIMgCAgJCYUgCIiMaoXOXbrCy8sLubm5iozpSlkoNSaPC8sxC2bBLIzZOovi4mLZY7ia+9d6eXl5qKiosOvXsaOteSztR4ks5NRZkoUgCDh3NgPtO3SERqMBAEyZOgOenp4oKCho8DE3hFnwuGAWxpgFs3CVLKSs87jRV4ekpCQkJSWhqKgIOp0OOp0OISEhtcLTaDQIDg62+YFj6bhS+zC3fUPtTNVLKTe3zBbkPI9pB/Zh69af0LhxIB5Pegru7u7MwkJKjcnjwnLMglkwC2O2zsLb21v2GK7m/rVecHAwAgIC7Pp17ChrHqX6USILOXVys7h9+xZWpC7H5cuX4ObuhgEDBpn93JiLWfC4YBbGmAWzcJUspKzzuNFnBo1GA61WWyuYusrUmou1+zC3fUPtTNVLKXfELAoKCrAidRkuXrwAAAgKDoYg6KHVejILBSg1JrOwHLNgFszCmC2zsPXPQWdy73Nrz69je1/zKN2PElnIqZOShUajwdGjh7F+3RpUVlbC09MTfr5+VsuIWfC4YBbGmAWzcIUspHw/N/qIrOhY+lGsXbsKFRUV8PDwwJixiejduy80Gg0EgddSIiIiInJUZWWlWLd2NU6ePAEAiIyMwpSp09GkSVOVZ0ZERK6MG31EVlBdXY2VK1Jw/Hg6AKBFi5aYMnU6goKC1Z0YEREREVns+vVr2LplE4qLi6HVajEibhQGDx7KT9YSEZHquNFHZAXu7u4oryiHVqvF0GEjMHTocLi5uak9LSIiIiJSgIeHB0pLSxEUFIyp02agefMWak+JiIgIADf6iBRTXV0NURTh6Xnn2nuTJk1FYWEBWrRoqfbUiIiIiMhCZWWl8PdvBAAICQnFrFlzEdWqNTw9PVWeGRER0f/ws+VECsjOysJnn36C9evWGMoCAgK4yUdERETk4ARBwKFfD+K9f/4DWVk3DOVt27XnJh8REdkdfqKPyAKCIGD//r3YumUz9Ho9SkpKUFpSAj9/f7WnRkREREQWunXrFlJTluLy5UsAgGPH0jFq1Gh1J0VERFQPbvQRyVRUVIT161YjM/MiAKBjp86YOHEKN/mIiIiIHJwoijhy+BDWr1+DyspKeHh4IjFxPHrG9oIoimpPj4iIyCRu9BFJJIoi0tOPYt3aVaiqqoKnpyfGjE1Er159oNFo1J4eEREREVmgvLwcS3/8AadO/QYAiIyMwpChI9CuXTtoNBpu9BERkV3jRp8ZRFGEIAiGrwVBMCqzBSXGldqHue0bameqXkq5uWXWVl5ejs2b1qOqqgrNm7fAlKnT0bRpEERRlLTwYxaWU2pMZmE5ZsEsmIUxW2dh63WJMxEEwfDPXl/Hjrz+lNOPElnIqRMEARkZp3Hq1G9wc3PD8OEjMWDgIOTn5zMLFbLgcSGvPbOwfj/MwhizsG4WUvrgRl8dkpOTkZycDL1eDwAoLCxEbm4utNo79y4RBAGFhYUQRdFQZgtKjCu1D3PbN9TOVL2UcnPLbGHI0BG4du0qBg58AHq9gNzcXMl9MAvLKTUms7Acs2AWzMKYrbMoLi6WPYaruX+tl5eXh4qKCrt+HTvy+lNOP0pkIadOEAS0bBmFzp27olv3aISEhOLmzZvMQqUseFzIa88srN8PszDGLKybhZR1Hjf66pCUlISkpCQUFRVBp9NBp9MhJCSkVngajQbBwcE2P3AsHVdqH+a2b6idqXop5eaWKa26uhpbt25GixaR6NatOwAgKCgIrVu3YRY2zsLcx2LtfpiFMWbBLJiFMVtn4e3tLXsMV3P/Wi84OBgBAQF2/Tp25PWnnH6UyMLcuqysG9i1awemTp0Bd3d3aDQaTJ/xMLOQ2F6JLHhcKNOeWVi/H2ZhjFlYNwsp6zxu9JlBo9FAq9XWCqauMrXmYu0+zG3fUDtT9VLKbZ3FjRs3kLL8R+Tm5iDd5wg6dOgAb28fxcZlFpZTakxmYTlmwSyYhTFbZmHrNYkzufe5tefXsSOvP+X0o0QW9dWJoog9e3Zh545tEAQBu3fvxMiR8czCgvZys+BxwSwswSyYhStkIeX7udFHVAdBELBv3x5s2/oT9Ho9/P39MWnyVMMmHxERERE5rlu38rFyxXJkZd0AAHTt2g0PPDBY5VkRERFZjht9RPe5ffs2UlOW4tKlTABAp05dMHHiZPj5+6s8MyIiIiKyhCiKOHz4EDasX4Oqqip4eXkhMXECYnr0hEajgSDwpjZEROTYuNFHdI+SkhL865MPUFFRAU9PT4x9cBxiY3tDo9GoPTUiIiIistCuXTuwbetPAICIiGaY8dAjaNo0SOVZERERKYcbfUT38Pf3R3RMT2TduI4pU2egadOmak+JiIiIiBTSs2cv/Jy2H/0HDEL79h0RGNhE7SkREREpiht95PLOnzuLpkHBCAwMBAAkJIyFVquFm5ubyjMjIiIiIktUV1fj2LF09OjREwCg0+nw5+dehLu7O3Jzc1WeHRERkfK40Ucuq7q6Gj9t3oi0tP1o1ao15j/6e2i1Wnh4eKg9NSIiIiKy0LVrV7Fs6X9RUHAb3t5e6NSpCwDA09OT1+IjIiKnxY0+ckk3blxHyvKlyM3NAQCEhIZBEASb336ciIiIiJSl1+uxe/dO7NyxDYIgoFGjAHh6eqk9LSIiIpvgRh+5FEEQsHfvbmzftgV6vR7+jRph8uRpaN++g9pTIyIiIiIL5efnIzVlKa5cuQwAaNuuPaZOnQF/f3+VZ0ZERGQb3Ogjl1FSXIwff/wBly5lAgC6dOmK8RMmw8/PT+WZEREREZGljqUfxerVK1BVVQUvL288mDge4eER8PX1VXtqRERENsONPnIZXt7eKC8vh6enJxITJ6BHz1hoNBq1p0VERERECvDw9EBVVRVatWqNyVOmQ6fT8YYbRETkcrjRR06trKwMXl5ecHNzg4eHB2bMeATuHu5o0qSp2lMjIiIiIguVlpYY/t+5c1fMnjMf7dq1h1ar5Q03iIjIJfHOA+S0zp3NwMcfvYc9e3YZykJCQ7nJR0REROTgqqqqsGb1Snz80fsoKfnfZl+HDh15czUiInJp/EQfOZ2qqir8tHkjfv75AADgxIljGDx4KNzc3FSeGRERERFZ6urVK0hZvhT5+TcBAJcvZ6J169Yqz4qIiMg+uMSfuyZOnIjAwEBMmTJF7amQlV2/fg2fJn9s2OTr128A/vjHJ7jJR0RE5KS4znMder0eO7Zvxef//hT5+TcREKDD3Hm/Q5cu3dSeGhERkd1wiU/0Pf3005g/fz6+/fZbtadCViIIAvbs2YXt27ZAEAQ0atQIkydPQ7v2HdSeGhEREVkR13mu4ebNPKSmLMPVq1cAAN27R2Pc+Inw8vLmDTeIiIju4RIbfUOHDsWuXbvUngZZ0e1bt7BzxzYIgoAuXbphwsRJ8PX1U3taREREZGVc57mGn9MO4OrVK/D29sa4cRMRHdMDAHjDDSIiovuofurunj17kJiYiIiICGg0GqxevdqoTXJyMqKiouDt7Y2+ffvi4MGDtp8o2bWmQUEYO3YcJk+ZhocenslNPiIiIjvAdR4pZVR8AnrG9sKTTz1r2OQjIiIiY6p/oq+0tBTR0dGYP38+Jk2aZFS/bNkyLFiwAIsXL0bfvn3x4YcfIj4+HhkZGQgJCQEAxMTEoKamxuh7t2zZgoiICLPnUllZicrKSsPXRUVFAABRFGv9tVAQBKMyW1BiXKl9mNu+oXam6qWU31tWWlqKdetWo3//gfD29oEgCOjVuw+AO3mJomjW45OLWZhXZm1KjcksLMcsmAWzMGbrLOzxk032tM4DTK/1BEEw/LPX17Ejrz/l9HP69Cn8evAXPPzILLi7u8Pd3R0TJ04x9GVOv3LqmIX89szC+v0wC2PMglm4ShZS+lB9oy8hIQEJCQkm699//3089thjmDdvHgBg8eLF2LBhA7766iu8+OKLAID09HRF5vLOO+/gtddeMyovLCxEbm4utNo7H4AUBAGFhYUQRdFQZgtKjCu1D3PbN9TOVL2U8rtlly5lYsf2rSgrK8WN69eRMCaRWUhop2QWDZVZm1JjMgvLMQtmwSyM2TqL4uJi2WNYiz2t8wDTa728vDxUVFTY9evYkdefUvqprq7G3r278NvJEwCAPXt2oWvX7rL6lVPHLOS3ZxbW74dZGGMWzMJVspCyzlN9o68+VVVVOHz4MBYuXGgo02q1iIuLQ1pamuLjLVy4EAsWLDB8XVRUhBYtWkCn0yEkJKRWeBqNBsHBwTY/cCwdV2of5rZvqJ2peinlFRUV2LN7J06cOAYACA4OweQp0+Du7sEsJLRTIgtzy6xNqTGZheWYBbNgFsZsnYW3t7fsMdRg63UeYHqtFxwcjICAALt+HTvy+tPcfq5evYLU1GW4lZ8PAIiJ6YmBAx+Al5eXrH7l1DEL+e2ZhfX7YRbGmAWzcJUspKzz7Hqj7+bNm9Dr9QgNDa1VHhoaijNnzpjdT1xcHI4dO4bS0lI0b94cKSkp6N+/v1E7Ly+vehcSpJ7r168hNWUZbt7MAwD06z8Ao0YlwM3NDXl5eSrPjoiIiKSy9ToP4FrPXun1euzetQO7d++EIAgICNBh4qQpaNQoAB4eHmpPj4iIyKHY9UafUrZt2yapfXJyMpKTk6HX6wHw1F21T2G5eTMPKct/hCAI8PHxQdzI0YiKaoXbt28zCxntLMmivmNAjSwc+ePh9dUzC2ZhCWbhulnY46m7tiB1nQcYr/V46q5ptngdb92yGWfOnAIAtG/fAUOGjoCnpycKCgqc/lQsKRz5/Z1ZKNMPszDGLJiFq2ThNKfuBgUFwc3NDTk5ObXKc3JyEBYWZrVxk5KSkJSUhKKiIuh0Op66q/IpLKGhoTjath3cPTzQv/8gtGzZklmolEV9z7saWTjyx8Prq2cWzMISzMJ1s3C0U3fVWucBxms9nrprmi1ex8OGj8DVq1cwZsyD6B4dI2nc+trJqXP1LCxpzyys3w+zMMYsmIWrZCFlnWe71GXw9PREbGwstm/fbigTBAHbt283eUoGOT5RFHH8WDqqqu7cFU+j0WDGQzMxbdpD8PHxUXl2REREpASu81xXSUkxfvvtpOHriIhm+PNzLxg2+YiIiEg+1T/RV1JSgvPnzxu+zszMRHp6Opo0aYKWLVtiwYIFmDNnDnr16oU+ffrgww8/RGlpqeHubNbAU3elt2+onbkfdy0vK8OOHVtx8eIFtG7TFgkJD9b7vDML6e2kfPTYVLm9ZOHIHw+vr55ZMAtLMAvXzcIeT921x3UewFN3pVD6dXzx4gXs3LEVlZWVmDbtYQSHhFg0bn3t5NS5UhY8LU4+ZsEsmIUxZmHdLBzq1N1Dhw5h2LBhhq/v3gltzpw5+OabbzB9+nTk5eXh5ZdfRnZ2NmJiYrB582ajCzcriafuSm/fUDtzPu56/txZrFqVipKSEri5uSEsNAzBwcFwc3Mz2QezkN5OykePTZXbSxaO/PHw+uqZBbOwBLNw3Szs8dRde1znATx1Vwqlxi0vL8fOndtw6v8+yRcaFoag4GCE1LPR5wqnYknhyO/vzEKZfpiFMWbBLFwlC4e66+7QoUMhimK9bZ544gk88cQTNpqRMY1GA61WWyuYusrUmou1+zC3fUPtTNXX1NRgw/q1OHjwZwBASEgopkydDjc3d7i5uTX4vDML6e1M1Uspt5cslBqTWViOWTALZmHMllnY+uegORxhnQeg1nNrz69jR13zXLlyGSkpS3ErPx8ajQaDBg1G3Mh4uLvX/6uIElnIqXPmLOT2wyyMMQtmwSyMMQvrZSHl+1Xf6HMEoihCEATD14IgGJXZghLjSu3D3PYNtTNVn52dhaU//oCCgtsAgP4DBmLkyNFwc3NDXl5eg887s5DezlS9lHJ7yUKpMZmF5ZgFs2AWxmydha1/FjoTQRAM/+z1deyoa57du3Zix46tEAQB/v6NMGXqdLRp09bQt6Xj1tdOTp0zZyG3H2ZhjFkwC2ZhjFlYNwspfXCjrw68Rp/09g21M1VfVlaOiopy+Pn5I25kPFq2jMTt27frbG9umS04YxZSyu0lC6XGZBaWYxbMglkYs3UW9niNPnvFa/SZz9Jx7z6v7dt3REyPnvDz80dubq5i49bXTk6dM2chtx9mYYxZMAtmYYxZWDcLh7pGnz3iNfqkt2+o3b31ZWWl8PdvBAAICgpC4riJaN26Dfz9/evtz9wyW3CWLO5/o5J77QE5j0cJSo3JLCzHLJgFszBm6yzs8Rp99orX6DOf1HFFUURpaYlhrTdyVDzatG2L1q3bIC8vj9dcsoAjv78zC2X6YRbGmAWzcJUsHOoafY7AnPOu1ZyLtftQ4nz3u44cOYRNG9djwsTJiI7uAQAICwuHv7+/0feZ+7wzC+ntpFxjwFS5vWThyNeBqK+eWTALSzAL18zC1j8Hncm9z609v47tfc1TUlyMlStTkJ9/E0lPPANPT09otVq0b9/B8MuOrd9T5NQ5QxZK98MsjDELZsEsjDELXqOPXEhpaQk2bFiLzIsXAAAnjh83bPQRERERkWM7deo3rFqZirKyUri7u+Pqlcto07ad2tMiIiJyOdzoM0NDF1i0FSXGldqHue3ra5eRcQarV6WipKQEbm5uiIsbhQEDH4AgmL7odV3l5pbZgqNmUV+9I2ah1JjMwnLMglkwC2O2zsLWPwudyd01iT2/ju11zVNZWYlNG9fh8OFDAIDQsDBMnToDoaFhFh2DSmQhp86Rs7BWP8zCGLNgFszCGLOwbhZS+uBGXx14Mw7p7etqV11djX37duPkieMAAJ2uMUYnjEVISChu3rxZb/91lZtbZguOloU59Y6YhVJjMgvLMQtmwSyM2ToL3ozDfLwZh/nqGzcr6wa2/LQJRUWFAICesb3Qr+8AaDRaoxtuqPGeIqfOUbOwZj/MwhizYBbMwhizsG4WvBmHhXgzDunt62p38eIFwyZf//4DER3TE+Hh4UYHh9wLW8p5PEpxtCzMqXfELJQak1lYjlkwC2ZhzNZZ8GYc5uPNOMxX37g/bd6AoqJC6HSNMXnKNLRq1VpWP5a0r6+dnDpHzcKa/TALY8yCWTALY8zCulnwZhwKM+cCi2rOxdp9yL2wZdu27RAXF4+WLVuiVes2hk9F3t+PJRe2lPN4lOJIWZhb74hZOPIFX+urZxbMwhLMwjWzsPXPQWdy73Nrz69je1vzTJw0BTt37kB8/Gh4e/vI7sfS9vW1k1PniFlYux9mYYxZMAtmYYxZ2MfNOLgiJMUUFNzGt9/8B7dv3zKUDRs+ghdiJiIiInJwoiji558PYN3a1YaygAAdxo+faNYmHxEREdkGP9FHFhNFEYd+PYiNG9ehuroa69atwezZ89SeFhEREREpoLS0FJs3b8C5sxkAgK5du6FV6zYqz4qIiIjqwo0+MzR0JxVbUWJcqX001L6kpARrVq/AmTOnAQBRUa3w4IPjjdqb6kdKublltmCPWZjbzpmyUGpMZmE5ZsEsmIUxW2dh65+FzkQQeNddU347eQKrV69ARUUF3N3dMWrUaLRoGSl5Hmq8p8ips+csHPn9nVko0w+zMMYsmIWrZCGlD2701YF33TWvfWbmRWzftgXl5WXQat0Q06Mn+vUbgKqqKrPvtCal3NwyW7C3LKS0c6YslBqTWViOWTALZmHM1lnwrrvm4113G1ZVVYU9e3bi9KnfAABBQcEYFZ+Apk2DcPPmTcn9qfGeIqfOHrMw5/FYsx9mYYxZMAtmYYxZWDcL3nXXQrzrbsPtf/vtBNavWw0ACAkNxeTJ0+Dm5u4Sd1FUalylspDazpmyUGpMZmE5ZsEsmIUxW2fBu+6aj3fdrZ8oivji889w9eoVaDQa9OgRi7EPjoOnp6fsPtV4T5FTZ29ZmPt4rNkPszDGLJgFszDGLKybBe+6qzBz7qSi5lys3Udd7Tt16oKw8HC0adMWI0eOhpubm8m76jY0riV3sJHzeJRiL1nIaedMWTjynZ3qq2cWzMISzMI1s7D1z0Fncu9za8+vY1u+hocMHYb169Zi8uSp8PXzh6enp0O+p8ips7cszJmztfthFsaYBbNgFsaYhX3cdZcbfWQWvV6PQ4cOIja2N9zc3ODu7o4//vEJeHh4ALizc01EREREjik3NweFhYVo1649gDt/1G3btr3hj7lERETkGLjRRw0qKLiNVStTcO3aVZSUlGD48DgAMGzyEREREZFjEkURP/98AJs3bYCHhweefGoBdDodgDtrPf4xl4iIyLFwo88MDd1JxVaUGFdKH6Io4uDBn7F50wbU1NTA29sbTZo0rfN7G+rXVL2UcnPLbMHWWUhp70pZKDUms7Acs2AWzMKYrbPghox8guCad90tKirCqlWpOH/uLAAgKqoVAOscO2q8p8ipUysLczALZsEsjDELZuEqWUjpgxt9deBdd4GyslJs37YVly5dBAA0a9YcI0cloFGjRnWevtFQv6bqpZSbW2YLtsxCantXykKpMZmF5ZgFs2AWxmydBe+6az7edRc4f/4sdu7YhoqKCri5uWHgoMHo3j0G5eUVKC+vMPuxmkuN9xQ5dWpkYS5mwSyYhTFmwSxcJQvedddCrn7X3YsXL2D5sv+itLQUbm5u6Nd/IOLiRsHd3fTLpaF+TdVLKTe3zBZslYWc9q6UhVJjMgvLMQtmwSyM2ToL3nXXfK58111BELB61QocPXoYABAeHoEpU6cjJCRU1mOVMq6t31Pk1NkyC6mYBbNgFsaYBbNwlSx4112FmXMnFTXnonQfAQEBqKysRGhYGKZMmQGtVgt3d/cGx2yoXyl3qjFV7mpZyG3vSlk48p2d6qtnFszCEszCNbOw9c9BZ3Lvc2vPr2MlXk9arRaenp7QaDQYPHgoho8YWe8fc5UaV04/SmQhp85WWcjBLJgFszDGLJiFK2Qh5fu50UcAgKKiQgQE3LnwckhIKObNfwzNm7eAVqvlndaIiIiIHFhNTQ0qKyvh5+cHABidMBYxPXqiZctIlWdGRERESuOffl2cXq/Htq0/4Z//711cvnzJUB4V1arBv+4SERERkX3LzcnB4sX/wo///R6CcOdC3p6entzkIyIiclLcyXFhN2/mYfnypbh+7SoA4MzpU4iMjFJ3UkRERERkMUEQ8PPPB/DT5o2oqamBj48v8vNvIjg4RO2pERERkRVxo88FiaKIgwd/xuZNG1BdXQ1vbx+MnzAR3bvHqD01IiIiIrJQUVEhVqxIwflzZwEA7dp3wKRJUxEQEKDyzIiIiMjauNHnYoqLi7F+3WpcupQJAGjdui2mTJ0Gna6xuhMjIiIiIoudPHEcq1evRHl5Gdzd3ZGQ8CD69usPjUaj9tSIiIjIBrjRZwZRFA3XNAHunApxf5ktKDHumTOncelSJtzd3TFyZDz69R8IrVZrsk9zx2yonal6KeXmltmCEuNK7YNZGFNqTGZhOWbBLJiFMVtnYeufhc5EEATDP3t9HZszN71ej+3bt6K8vAwREc0wZep0BAeHQBRFiKJY72OS+1it1Y8SWcipUyoLa2AWzIJZGGMWzMJVspDSBzf66pCcnIzk5GTo9XoAQGFhIXJzcw23MxYEAYWFhRBF0aa3q1Zi3ObNW6BT5y6Iju6B4OAQ3Lx5U5ExG2pnql5KublltqDEuFL7YBbGlBqTWViOWTALZmHM1lkUFxfLHsPV3L/Wy8vLQ0VFhV2/js2d2/ARo3Dh/Fn07tMPogjk5uZKfn6kPFZr9aNEFnLqlMxCacyCWTALY8yCWbhKFlLWedzoq0NSUhKSkpJQVFQEnU4HnU6HkJCQWuFpNBoEBwfb/MCROu6VK5exfdsWPPTwLHh7e0MQBIwYMcrsPswds6F2puqllJtbZgtKjCu1D2ZhTKkxmYXlmAWzYBbGbJ2Ft7e37DFczf1rveDgYAQEBNj167iuNjU1NdixfSu8vX0weMhQAEBISAi6dOmixNNk1mO1Vj9KZCGnTm4WtsAsmAWzMMYsmIWrZCFlnceNPjNoNBpotdpawdRVptZc6qLX67Fjxzbs3rUDoihi187tGDM2UVIfUsdsqJ2peinljpiFkn0wC2NKjcksLMcsmAWzMGbLLGz9c9CZ3Pvc2vPr+N6vc3KykbJ8KbKybkCr1SI6pgcCAwMlP3ZzOPJ7ipw6qVnYErNgFszCGLNgFq6QhZTv50afE8rLy0XK8qW4fv0aACCmR08MHxGn8qyIiIiIyFKCIODntAP46aeNqKmpga+vLyZMnGK1TT4iIiJyLNzocyKiKOKXn9OwefMGVFdXw8fHB+MnTEa3bt3VnhoRERERWaikpBgbN6zDhQvnAADt23fAxElTERAQoPLMiIiIyF5wo8+J7Nm9E1u2bAYAtG3bDpMmT4NOp1N5VkRERERkqerqaixf9l+UlpbCw8MDoxPGom/f/tBoNGpPjYiIiOwIN/qcSK/effHrr79gwMAH0K/fAF6rh4iIiMhJeHh4oGfP3rh48TymTpuB4OAQtadEREREdogbfQ6ssrICx9LT0btPX2g0Gvj5+eGZZ5+HuztjJSIiInJ0169dRVVVJVq2jAQARMf0wIi4kfDw8FB5ZkRERGSvuCPkoC5fvoSU5Utx+/YteHp5IiamJwBwk4+IiIjIwdXU1GDrls3Yt28PmjRtiieeeAYeHh7QaDRwc3NTe3pERERkx7gr5GD0ej22bv0Je/fsgiiKaNw4EI11vMsaERERkTPIyc7G8pQfkZ2VBQBoFdVK5RkRERGRI+FGnwPJzc1FyvIfkZeXCwDo0SMWDyaOg7e3j8ozIyIiIiJLiKKIA/v3YuvWn1BTUwNfXz8MHTYC/fvfue6yIAhqT5GIiIgcADf6zCCKYq3FlSAIRmXWdvToYaxdswo1NTXw8fHBuPGT0LVrN8N8zCV17ua2b6idqXop5eaW2YIS4zILyyk1JrOwHLNgFszCmK2z4EaQfIIgGP6p8TouLy/H6lWpuHbtKgCgffsOGDd+EioqKgxtHXnNI6cfJbKQU2fO+wezkN6OWSjTD7MwxiyYhatkIaUPbvTVITk5GcnJydDr9QCAwsJC5ObmGu5iKwgCCgsLIYqize5sKwgiampqEB4egfj4MWgUEIDc3FwZ/Uibu7ntG2pnql5KublltqDEuMzCckqNySwsxyyYBbMwZussiouLZY/hau5f6+Xl5Rk21dR4Hev1egiCAHd3dwx6YAi6du2O8vLyWm0dec0jpx8lspBTZ877B7OQ3o5ZKNMPszDGLJiFq2QhZZ3Hjb46JCUlISkpCUVFRdDpdNDpdAgJCakVnkajQXBwsFUPnMKCAugaNwYAhISEoElgIHx8/WrNRSqpcze3fUPtTNVLKTe3zBaUGJdZWE6pMZmF5ZgFs2AWxmydhbe3t+wxXM39a73g4GAEBATY9HVcWVkJrVYLLy8vCIKA+NFj4e/vh5CQ0Dr7cOQ1j5x+lMhCTp057x/MQno7ZqFMP8zCGLNgFq6ShZR1Hjf6zKDRaKDVamsFU1eZUioqKrB+/Rr8dvIEnnxqAZo0aQIAaNW6jeGThZaMK3Xu5rZvqJ2peinlts6iPkqMyywsp9SYzMJyzIJZMAtjtszC1j8Hncm9z60tXseXLmVi5YoUtGnTFpOnTAMA+Pv7G/0x9/4+HHnNI6cfJbKQU2fO+wezkN6OWSjTD7MwxiyYhStkIeX7udFnZy5lZiI1dSlu374NjUaDixfPo0mTPmpPi4iIiIgsVFNTg337diP96BGIoohLlzJRXl4OLy8vtadGREREToIbfXaipqYG27dvxd49uyCKIgIDAzFl6gxERbVSe2pEREREZKHs7CwsX/4jcrKzAQC9evXBmLGJhlN3iYiIiJTAjT47kJuTg+UpPyLrxg0AQM+evTD2wXG81g4RERGRgxMEAQcO7MOWnzZBr9fDx8cHEydNRZcuXdWeGhERETkhbvTZgfT0I8i6cQO+vr6YMGEyunTtpvaUiIiIiEgBFRXl2LN7F/R6PTp06IhBDwzhGRtERERkNdzoswPDR4xEVVUVBg8ZhoCAALWnQ0REREQK8fX1w+QpU1FYWIjY2N7Iy8tTe0pERETkxLjRZwfc3d3xYOJ4tadBRERERFbQoUMnAOC1+IiIiMjqbHuvZSIiIiIiIiIiIrIKbvQRERERERERERE5AW70EREREREREREROQFu9BERERERERERETkBbvQRERERERERERE5AW70EREREREREREROQFu9BERERERERERETkBp9/ou3r1KoYOHYrOnTuje/fuSElJUXtKRERERKQArvOIiIiIanNXewLW5u7ujg8//BAxMTHIzs5GbGwsxowZAz8/P7WnRkREREQW4DqPiIiIqDan3+gLDw9HeHg4ACAsLAxBQUG4desWF4BEREREDo7rPCIiIqLaVD91d8+ePUhMTERERAQ0Gg1Wr15t1CY5ORlRUVHw9vZG3759cfDgQVljHT58GHq9Hi1atLBw1kRERETUEK7ziIiIiGxL9U/0lZaWIjo6GvPnz8ekSZOM6pctW4YFCxZg8eLF6Nu3Lz788EPEx8cjIyMDISEhAICYmBjU1NQYfe+WLVsQEREBALh16xZmz56NL774wuRcKisrUVlZafi6qKgIACCKIgRBMJQLgmBUZgtKjCu1D3PbN9TOVL2UcnPLbIFZ2EcWSo3JLCzHLJgFszBm6yxs/bPQHPa0zgNMr/UEQTD8s9fXsSOveeT0o0QWcuqYhfz2zML6/TALY8yCWbhKFlL6UH2jLyEhAQkJCSbr33//fTz22GOYN28eAGDx4sXYsGEDvvrqK7z44osAgPT09HrHqKysxIQJE/Diiy9iwIABJtu98847eO2114zKCwsLkZubC632zgcgBUFAYWEhRFE0lNmCEuNK7cPc9g21M1UvpdzcMltgFvaRhVJjMgvLMQtmwSyM2TqL4uJi2WNYiz2t8wDTa728vDxUVFTY9evYkdc8cvpRIgs5dcxCfntmYf1+mIUxZsEsXCULKes81Tf66lNVVYXDhw9j4cKFhjKtVou4uDikpaWZ1Ycoipg7dy6GDx+OWbNm1dt24cKFWLBggeHroqIitGjRAjqdDiEhIbXC02g0CA4OtvmBY+m4Uvswt31D7UzVSyk3t8wWmIV9ZKHUmMzCcsyCWTALY7bOwtvbW/YYarD1Og8wvdYLDg5GQECAXb+OHXnNI6cfJbKQU8cs5LdnFtbvh1kYYxbMwlWykLLOs13qMty8eRN6vR6hoaG1ykNDQ5GdnW1WH/v378eyZcuwevVqxMTEICYmBidOnKizrZeXFwICAmr9IyIiIiLl2XqdB3CtR0RERM7Prj/Rp4RBgwZBEKSdD52cnIzk5GTD9WCuX79u9HHMoqIiFBUV2XyH3NJxpfZhbvuG2pmql1JubpktMAv7yEKpMZmF5ZgFs2AWxmydRUlJCYA7n3JzFXLWeYDxWi8zMxP+/v52/Tp25DWPnH6UyEJOHbOQ355ZWL8fZmGMWTALV8lCyjrPrjf6goKC4ObmhpycnFrlOTk5CAsLs9q4SUlJSEpKwrVr19CiRQsMHz7camMRERERKam4uBg6nU7taTRIrXUeYLzW69mzp1XHIyIiIlKCOes8u97o8/T0RGxsLLZv344JEyYAuLNDun37djzxxBNWHz8iIgJXr17F8OHDcejQIUN5UdGd67lcvXrV5qd89O7dG7/++qtN+zC3fUPtTNVLKb+/jFnIa+dMWSiRg5x+mIUxZsEsmIUxW2YhiiKKi4sNd6K1d2qv84D/rfUaNWoEjUYDwH5fx46+5pHTjxJZyKljFvLbMwvr98MsjDELZuEKWUhZ56m+0VdSUoLz588bvs7MzER6ejqaNGmCli1bYsGCBZgzZw569eqFPn364MMPP0Rpaanh7mzWpNVq0bx5c7i7u9cZihrXdnFzc7N4TKl9mNu+oXam6qWUm2rLLKS1c6YslMhBTj/MwhizYBbMwpits7C3T/LZ8zoP+N9a7172/jp21DWPnH6UyEJOHbOQ355ZWL8fZmGMWTALV8nC3HWe6ht9hw4dwrBhwwxf370T2pw5c/DNN99g+vTpyMvLw8svv4zs7GzExMRg8+bNRhdutqakpCSbjdUQJeYitQ9z2zfUzlS9lHJmwSzup9Q8mIXlmAWzYBbG1MrCXjjCOu9+fB0bc+T3FDl1zEJ+e2Zh/X6YhTFmwSyYRW0a0ZWu2KyQoqIi6HQ6FBYW8m5tKmMW9oNZ2A9mYT+Yhf1gFuTo+Bq2H8zCfjAL+8Es7AezsB9qZWG7W7A4ES8vL7zyyivw8vJSeyouj1nYD2ZhP5iF/WAW9oNZkKPja9h+MAv7wSzsB7OwH8zCfqiVBT/RR0RERERERERE5AT4iT4iIiIiIiIiIiInwI0+IiIiIiIiIiIiJ8CNPiIiIiIiIiIiIifAjT4iIiIiIiIiIiInwI0+IiIiIiIiIiIiJ8CNPiu6evUqhg4dis6dO6N79+5ISUlRe0oubeLEiQgMDMSUKVPUnorLWb9+PTp06IB27drhyy+/VHs6Lo3HgX3gzwf7UVBQgF69eiEmJgZdu3bFF198ofaUiGSJiopC9+7dERMTg2HDhqk9HZeWmZmJYcOGoXPnzujWrRtKS0vVnpJLysjIQExMjOGfj48PVq9erfa0XNYHH3yALl26oHPnznjqqacgiqLaU3JZ//znP9GlSxd07doVP/zwg9rTcTmmfh9T8ndmjcgjzGqysrKQk5ODmJgYZGdnIzY2FmfPnoWfn5/aU3NJu3btQnFxMb799lukpqaqPR2XUVNTg86dO2Pnzp3Q6XSIjY3FgQMH0LRpU7Wn5pJ4HNgH/nywH3q9HpWVlfD19UVpaSm6du2KQ4cO8T2KHE5UVBROnjwJf39/tafi8oYMGYI333wTDzzwAG7duoWAgAC4u7urPS2XVlJSgqioKFy+fJk/a1WQl5eHfv364bfffoOHhwcGDx6Mf/7zn+jfv7/aU3M5J06cwJw5c3DgwAGIoohhw4Zh8+bNaNy4sdpTcxl1/T6m9O/M/ESfFYWHhyMmJgYAEBYWhqCgINy6dUvdSbmwoUOHolGjRmpPw+UcPHgQXbp0QbNmzeDv74+EhARs2bJF7Wm5LB4H9oE/H+yHm5sbfH19AQCVlZUQRZGfMiAi2e5uZDzwwAMAgCZNmnCTzw6sXbsWI0aM4CafimpqalBRUYHq6mpUV1cjJCRE7Sm5pNOnT6N///7w9vaGj48PoqOjsXnzZrWn5VLq+n1M6d+ZXXqjb8+ePUhMTERERAQ0Gk2dH+VOTk5GVFQUvL290bdvXxw8eFDWWIcPH4Zer0eLFi0snLVzsmUWJI2l2dy4cQPNmjUzfN2sWTNcv37dFlN3OjxO7IeSWfDng2WUyKKgoADR0dFo3rw5nn/+eQQFBdlo9kR3KPE61mg0GDJkCHr37o0lS5bYaObOx9Iszp07B39/fyQmJqJnz554++23bTh756Lkz9rly5dj+vTpVp6x87I0i+DgYDz33HNo2bIlIiIiEBcXhzZt2tjwETgPS7Po2rUrdu3ahYKCAty+fRu7du3i72YSWOv3MaV/Z3bpjb7S0lJER0cjOTm5zvply5ZhwYIFeOWVV3DkyBFER0cjPj4eubm5hjZ3r+lz/78bN24Y2ty6dQuzZ8/G559/bvXH5KhslQVJp0Q2pAxmYT+UyoI/HyynRBaNGzfGsWPHkJmZif/+97/Iycmx1fSJACjzOt63bx8OHz6MtWvX4u2338bx48dtNX2nYmkWNTU12Lt3Lz799FOkpaVh69at2Lp1qy0fgtNQ6mdtUVERDhw4gDFjxthi2k7J0ixu376N9evX49KlS7h+/ToOHDiAPXv22PIhOA1Ls7h7jcThw4dj0qRJ6NevH9zc3Gz5EByaw/w+JpIoiqIIQFy1alWtsj59+ohJSUmGr/V6vRgRESG+8847ZvdbUVEhPvDAA+J3332n1FSdnrWyEEVR3Llzpzh58mQlpumS5GSzf/9+ccKECYb6p59+WlyyZIlN5uvMLDlOeBwoS24W/PmgPCV+fvzpT38SU1JSrDlNonop8Tp+7rnnxK+//tqKs3QNcrI4cOCAOGrUKEP9okWLxEWLFtlkvs7MkuPiu+++Ex955BFbTNMlyMli+fLl4uOPP26oX7RokfiPf/zDJvN1Zkr8vHj00UfF9evXW3OaTkvJ38eU/p3ZpT/RV5+qqiocPnwYcXFxhjKtVou4uDikpaWZ1Ycoipg7dy6GDx+OWbNmWWuqTk+JLMg6zMmmT58+OHnyJK5fv46SkhJs2rQJ8fHxak3ZafE4sR/mZMGfD7ZhThY5OTkoLi4GABQWFmLPnj3o0KGDKvMlqos5r+PS0lLD67ikpAQ7duxAly5dVJmvMzMni969eyM3Nxe3b9+GIAjYs2cPOnXqpNaUnZaUdQ9P27Uuc7Jo0aIFDhw4gIqKCuj1euzatYs/a63A3OPi7qfLMjIycPDgQf5uphBLfh9T+ndmXhnWhJs3b0Kv1yM0NLRWeWhoKM6cOWNWH/v378eyZcvQvXt3w7nb33//Pbp166b0dJ2aElkAQFxcHI4dO4bS0lI0b94cKSkpvNOThczJxt3dHe+99x6GDRsGQRDwl7/8hXeztAJzjxMeB9ZnThb8+WAb5mRx+fJl/P73vzfchOPJJ59kDmRXzHkd5+TkYOLEiQDu3En6scceQ+/evW0+V2dn7rrn7bffxuDBgyGKIkaNGoUHH3xQjek6NXPXPYWFhTh48CBWrFhh6ym6DHOy6NevH8aMGYMePXpAq9VixIgRGDdunBrTdWrmHhfjx49HYWEh/Pz88PXXX/OGQQqx9PcxJX9nZqJWNGjQIAiCoPY06P9s27ZN7Sm4rHHjxvGHuZ3gcWAf+PPBfvTp0wfp6elqT4PIIq1bt8axY8fUngb9n4SEBCQkJKg9DQKg0+l43VU78dZbb+Gtt95SexoE8GwflZn6fUzJ35l56q4JQUFBcHNzM/rBkJOTg7CwMJVm5ZqYhf1iNvaDWdgPZmE/mAU5A76O7QezsB/Mwn4wC/vBLNRlT88/N/pM8PT0RGxsLLZv324oEwQB27dv52luNsYs7BezsR/Mwn4wC/vBLMgZ8HVsP5iF/WAW9oNZ2A9moS57ev5d+tTdkpISnD9/3vB1ZmYm0tPT0aRJE7Rs2RILFizAnDlz0KtXL/Tp0wcffvghSktLMW/ePBVn7ZyYhf1iNvaDWdgPZmE/mAU5A76O7QezsB/Mwn4wC/vBLNTlMM+/7Pv1OoGdO3eKAIz+zZkzx9Dmk08+EVu2bCl6enqKffr0EX/++Wf1JuzEmIX9Yjb2g1nYD2ZhP5gFOQO+ju0Hs7AfzMJ+MAv7wSzU5SjPv0YURVGZLUMiIiIiIiIiIiJSC6/RR0RERERERERE5AS40UdEREREREREROQEuNFHRERERERERETkBLjRR0RERERERERE5AS40UdEREREREREROQEuNFHRERERERERETkBLjRR0RERERERERE5AS40UdEREREREREROQEuNFHRERERERERETkBLjRR0Q2ExUVhQ8//NDs9q+++ipiYmJM1u/atQsajQYFBQUWz41cy3/+8x+MGjVKsf5mzJiB9957T7H+iIiI7NXcuXMxYcIEi/oYOnQonnnmGcPXUteIrkCj0WD16tVqT8NIfn4+QkJCcOnSJQD2ux6vqqpCVFQUDh06pPZUiGyOG31Eduz+hVReXh7+9Kc/oWXLlvDy8kJYWBji4+Oxf/9+QxsulOp3/8LSkV26dAkajQbp6elmf09+fj6aN29utCC7u0i7/192dnat709OTkZUVBS8vb3Rt29fHDx4sMExi4qK8NJLL6FLly7w8fFB06ZN0bt3byxatAi3b982tBs6dGitsUNDQzF16lRcvnzZqM/Lly/Dx8cHJSUlZj/2uyoqKvDSSy/hlVdekfy9pvz973/HW2+9hcLCQsX6JCIiMofctY0zrYksVddzofYGVlZWFhISEqzWf2ZmJh5++GFERETA29sbzZs3x/jx43HmzJl6v++tt97C+PHjERUVpdhc5KxpG+Lp6YnnnnsOL7zwgmJ9EjkKbvQROZDJkyfj6NGj+Pbbb3H27FmsXbsWQ4cORX5+vtpTIwfx6KOPonv37ibrMzIykJWVZfgXEhJiqFu2bBkWLFiAV155BUeOHEF0dDTi4+ORm5trsr9bt26hX79++Prrr/Hcc8/hl19+wZEjR/DWW2/h6NGj+O9//1ur/WOPPYasrCzcuHEDa9aswdWrVzFz5kyjftesWYNhw4bB399f8nOQmpqKgIAADBw4UPL3mtK1a1e0adMGP/zwg2J9EhERkeMTRRE1NTWSvy8sLAxeXl5WmBFQXV2NkSNHorCwECtXrkRGRgaWLVuGbt261buxWVZWhv/85z949NFHrTIvpT3yyCPYt28ffvvtN7WnQmRT3OgjchAFBQXYu3cv/vGPf2DYsGGIjIxEnz59sHDhQowbN052v1FRUXjzzTcxe/Zs+Pv7IzIyEmvXrkVeXh7Gjx8Pf39/dO/e3ehj7ytWrECXLl3g5eWFqKgoo9MWc3NzkZiYCB8fH7Rq1QpLliyp8zH97ne/Q3BwMAICAjB8+HAcO3ZM9mPJz8/HQw89hGbNmsHX1xfdunXDjz/+aKifO3cudu/ejY8++sjwqbG7px2cPHkSCQkJ8Pf3R2hoKGbNmoWbN28avnfo0KF46qmn8Je//AVNmjRBWFgYXn31VaPH84c//AGhoaHw9vZG165dsX79epSWliIgIACpqam12q9evRp+fn4oLi6u8/Fs3rwZgwYNQuPGjdG0aVM8+OCDuHDhgqG+VatWAIAePXpAo9Fg6NCh9T4/n332GQoKCvDcc8+ZbBMSEoKwsDDDP632fz8m3n//fTz22GOYN28eOnfujMWLF8PX1xdfffWVyf7++te/4sqVKzh48CDmzZuH7t27IzIyEqNGjcKPP/6Ixx9/vFZ7X19fhIWFITw8HP369cMTTzyBI0eOGPW7Zs0aw+t+165d6NOnD/z8/NC4cWMMHDiwzk8B3rV06VIkJibWKqvrL/kTJkzA3LlzDV9/+umnaNeuHby9vREaGoopU6bUap+YmIilS5eaHJeIiEhp9a1tdu/ejT59+sDLywvh4eF48cUXDRtOpr5Pr9fj0UcfRatWreDj44MOHTrgo48+kjSnhtZjclRVVeGJJ55AeHg4vL29ERkZiXfeecdQb2oNZs58TD0Xw4YNAwAEBgZCo9EY1gSCIOCdd94xPEfR0dG11nh3Pwm4adMmxMbGwsvLC/v27ZP8mO49dffVV1+t88yLb775xqw53e+3337DhQsX8Omnn6Jfv36IjIzEwIED8eabb6Jfv34mv2/jxo3w8vKqs83+/fvRvXt3eHt7o1+/fjh58iQAmLUOrm9N++WXX6JTp07w9vZGx44d8emnn5r9HAYGBmLgwIFcn5HL4UYfkYPw9/eHv78/Vq9ejcrKSkX7/uCDDzBw4EAcPXoUY8eOxaxZszB79mzMnDkTR44cQZs2bTB79myIoggAOHz4MKZNm4YZM2bgxIkTePXVV/HSSy8ZFhvAnUXT1atXsXPnTqSmpuLTTz81+uTX1KlTkZubi02bNuHw4cPo2bMnRowYgVu3bsl6HBUVFYiNjcWGDRtw8uRJ/P73v8esWbMMp5d+9NFH6N+/v+FTY1lZWWjRogUKCgowfPhw9OjRA4cOHcLmzZuRk5ODadOm1er/22+/hZ+fH3755RcsWrQIr7/+OrZu3QrgzgIrISEB+/fvxw8//IBTp07h3XffhZubG/z8/DBjxgx8/fXXtfr7+uuvMWXKFDRq1KjOx1NaWooFCxbg0KFD2L59O7RaLSZOnAhBEADA8Li2bduGrKwsrFy50uRzc+rUKbz++uv47rvvam3e3S8mJgbh4eEYOXJkrVPCq6qqcPjwYcTFxRnKtFot4uLikJaWVmdfgiBg2bJlmDlzJiIiIupso9FoTM7l1q1bWL58Ofr27VurvKCgAPv27cO4ceNQU1ODCRMmYMiQITh+/DjS0tLw+9//vt5+9+3bh169epmsr8uhQ4fw1FNP4fXXX0dGRgY2b96MwYMH12rTp08fHDx4UPHjk4iIyBRTa5vr169jzJgx6N27N44dO4bPPvsM//nPf/Dmm2/W+32CIKB58+ZISUnBqVOn8PLLL+Ovf/0rli9fbvacGlqPyfHxxx9j7dq1WL58OTIyMrBkyRLDqaP1rcHMmY+p52LFihUA/ne2w90Nz3feeQffffcdFi9ejN9++w3PPvssZs6cid27d9ea84svvoh3330Xp0+frvNsivoe0/2ee+65Wmdc/POf/4Svr69hPWPunO4KDg6GVqtFamoq9Hq92Tns3bsXsbGxddY9//zzeO+99/Drr78iODgYiYmJqK6uNmsdbGpNu2TJErz88st46623cPr0abz99tt46aWX8O2335r9HPbp0wd79+41+zESOQWRiOzWnDlzxPHjxxu+Tk1NFQMDA0Vvb29xwIAB4sKFC8Vjx47V+p7IyEjxgw8+MHuMyMhIcebMmYavs7KyRADiSy+9ZChLS0sTAYhZWVmiKIriww8/LI4cObJWP88//7zYuXNnURRFMSMjQwQgHjx40FB/+vRpEYBhbnv37hUDAgLEioqKWv20adNG/Pe//y2Koii+8sorYnR0tMm579y5UwQg3r5922SbsWPHin/+858NXw8ZMkR8+umna7V54403xFGjRtUqu3r1qghAzMjIMHzfoEGDarXp3bu3+MILL4iiKIo//fSTqNVqDe3v98svv4hubm7ijRs3RFEUxZycHNHd3V3ctWuXybnfLy8vTwQgnjhxQhRFUczMzBQBiEePHq33+yoqKsTu3buL33//vSiKdT9vZ86cERcvXiweOnRI3L9/vzhv3jzR3d1dPHz4sCiKonj9+nURgHjgwIFafT///PNinz596hw3OztbBCC+//77tcp79uwp+vn5iX5+fuKMGTMM5UOGDBE9PDxEPz8/0dfXVwQgtm/fXszMzKz1/UuWLBF79eoliqIo5ufniwDMfh5v374tAhD37NlTq7yu18X48ePFOXPmiKIoiitWrBADAgLEoqIik30fO3ZMBCBeunTJrLkQEREpoa6fYX/961/FDh06iIIgGMqSk5NFf39/Ua/Xm/y+uiQlJYmTJ082fH3/+tQcDa3HGlq/Pvnkk+Lw4cNrPZ67GlqDyZmPKNa9XqqoqBB9fX2N1kOPPvqo+NBDD9X6vtWrV9c7h/oekyiKIgBx1apVRuVpaWmit7e3uGzZMrPnVJd//etfoq+vr9ioUSNx2LBh4uuvvy5euHCh3jmPHz9enD9/fq2yu4936dKlhrL8/HzRx8fHMMeG1sGm1rRt2rQR//vf/9Yqe+ONN8T+/fuLotjwcyiKovjRRx+JUVFR9T4uImfDT/QROZDJkyfjxo0bWLt2LUaPHo1du3ahZ8+etT5JJ8e9f2UMDQ0FAHTr1s2o7O4n8k6fPm10fbOBAwfi3Llz0Ov1OH36NNzd3Wv9xa9jx45o3Lix4etjx46hpKQETZs2NXxa0d/fH5mZmbVOT5VCr9fjjTfeQLdu3dCkSRP4+/vjp59+wpUrV+r9vmPHjmHnzp215tGxY0cAqDWX+/8aGx4ebnhO0tPT0bx5c7Rv377OMfr06YMuXboY/gL5ww8/IDIy0uhTYfc6d+4cHnroIbRu3RoBAQGGv1A29Hjut3DhQnTq1KnOa93d1aFDB/zhD39AbGwsBgwYgK+++goDBgzABx98IGksc6xatQrp6emIj49HeXl5rbpHHnkE6enpOHbsGPbt24e2bdti1KhRtU5vvve03SZNmmDu3LmIj49HYmIiPvroI2RlZZkc++543t7ekuY8cuRIREZGonXr1pg1axaWLFmCsrKyWm18fHwAwKiciIjI1k6fPo3+/fvX+oT7wIEDUVJSgmvXrtX7vcnJyYiNjUVwcDD8/f3x+eefS1p7yF2P3fXHP/6x1poMuHOmSHp6Ojp06ICnnnoKW7ZsMbRvaA1m6Xzudf78eZSVlWHkyJG15vjdd98ZrV8bOnugvsdkypUrVzBhwgQ899xzhjNPpMzpXklJScjOzsaSJUvQv39/pKSkoEuXLoazVepSXl5ucg3Vv39/w/+bNGmCDh064PTp0wDkrYNLS0tx4cIFPProo7Ue15tvvml4XOY8hz4+PlybkcvhRh+Rg/H29sbIkSPx0ksv4cCBA5g7d67Fdw/18PAw/P/ugrCusrunjCqhpKQE4eHhSE9Pr/UvIyMDzz//vKw+/9//+3/46KOP8MILL2Dnzp2GzaSqqqoG55KYmGg0l3PnztVagNz7nAB3npe7z8ndTZ76/O53vzNsyn799deYN29evaeYJiYm4tatW/jiiy/wyy+/4JdffgGABh/P/Xbs2IGUlBS4u7vD3d0dI0aMAAAEBQXV+9rp06cPzp8/b2jr5uaGnJycWm1ycnIQFhZW5/cHBwejcePGyMjIqFXesmVLtG3bts5TlnU6Hdq2bYu2bdti4MCB+M9//oNz585h2bJlAO489s2bN9e6LuXXX3+NtLQ0DBgwAMuWLUP79u3x888/1zmnpk2bQqPR1LrbL3DnNGTx/05Nv6u6utrw/0aNGuHIkSP48ccfER4ejpdffhnR0dG1Llh995Tz4ODgOscmIiKyd0uXLsVzzz2HRx99FFu2bEF6ejrmzZsnae0hdz121+uvv15rPQYAPXv2RGZmJt544w2Ul5dj2rRphmvlNrQGs3Q+9yopKQEAbNiwodYcT506ZXQNOj8/v3r7qu8x1aW0tBTjxo1D//798frrr8ua0/0aNWqExMREvPXWWzh27BgeeOABw+nddQkKCjJaQ5lL6jr47uP64osvaj2ukydPGtZ55jyHt27d4tqMXA43+ogcXOfOnVFaWmrTMTt16lTr+m3AnQvwtm/fHm5ubujYsSNqampw+PBhQ31GRkatTZGePXsiOzsb7u7uho2du/+CgoJkzWv//v0YP348Zs6ciejoaLRu3Rpnz56t1cbT09PoWiQ9e/bEb7/9hqioKKO5NLRIu6t79+64du2a0Xj3mjlzJi5fvoyPP/4Yp06dwpw5c0y2zc/PR0ZGBv7+979jxIgR6NSpk9HCytPTEwAavLbKihUrcOzYMcMC6csvvwRw5zorSUlJJr8vPT0d4eHhhrFiY2Oxfft2Q70gCNi+fXutv+DeS6vVYtq0afjhhx9w48aNeudoyt3r69z9JN6uXbsQGBiI6OjoWu169OiBhQsX4sCBA+jatavR3Xzv8vT0ROfOnXHq1Kla5cHBwbU+CajX6w0Xkb7L3d0dcXFxWLRoEY4fP45Lly5hx44dhvqTJ0+iefPmsl+/REREctS1tunUqRPS0tJq/RFr//79aNSoEZo3b27y+/bv348BAwbg8ccfR48ePdC2bVvJZ1qYsx6rT0hISK212F0BAQGYPn06vvjiCyxbtgwrVqzArVu3GlyDyV0f1rXO6ty5M7y8vHDlyhWjNWOLFi3MfowNPab7iaKImTNnQhAEfP/997U2yJSak0ajQceOHev9vaJHjx5Ga6i77v0j6+3bt3H27Fl06tTJUFbfOriu5zo0NBQRERG4ePGi0eO6e/MOoOHn8OTJk+jRo4fZzwORM3BXewJEZJ78/HxMnToV8+fPR/fu3dGoUSMcOnQIixYtwvjx42u1vX79uuEvoHdFRkYiMDBQkbn8+c9/Ru/evfHGG29g+vTpSEtLw7/+9S/DXbA6dOiA0aNH4w9/+AM+++wzuLu745lnnqn1F9e4uDj0798fEyZMwKJFi9C+fXvcuHEDGzZswMSJEyXfLAEA2rVrh9TUVBw4cACBgYF4//33kZOTg86dOxvaREVF4ZdffsGlS5fg7++PJk2aICkpCV988QUeeughw111z58/j6VLl+LLL780bDbVZ8iQIRg8eDAmT56M999/H23btsWZM2eg0WgwevRoAHfu/DVp0iQ8//zzGDVqlGGhXZfAwEA0bdoUn3/+OcLDw3HlyhW8+OKLtdqEhITAx8cHmzdvRvPmzeHt7Q2dTmfUV5s2bWp9ffduwp06dTKcTv3hhx+iVatW6NKlCyoqKvDll19ix44dtU6BWLBgAebMmYNevXqhT58++PDDD1FaWop58+aZfBxvv/224a64r7/+Onr16gU/Pz/DjTO6du1aq31ZWRmys7MB3Pm04BtvvAFvb2+MGjUKALB27dpan+bLzMzE559/jnHjxiEiIgIZGRk4d+4cZs+ebXJO8fHx2LdvX6277A4fPhwLFizAhg0b0KZNG7z//vu1NqbXr1+PixcvYvDgwQgMDMTGjRshCAI6dOhgaLN3717DPImIiGylrrXN448/jg8//BBPPvkknnjiCWRkZOCVV17BggULDDflquv72rVrh++++w4//fQTWrVqhe+//x6//vprrY2VhpizHpPq/fffR3h4OHr06AGtVouUlBSEhYWhcePGDa7B5K4PIyMjodFosH79eowZMwY+Pj5o1KgRnnvuOTz77LMQBAGDBg1CYWEh9u/fj4CAgHr/iCvlMd3v1VdfxbZt27BlyxaUlJQYPu2m0+lkzSk9PR2vvPIKZs2ahc6dO8PT0xO7d+/GV199hRdeeMHknOPj47Fw4ULcvn3b6PeK119/HU2bNkVoaCj+9re/ISgoCBMmTDDU17cONrWmfe211/DUU09Bp9Nh9OjRqKysxKFDh3D79m0sWLDArOdw7969eOONN8xMhchJqHyNQCKqx70XO66oqBBffPFFsWfPnqJOpxN9fX3FDh06iH//+9/FsrIyw/dERkaKAIz+3b0Rw/3quvgx7rv4b10XyE1NTRU7d+4senh4iC1bthT/3//7f7X6yMrKEseOHSt6eXmJLVu2FL/77jujsYqKisQnn3xSjIiIED08PMQWLVqIjzzyiHjlyhVRFKXfjCM/P18cP3686O/vL4aEhIh///vfxdmzZ9e6YHRGRobYr18/0cfHRwRguNHD2bNnxYkTJ4qNGzcWfXx8xI4dO4rPPPOM4eK+Dd2s4e748+bNE5s2bSp6e3uLXbt2FdevX1/re7Zv3y4CEJcvX27ycd21detWsVOnTqKXl5fYvXt3cdeuXUbZfPHFF2KLFi1ErVYrDhkypME+63reRFEU//GPf4ht2rQRvb29xSZNmohDhw4Vd+zYYfS9n3zyidiyZUvR09NT7NOnj/jzzz83OF5BQYG4cOFCsWPHjqKXl5fo4+Mjdu/eXXzppZfE/Px8Q7shQ4bUes0GBgaKQ4YMqTWPFi1aiFu3bjV8nZ2dLU6YMEEMDw8XPT09xcjISPHll182XGi8Lr/99pvo4+MjFhQUGMqqqqrEP/3pT2KTJk3EkJAQ8Z133qmV7969e8UhQ4aIgYGBhvnfvcC0KIpieXm5qNPpxLS0tAafDyIiIiWZWtvs2rVL7N27t+jp6SmGhYWJL7zwglhdXV3v91VUVIhz584VdTqd2LhxY/FPf/qT+OKLL9ZajzV0Mw5z1mNSb8bx+eefizExMaKfn58YEBAgjhgxQjxy5EitMU2twSxZH77++utiWFiYqNFoDGsCQRDEDz/8UOzQoYPo4eEhBgcHi/Hx8eLu3btFUTTvZnHmPKZ713z3r5Hu/vv666/NmtP98vLyxKeeekrs2rWr6O/vLzZq1Ejs1q2b+M9//rPeNZQoimKfPn3ExYsXG76++3jXrVsndunSxbBGvP+GgaJY/zrY1Jp2yZIlYkxMjOjp6SkGBgaKgwcPFleuXGnWc3jgwAGxcePGtX5XInIFGlG876JERERkNd9//z2effZZ3Lhxw3CaApnnyJEjGD58OPLy8oyulyjV1KlT0bNnTyxcuFCRuX322WdYtWqVWRfSJiIiInJUGzZswPPPP4+TJ08aPh1qLluvg6dPn47o6Gj89a9/tfpYRPaE1+gjIrKBsrIyXLhwAe+++y7+8Ic/cJNPhpqaGnzyyScWb/IBdy7MffdOfkrw8PDAJ598olh/RERERPZo7Nix+P3vf4/r16+b/T1qrIOrqqrQrVs3/P/27aAGgBCGoiDK8IQDLhVUh2uCJeRnxkCvzUu71vp9FrzGRR/ABXvvUVVjzjm6+2hkAgCAV9mD4S6hDwAAAAACeN0FAAAAgABCHwAAAAAEEPoAAAAAIIDQBwAAAAABhD4AAAAACCD0AQAAAEAAoQ8AAAAAAgh9AAAAABDgA94MAM31QjGMAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if fc_fit is not None:\n", + " BLUE, ORANGE, GRAY, INK = \"#2a78d6\", \"#eb6834\", \"#7a7975\", \"#52514e\"\n", + " fig, (ax_fit, ax_size) = plt.subplots(1, 2, figsize=(13, 5))\n", + "\n", + " # Left: measured vs. model, with the fitted line and the alpha = 0 nominal identity.\n", + " xs = np.logspace(np.log10(t_model_nom.min()), np.log10(t_model_nom.max()), 200)\n", + " ax_fit.loglog(t_model_nom * 1e6, t_meas * 1e6, \"o\", color=BLUE, ms=6,\n", + " label=\"measured (nccl-tests alltoall, FC)\")\n", + " ax_fit.loglog(xs * 1e6, xs * 1e6, \"--\", color=GRAY, lw=1.5,\n", + " label=\"y = x (nominal model, alpha = 0)\")\n", + " ax_fit.loglog(xs * 1e6, (alpha_fit_s + fc_fit[\"slope\"] * xs) * 1e6, \"-\", color=ORANGE, lw=2,\n", + " label=f\"fit: a = {alpha_fit_s * 1e6:.1f} us, b = {fc_fit['slope']:.2f}, \"\n", + " f\"R^2 = {fc_fit['r2']:.3f}\")\n", + " ax_fit.set_xlabel(f\"ISL model latency at {NVLINK_H100_GBPS:g} GB/s (us)\")\n", + " ax_fit.set_ylabel(\"measured latency (us)\")\n", + " ax_fit.set_title(\"Measured vs. ISL model (all-scatter, 8x H100 FC)\")\n", + " ax_fit.grid(True, which=\"both\", alpha=0.3)\n", + " ax_fit.legend()\n", + "\n", + " # Right: latency vs. total size, nominal and calibrated model through the measured points.\n", + " ax_size.loglog(S_fc, t_meas * 1e6, \"o\", color=BLUE, ms=6, label=\"measured\")\n", + " ax_size.loglog(S_fc, t_model_nom * 1e6, \"--\", color=GRAY, lw=1.5,\n", + " label=f\"model, nominal {NVLINK_H100_GBPS:g} GB/s, alpha = 0\")\n", + " ax_size.loglog(S_fc, t_model_cal * 1e6, \"-\", color=ORANGE, lw=2,\n", + " label=f\"model, calibrated ({bw_eff / 1e9:.0f} GB/s, alpha = {alpha_fit_s * 1e6:.1f} us)\")\n", + " square_label_by_size = {\n", + " M * K * dtype_bytes: f\"{M}x{K}\" for M, K in MATMUL_SHAPES if M == K and M >= 1024\n", + " }\n", + " # Label each size once (its first row in size order) even if several runs contributed\n", + " # the same size; the points themselves are still all drawn above.\n", + " seen_sizes = set()\n", + " labelled = []\n", + " for s, t in zip(S_fc, t_meas):\n", + " if int(s) in square_label_by_size and int(s) not in seen_sizes:\n", + " seen_sizes.add(int(s))\n", + " labelled.append((s, t))\n", + " for i, (s, t) in enumerate(labelled):\n", + " last = i == len(labelled) - 1\n", + " # Square shapes >= 1024 only, alternating above/below the point so labels crowding\n", + " # the latency floor stay legible; the largest size is right-aligned so it does not\n", + " # run past the axes edge.\n", + " ax_size.annotate(square_label_by_size[int(s)], (s, t * 1e6), textcoords=\"offset points\",\n", + " xytext=(-6, 6) if last else (6, 6 if i % 2 == 0 else -12),\n", + " ha=\"right\" if last else \"left\", fontsize=8, color=INK)\n", + " ax_size.set_xlabel(\"total all-scatter size S (bytes)\")\n", + " ax_size.set_ylabel(\"latency (us)\")\n", + " ax_size.set_title(\"Latency vs. matmul operand size\")\n", + " ax_size.grid(True, which=\"both\", alpha=0.3)\n", + " ax_size.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"no FC alltoall data -- nothing to plot\")" + ] } ], "metadata": { diff --git a/notebooks/astrasim2_correlation/correlation/.gitignore b/notebooks/astrasim2_correlation/correlation/.gitignore deleted file mode 100644 index 6bddd09b..00000000 --- a/notebooks/astrasim2_correlation/correlation/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -keys/ -.state/ -*.pem -logs/ diff --git a/notebooks/astrasim2_correlation/correlation/README.md b/notebooks/astrasim2_correlation/correlation/README.md deleted file mode 100644 index 4eb33486..00000000 --- a/notebooks/astrasim2_correlation/correlation/README.md +++ /dev/null @@ -1,188 +0,0 @@ -# correlation/ -- AWS provisioning for NCCL profiling - -This directory is the **empirical leg** of the ISL-model correlation -study: it profiles real NCCL collective-communication performance on one -AWS `p5.48xlarge` instance (8x H100, NVSwitch), both in the GPUs' native -fully-connected (FC) topology and under a logical torus overlay, so those -measurements can be compared against the model's and ASTRA-sim's -predictions. - -## Prerequisites - -- **AWS credentials** available to boto3 via the standard mechanisms (env - vars `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN`, a - named profile via `AWS_PROFILE`, or an attached IAM role). This - directory never manages or stores credentials itself. -- **boto3** installed in whatever Python environment you run these - scripts from: `pip install boto3`. It is deliberately *not* a dependency - of the `accelforge` package -- only this notebook's provisioning corner - needs it. -- **Service Quotas.** `p5.48xlarge` requires 192 vCPUs of quota. Before - your first run, check the Service Quotas console for your target region - and confirm both: - - the on-demand P-instance quota ("Running On-Demand P instances" or - equivalent, depending on current AWS naming), and - - the corresponding Spot P-instance vCPU quota (if you intend to use - `--purchasing spot` or the default `spot-then-ondemand`), - - are each **>= 192 vCPUs**. Quota codes and exact names change over time - and vary by account/region presentation -- look them up in the console - rather than trusting a hardcoded code here. A quota that's too low - surfaces as an `InsufficientInstanceCapacity`-adjacent or limit-exceeded - error at launch time. - -## Cost warning - -`p5.48xlarge` on-demand pricing is roughly **$30-55/hr**, depending on -region and current AWS pricing -- **verify current pricing** at - before running anything. -A full profiling sweep (FC + torus, all collectives, the full message-size -range) is expected to take well under an hour, but see the dead-man-timer -note below: the default 120-minute timer is comfortably above that -estimate, but a large custom `--min-mib`/`--max-mib` range or collective -list can push a full sweep close to or past it. Spot pricing is -substantially cheaper when capacity is available, which is why -`spot-then-ondemand` is the default purchasing mode -- see -`provision.launch_instance`'s docstring for the exact fallback conditions. -Both `orchestrate.py` and `provision.py` print this same warning and -require an interactive `yes` confirmation before touching AWS (skippable -with `--yes`; not required for `--dry-run`). - -## Quickstart - -**`orchestrate.py` is the end-to-end path.** It provisions its own -instance, pushes the profiling scripts, runs the full FC and/or torus -sweep, fetches the results locally, and tears the instance down again -- -all in one command: - -```bash -# 1. Sanity-check permissions and request shape without launching anything -# (still creates a real SSH key pair + security group -- see the -# --dry-run note below): -python orchestrate.py --topology both --dry-run - -# 2. The real run: provision, profile both legs, fetch results, tear down. -python orchestrate.py --topology both -``` - -**Do not run `provision.py` before `orchestrate.py`.** `orchestrate.py` -provisions its *own* instance internally; running `provision.py` first -would launch a *second*, independent `p5.48xlarge` instance that nothing -in the `orchestrate.py` run above knows about or tears down, silently -doubling your bill. `provision.py`/`teardown.py` are for advanced, manual -control only -- see the section below. - -Every `Config` field (region, purchasing mode, topology, message-size -range, collectives, ...) is a CLI flag; run `python orchestrate.py --help` -for the full list, or pass `--config some.yaml` to load a batch of -overrides from YAML (CLI flags still win over anything in the YAML file). -Useful flags: - -- `--keep-alive`: skip `orchestrate.py`'s own teardown at the end, leaving - the instance running (its SSH command is printed) for manual - inspection. The on-instance dead-man timer still applies regardless. -- `--dead-man-minutes `: extend the dead-man timer past its 120-minute - default -- see the note below. -- `--ssh-cidr /32`: skip auto-detecting your IP for the SSH - security-group rule; required if IP auto-detection fails (see - `provision.caller_ip`'s docstring). - -### Fetched data layout - -Each leg's results land under `data///`: - -``` -data/// -├── csv/ # parsed, unified-schema CSVs consumed by the correlation notebook -├── raw/ # raw nccl-tests/torus_bench stdout logs, one per collective -└── metadata.txt # machine/software provenance (nvidia-smi topo, driver/NCCL versions, git rev) -``` - -`` is `fc` or `torus`. `data/` is the one subdirectory of this -project *not* gitignored -- fetched results are committed intentionally. - -## Advanced / manual control: `provision.py` + `teardown.py` - -`provision.py` and `teardown.py` are the individual "up" and "down" halves -`orchestrate.py` composes internally. Use them directly only if you need -manual control between provisioning and profiling (e.g. debugging the -instance by hand, or running a custom workload instead of -`run_profile.sh`) -- most users should use `orchestrate.py` above instead. - -```bash -# Provision one instance and leave it running (see the warning below): -python provision.py - -# ... do whatever manual work you need on the instance ... - -# Tear down that one run: -python teardown.py --run-id - -# Tear down everything this study has tagged, across every region this -# study's local state knows about: -python teardown.py --all - -# Also delete the SSH key pair (AWS-side) and local PEM: -python teardown.py --run-id --delete-key - -# Audit: confirm nothing is left running, without tearing anything down. -# Exits 0 ("no running instances") when clean, 1 with a table otherwise -- -# safe to use as a post-teardown check or a periodic cron/CI safety net. -python teardown.py --verify -``` - -**Warning: `provision.py` leaves the instance running with NO dead-man -timer until `setup_node.sh` is run on it.** The dead-man timer is armed -*by* `setup_node.sh` (a step `orchestrate.py` always runs for you, but -`provision.py` alone does not reach) -- so an instance provisioned via -`provision.py` and never followed up with `setup_node.sh` (or -`teardown.py`) will run, and bill, indefinitely with no automatic -backstop. If you provision manually, either run `setup_node.sh` on the -instance promptly or tear it down yourself as soon as you're done. - -`--dry-run` on **both** `provision.py` and `orchestrate.py` performs -`DryRun=True` authorization checks only and launches no instance -- but a -**real** SSH key pair and security group ARE still created in AWS either -way (there is no dry-run equivalent for those two calls). Clean them up -with: - -```bash -python teardown.py --run-id --delete-key -``` - -`teardown.py` discovers resources two ways and reconciles them: local -`.state/*.json` files written by `provision.py`/`orchestrate.py`, and a -live `describe_instances` search by `Project`/`RunId` tags. The tag search -is authoritative, so teardown still works even if a state file was lost or -a run was started from a different machine. `--region` defaults to -*resolving per run* rather than to a fixed region: an explicit `--region` -flag always wins, otherwise each run's own state file (if any) supplies -its region, otherwise `us-east-1` (`Config`'s default) is used -- so -`--all`/`--verify` correctly span every region this study's local state -knows about in one invocation, not just one hardcoded region. - -## Safety guardrails - -- **Dead-man timer -- armed by `setup_node.sh`, not at instance launch.** - Once `setup_node.sh` has run on the instance (always true for an - `orchestrate.py` run; not automatic for a manual `provision.py` one -- - see the warning above), an on-instance timer force-shuts-down the box - after `--dead-man-minutes` (default 120) regardless of whether - `teardown.py` was ever run -- a backstop against a forgotten or failed - teardown. Re-running `setup_node.sh` (e.g. via a second `orchestrate.py` - leg) pushes the deadline back rather than erroring or stacking. Note - that the 120-minute default can be tight for a long custom sweep -- - raise it with `--dead-man-minutes` if you expect to run past it. -- **`InstanceInitiatedShutdownBehavior=terminate`.** The instance is - launched so that an in-instance `shutdown` (including the dead-man - timer firing) *terminates* it rather than merely stopping it, so it - cannot be left billing in a stopped state. -- **Teardown-in-`finally`.** `orchestrate.py` calls teardown from a - `finally` block around the provisioning-through-profiling sequence, so a - mid-sweep crash or a failed leg still tears the instance down (unless - `--keep-alive` was passed). -- **PEM files are gitignored.** `keys/`, `.state/`, `*.pem`, and `logs/` - are all excluded (see `.gitignore`) -- private key material and - per-run state never get committed. `data/` (fetched profiling CSVs) is - the one subdirectory *not* ignored; those results are committed - intentionally. diff --git a/notebooks/astrasim2_correlation/correlation/config.py b/notebooks/astrasim2_correlation/correlation/config.py deleted file mode 100644 index 9fb675f0..00000000 --- a/notebooks/astrasim2_correlation/correlation/config.py +++ /dev/null @@ -1,666 +0,0 @@ -"""Shared configuration for the NCCL profiling correlation-study AWS scripts. - -This module defines :class:`Config`, the single source of truth for every -tunable knob used by ``provision.py``, ``teardown.py``, and (per the plan) -the sibling ``orchestrate.py`` that a later work package will add. Keeping -the configuration in one frozen dataclass -- rather than threading loose -kwargs through each script -- means every script agrees on defaults, -validation, and CLI flag names without duplicating logic. - -Construction paths -------------------- -``Config`` instances can be built three ways, all of which funnel through -the same validation in :meth:`Config.__post_init__`: - -1. Directly, e.g. ``Config(region="us-west-2")`` -- handy for tests and for - any future caller that wants a config without touching argparse at all. -2. Via :meth:`Config.from_args`, which owns its own - :class:`argparse.ArgumentParser` end to end. -3. Via :meth:`Config.add_args` + :meth:`Config.from_parsed`, which lets a - *caller* (``provision.py``, ``teardown.py``) build one shared parser, - add its own script-specific flags (e.g. ``--dry-run``), and only then - hand the resulting namespace back to ``Config`` to extract just the - fields that belong to it. This is the path ``provision.py`` and - ``teardown.py`` actually use. - -Design: frozen dataclass -------------------------- -``Config`` is declared ``frozen=True`` so that once built it can be passed -into ``provision.py`` functions (``launch_instance``, ``write_state``, ...) -without any risk of one function's edits leaking into another's view of the -same run. Frozen dataclasses cannot assign to ``self.`` in the usual -way, so any post-construction normalization (parsing "2x2x2" into a tuple, -generating a run id, coercing str paths to :class:`pathlib.Path`) goes -through ``object.__setattr__`` inside ``__post_init__`` -- this is the -standard, documented escape hatch for "derive a field after validation" on -a frozen dataclass. -""" - -from __future__ import annotations - -import argparse -import dataclasses -import datetime -import math -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -# Design: anchor key_dir/state_dir to this file's directory (not the -# process cwd) so that `python provision.py` behaves identically no matter -# where the user's shell happens to be sitting when they invoke it. -_THIS_DIR = Path(__file__).resolve().parent - -_ALLOWED_PURCHASING = frozenset({"spot", "ondemand", "spot-then-ondemand"}) -_ALLOWED_TOPOLOGY = frozenset({"fc", "torus", "both"}) - -# Default sweep of NCCL collectives profiled by the (sibling-work-package) -# profiling scripts. Kept here, not in profiling code, so a single -# `--collectives` override on the CLI is the one place a user needs to -# change to alter the sweep. -_DEFAULT_COLLECTIVES: Tuple[str, ...] = ( - "all_reduce", - "all_gather", - "reduce_scatter", - "alltoall", - "broadcast", - "sendrecv", -) - -# The study profiles one 8x H100 node; a torus overlay must therefore have -# axis dimensions whose product is exactly 8 (one "logical GPU slot" per -# axis position), regardless of how many axes are used. -_TORUS_GPU_COUNT = 8 - -_RUN_ID_TIME_FORMAT = "%Y%m%d-%H%M%S" - - -def _default_run_id() -> str: - """Generate a fresh, sortable run identifier. - - Returns - ------- - str - ``"correl-" + UTC timestamp`` formatted as - ``%Y%m%d-%H%M%S`` (e.g. ``"correl-20260716-161503"``). Lexicographic - sort order matches chronological order, which is convenient when - listing ``.state/*.json`` files or AMI/SG names in a shell. - - Notes - ----- - Uses UTC (not local time) so run ids are unambiguous and comparable - regardless of which machine or timezone invokes the script. - """ - return "correl-" + datetime.datetime.now(datetime.timezone.utc).strftime( - _RUN_ID_TIME_FORMAT - ) - - -def _is_power_of_two(n: int) -> bool: - """Return whether a positive integer is an exact power of two. - - Parameters - ---------- - n : int - Value to test. - - Returns - ------- - bool - ``True`` if ``n > 0`` and ``n & (n - 1) == 0``, ``False`` otherwise - (including for ``n <= 0``). - - Examples - -------- - >>> _is_power_of_two(1024) - True - >>> _is_power_of_two(0) - False - >>> _is_power_of_two(3) - False - """ - return n > 0 and (n & (n - 1)) == 0 - - -def _parse_torus_dims(value: Any) -> Tuple[int, ...]: - """Normalize a torus-dimensions spec into a tuple of ints. - - Accepts the three shapes this value can arrive in depending on - construction path: an already-correct ``tuple[int, ...]`` (direct - ``Config(...)`` construction), a ``list``/``tuple`` of ints or numeric - strings (from a parsed YAML ``--config`` file), or a delimited string - like ``"2x2x2"`` (from the CLI, where argparse hands raw strings to - ``type=`` callables). - - Parameters - ---------- - value : Any - Torus dimensions in any of the accepted shapes described above. - - Returns - ------- - tuple[int, ...] - The parsed per-axis dimensions, in the order given. - - Raises - ------ - ValueError - If ``value`` is an empty string, or any component cannot be parsed - as an integer. - - Examples - -------- - >>> _parse_torus_dims("2x2x2") - (2, 2, 2) - >>> _parse_torus_dims([2, 4]) - (2, 4) - >>> _parse_torus_dims((8,)) - (8,) - """ - if isinstance(value, (list, tuple)): - try: - return tuple(int(v) for v in value) - except (TypeError, ValueError) as exc: - raise ValueError( - f"torus_dims entries must all be integers, got {value!r}" - ) from exc - - text = str(value).strip().lower() - if not text: - raise ValueError("torus_dims must not be empty") - try: - return tuple(int(part) for part in text.split("x")) - except ValueError as exc: - raise ValueError( - f"torus_dims={value!r} is not a valid dims string; " - "expected a form like '2x2x2'" - ) from exc - - -def _parse_collectives(value: Any) -> Tuple[str, ...]: - """Normalize a collectives spec into a tuple of collective names. - - Parameters - ---------- - value : Any - Either a ``list``/``tuple`` of collective-name strings (from a - parsed YAML ``--config`` file or direct construction), or a - comma-separated string (from the CLI). - - Returns - ------- - tuple[str, ...] - The parsed collective names, in the order given, with surrounding - whitespace stripped and empty entries dropped (so a trailing comma - like ``"all_reduce,"`` does not produce a spurious ``""`` entry). - - Examples - -------- - >>> _parse_collectives("all_reduce, broadcast") - ('all_reduce', 'broadcast') - >>> _parse_collectives(["all_reduce", "broadcast"]) - ('all_reduce', 'broadcast') - """ - if isinstance(value, (list, tuple)): - return tuple(str(v) for v in value) - return tuple(part.strip() for part in str(value).split(",") if part.strip()) - - -@dataclasses.dataclass(frozen=True) -class Config: - """Immutable configuration for one correlation-study AWS provisioning run. - - All fields have defaults, so ``Config()`` alone yields a fully valid - configuration (one on-demand-or-spot ``p5.48xlarge`` in ``us-east-1``, - fully-connected topology, a freshly generated ``run_id``). Every - construction path (direct, :meth:`from_args`, :meth:`from_parsed`) runs - the same validation in :meth:`__post_init__`, so an invalid ``Config`` - can never be observed by downstream code. - - Parameters - ---------- - region : str, default "us-east-1" - AWS region to provision in. Also used for the SSM AMI lookup and - must have the requested ``instance_type`` available. - availability_zone : str or None, default None - Specific AZ within ``region`` to pin the instance to. ``None`` - lets AWS/the spot fleet choose. - instance_type : str, default "p5.48xlarge" - EC2 instance type. The 8x H100 NVSwitch topology this study - profiles is specific to ``p5.48xlarge``; other types are accepted - without validation but are not what the rest of this work package - was designed against. - purchasing : str, default "spot-then-ondemand" - One of ``"spot"``, ``"ondemand"``, ``"spot-then-ondemand"``. See - ``provision.launch_instance`` for the fallback semantics of the - combined mode. - topology : str, default "fc" - One of ``"fc"`` (fully connected, the NVSwitch-native topology), - ``"torus"`` (a logical torus overlay profiled on top of the same - physical node), or ``"both"``. - torus_dims : tuple[int, ...], default (2, 2, 2) - Per-axis dimensions of the logical torus. Precondition: the - product of all dimensions must equal 8 (one axis slot per GPU on - the node). Accepts a ``"2x2x2"``-style string on construction and - normalizes it to a tuple of ints. - collectives : tuple[str, ...], default (all_reduce, all_gather, - reduce_scatter, alltoall, broadcast, sendrecv) - NCCL collectives the (sibling) profiling scripts should sweep. - Accepts a comma-separated string on construction. - min_mib : int, default 1 - Smallest message size, in MiB, in the profiling sweep. - Precondition: must be a power of two and ``<= max_mib``. - max_mib : int, default 1024 - Largest message size, in MiB, in the profiling sweep. - Precondition: must be a power of two and ``>= min_mib``. - run_id : str or None, default None - Identifier used to name/tag every AWS resource created for this - run (instance, security group, key pair, state file). If left as - ``None``, a fresh id is generated in :meth:`__post_init__` as - ``"correl-" + UTC timestamp``. See the NOTE in - :meth:`__post_init__` for why generation happens there rather than - only in :meth:`from_args`. - tag_project : str, default "accelforge-correlation" - Value written to the ``Project`` tag on every AWS resource this - run creates; also what ``teardown.py`` filters - ``describe_instances`` on to discover a run's resources. - ssh_user : str, default "ubuntu" - Login user baked into the deep-learning AMI, used when printing - the ``ssh`` command at the end of provisioning. - ami_ssm_parameter : str, default - "/aws/service/deeplearning/ami/x86_64/base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" - SSM public parameter name that resolves to the latest matching - Deep Learning AMI id for ``region``. - root_volume_gb : int, default 200 - Size, in GiB, of the root ``gp3`` EBS volume attached at - ``/dev/sda1``. - dead_man_minutes : int, default 120 - Minutes of wall-clock time after which the on-instance dead-man - timer (armed by setup scripts owned by a sibling work package) - force-shuts-down the instance. This is a safety net independent of - ``teardown.py`` actually being run; see the README's guardrails - section. - key_dir : pathlib.Path, default "/keys" - Directory where generated SSH private keys are written. - state_dir : pathlib.Path, default "/.state" - Directory where per-run provisioning state JSON is written. - - Raises - ------ - ValueError - Raised by :meth:`__post_init__` if ``purchasing`` or ``topology`` - is not one of the allowed values, if ``torus_dims`` does not - multiply out to 8 or contains a non-positive entry, if - ``min_mib > max_mib``, or if either ``min_mib`` or ``max_mib`` is - not a power of two. - - Examples - -------- - >>> cfg = Config(region="us-west-2", purchasing="ondemand") - >>> cfg.region, cfg.purchasing - ('us-west-2', 'ondemand') - >>> cfg.run_id is not None - True - """ - - region: str = "us-east-1" - availability_zone: Optional[str] = None - instance_type: str = "p5.48xlarge" - purchasing: str = "spot-then-ondemand" - topology: str = "fc" - torus_dims: Tuple[int, ...] = (2, 2, 2) - collectives: Tuple[str, ...] = _DEFAULT_COLLECTIVES - min_mib: int = 1 - max_mib: int = 1024 - run_id: Optional[str] = None - tag_project: str = "accelforge-correlation" - ssh_user: str = "ubuntu" - ami_ssm_parameter: str = ( - "/aws/service/deeplearning/ami/x86_64/" - "base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" - ) - root_volume_gb: int = 200 - dead_man_minutes: int = 120 - key_dir: Path = _THIS_DIR / "keys" - state_dir: Path = _THIS_DIR / ".state" - - def __post_init__(self) -> None: - """Normalize field representations and validate all preconditions. - - Raises - ------ - ValueError - See the class docstring's ``Raises`` section; this method is - where every one of those checks is actually enforced. - - Notes - ----- - Runs for *every* construction path (``Config(...)`` directly, - :meth:`from_args`, :meth:`from_parsed`), because dataclasses always - call ``__post_init__`` after ``__init__``. This is a deliberate - choice over validating only inside :meth:`from_args`: it means a - test (or a future caller) that builds ``Config(purchasing="bogus")`` - directly fails loudly at construction time instead of silently - producing an invalid config that only misbehaves once it reaches - AWS calls. - """ - # Frozen dataclasses disallow `self.field = ...`; object.__setattr__ - # is the standard, documented way to set fields from __post_init__. - object.__setattr__(self, "torus_dims", _parse_torus_dims(self.torus_dims)) - object.__setattr__(self, "collectives", _parse_collectives(self.collectives)) - object.__setattr__(self, "key_dir", Path(self.key_dir)) - object.__setattr__(self, "state_dir", Path(self.state_dir)) - - if self.run_id is None: - # NOTE: the work-package spec describes run_id's default as - # "generated ... at parse time", which most literally refers to - # Config.from_args. We instead generate it here, in - # __post_init__, so every construction path gets a valid run_id - # -- see the docstring Notes above for why. This is the more - # conservative reading: it can never produce a Config with - # run_id=None reaching AWS tag values, which the "at parse - # time" phrasing on its own does not guarantee for direct - # `Config(...)` construction. - object.__setattr__(self, "run_id", _default_run_id()) - - if self.purchasing not in _ALLOWED_PURCHASING: - raise ValueError( - f"purchasing={self.purchasing!r} is not one of " - f"{sorted(_ALLOWED_PURCHASING)}" - ) - if self.topology not in _ALLOWED_TOPOLOGY: - raise ValueError( - f"topology={self.topology!r} is not one of {sorted(_ALLOWED_TOPOLOGY)}" - ) - - # A negative-dimension axis (e.g. (-2, -2, 2)) could still multiply - # out to 8, silently passing a bare product check; reject it - # explicitly since it is never physically meaningful for a torus. - if any(d <= 0 for d in self.torus_dims): - raise ValueError( - f"torus_dims={self.torus_dims!r} must all be positive integers" - ) - product = math.prod(self.torus_dims) - if product != _TORUS_GPU_COUNT: - raise ValueError( - f"torus_dims={self.torus_dims!r} has product {product}, " - f"expected {_TORUS_GPU_COUNT} (one p5.48xlarge node = " - f"{_TORUS_GPU_COUNT} GPUs)" - ) - - if self.min_mib > self.max_mib: - raise ValueError( - f"min_mib={self.min_mib} must be <= max_mib={self.max_mib}" - ) - if not _is_power_of_two(self.min_mib): - raise ValueError(f"min_mib={self.min_mib} must be a power of two") - if not _is_power_of_two(self.max_mib): - raise ValueError(f"max_mib={self.max_mib} must be a power of two") - - @classmethod - def add_args(cls, parser: argparse.ArgumentParser) -> None: - """Register every ``Config`` field as a kebab-case CLI flag. - - Intended to be called by a script's own parser setup (see - ``provision.py``/``teardown.py``) *before* that script adds its - own extra flags (e.g. ``--dry-run``), so the two flag sets share - one ``argparse.ArgumentParser`` and one ``--help`` output. - - Parameters - ---------- - parser : argparse.ArgumentParser - Parser to add arguments to, mutated in place. - - Notes - ----- - Design: every flag added here uses ``default=argparse.SUPPRESS`` - instead of the field's real default. This means an unset flag is - simply *absent* from the parsed namespace, which is exactly what - :meth:`from_parsed` needs to implement "CLI flags override - ``--config`` YAML values override dataclass defaults" -- if every - flag instead defaulted to its real value, :meth:`from_parsed` could - not distinguish "user explicitly passed the default value" from - "user didn't pass this flag at all", and CLI flags could never be - overridden by anything. - """ - # Only used to render human-readable defaults into --help text; - # never used for the actual default values (see Notes above). - defaults = cls() - - parser.add_argument( - "--config", - type=str, - default=None, - help=( - "Path to a YAML file of Config field overrides, applied " - "before CLI flags (CLI flags always win over --config)." - ), - ) - parser.add_argument( - "--region", - type=str, - default=argparse.SUPPRESS, - help=f"AWS region (default: {defaults.region!r}).", - ) - parser.add_argument( - "--availability-zone", - type=str, - default=argparse.SUPPRESS, - help="AWS availability zone, e.g. us-east-1a (default: let AWS choose).", - ) - parser.add_argument( - "--instance-type", - type=str, - default=argparse.SUPPRESS, - help=f"EC2 instance type (default: {defaults.instance_type!r}).", - ) - parser.add_argument( - "--purchasing", - type=str, - choices=sorted(_ALLOWED_PURCHASING), - default=argparse.SUPPRESS, - help=f"Purchasing strategy (default: {defaults.purchasing!r}).", - ) - parser.add_argument( - "--topology", - type=str, - choices=sorted(_ALLOWED_TOPOLOGY), - default=argparse.SUPPRESS, - help=f"NCCL topology to profile (default: {defaults.topology!r}).", - ) - parser.add_argument( - "--torus-dims", - type=_parse_torus_dims, - default=argparse.SUPPRESS, - help=( - "Logical torus dims as e.g. '2x2x2'; product must be 8 " - f"(default: {'x'.join(str(d) for d in defaults.torus_dims)!r})." - ), - ) - parser.add_argument( - "--collectives", - type=_parse_collectives, - default=argparse.SUPPRESS, - help=( - "Comma-separated NCCL collectives to sweep " - f"(default: {','.join(defaults.collectives)!r})." - ), - ) - parser.add_argument( - "--min-mib", - type=int, - default=argparse.SUPPRESS, - help=( - "Smallest message size in MiB, must be a power of two " - f"(default: {defaults.min_mib})." - ), - ) - parser.add_argument( - "--max-mib", - type=int, - default=argparse.SUPPRESS, - help=( - "Largest message size in MiB, must be a power of two " - f"(default: {defaults.max_mib})." - ), - ) - parser.add_argument( - "--run-id", - type=str, - default=argparse.SUPPRESS, - help=( - "Run identifier used to tag/name all resources " - "(default: generated as 'correl-')." - ), - ) - parser.add_argument( - "--tag-project", - type=str, - default=argparse.SUPPRESS, - help=f"Value for the 'Project' tag on all resources (default: {defaults.tag_project!r}).", - ) - parser.add_argument( - "--ssh-user", - type=str, - default=argparse.SUPPRESS, - help=f"SSH login user for the AMI (default: {defaults.ssh_user!r}).", - ) - parser.add_argument( - "--ami-ssm-parameter", - type=str, - default=argparse.SUPPRESS, - help="SSM parameter name to resolve the AMI id from.", - ) - parser.add_argument( - "--root-volume-gb", - type=int, - default=argparse.SUPPRESS, - help=f"Root EBS volume size in GiB (default: {defaults.root_volume_gb}).", - ) - parser.add_argument( - "--dead-man-minutes", - type=int, - default=argparse.SUPPRESS, - help=( - "Minutes before the on-instance dead-man timer force-shuts-down " - f"the instance (default: {defaults.dead_man_minutes})." - ), - ) - parser.add_argument( - "--key-dir", - type=Path, - default=argparse.SUPPRESS, - help=f"Directory to store the generated SSH private key (default: {defaults.key_dir}).", - ) - parser.add_argument( - "--state-dir", - type=Path, - default=argparse.SUPPRESS, - help=f"Directory to store per-run provisioning state JSON (default: {defaults.state_dir}).", - ) - - @classmethod - def from_parsed(cls, namespace: argparse.Namespace) -> "Config": - """Build a :class:`Config` from an already-parsed argparse namespace. - - Meant to be used together with :meth:`add_args`: a caller builds - one ``ArgumentParser``, calls ``Config.add_args(parser)``, adds its - own extra flags, calls ``parser.parse_args(argv)``, and passes the - resulting namespace here. Any namespace attributes that are not - ``Config`` field names (e.g. a caller's own ``--dry-run``) are - ignored, so the same namespace can safely be shared with - script-specific flags. - - Parameters - ---------- - namespace : argparse.Namespace - Parsed CLI arguments, as produced by - ``parser.parse_args(...)`` on a parser that included - :meth:`add_args`'s flags. If it has a ``config`` attribute - (from the ``--config`` flag) that is truthy, that path is - loaded as a YAML mapping of field overrides. - - Returns - ------- - Config - A validated ``Config`` built from, in increasing priority: - dataclass defaults, then ``--config`` YAML values, then - explicitly-passed CLI flags. - - Raises - ------ - ValueError - If ``--config`` points at a YAML document whose top level is - not a mapping, or if any field fails :meth:`__post_init__` - validation. - OSError - If ``--config`` points at a path that cannot be opened. - """ - field_names = {f.name for f in dataclasses.fields(cls)} - values: Dict[str, Any] = {} - - config_path = getattr(namespace, "config", None) - if config_path: - # Design: import PyYAML lazily, inside this branch, rather than - # at module top. Only the --config path needs it; a plain - # `provision.py --help` (or any run that never passes - # --config) must keep working even in an environment that - # only has boto3 installed and not PyYAML. - import yaml - - with open(config_path, "r") as fh: - yaml_values = yaml.safe_load(fh) - if yaml_values is None: - yaml_values = {} - if not isinstance(yaml_values, dict): - raise ValueError( - f"--config file {config_path!r} must contain a top-level " - f"YAML mapping, got {type(yaml_values).__name__}" - ) - # Silently drop unknown keys rather than raising: this lets a - # single shared YAML file carry keys meant for other tools - # (e.g. a future orchestrate.py section) without every - # consumer needing to know about every other consumer's keys. - values.update({k: v for k, v in yaml_values.items() if k in field_names}) - - # CLI flags win over --config values. Because add_args() gives - # every flag default=argparse.SUPPRESS, `namespace` only carries a - # key for a field the user actually typed on the command line, so - # this unconditional overwrite is exactly "CLI beats YAML beats - # dataclass default". - for key, value in vars(namespace).items(): - if key in field_names: - values[key] = value - - return cls(**values) - - @classmethod - def from_args(cls, argv: Optional[List[str]] = None) -> "Config": - """Parse ``argv`` with a fresh, ``Config``-only parser. - - Convenience wrapper around :meth:`add_args` + :meth:`from_parsed` - for callers that only need ``Config``'s own flags and do not have - any script-specific flags of their own to add. - - Parameters - ---------- - argv : list[str] or None, default None - Argument list to parse, as passed to - ``argparse.ArgumentParser.parse_args``. ``None`` means "read - from ``sys.argv[1:]``", argparse's own default behavior. - - Returns - ------- - Config - The parsed, validated configuration. - - Examples - -------- - >>> Config.from_args(["--region", "us-west-2", "--purchasing", "ondemand"]).region - 'us-west-2' - """ - parser = argparse.ArgumentParser( - description="accelforge NCCL correlation-study provisioning config." - ) - cls.add_args(parser) - namespace = parser.parse_args(argv) - return cls.from_parsed(namespace) diff --git a/notebooks/astrasim2_correlation/correlation/data/.gitkeep b/notebooks/astrasim2_correlation/correlation/data/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/notebooks/astrasim2_correlation/correlation/orchestrate.py b/notebooks/astrasim2_correlation/correlation/orchestrate.py deleted file mode 100644 index 00d5dddd..00000000 --- a/notebooks/astrasim2_correlation/correlation/orchestrate.py +++ /dev/null @@ -1,935 +0,0 @@ -"""End-to-end CLI for the NCCL correlation study's empirical leg. - -This is the "run everything" work package: it ties together the -provisioning infrastructure (``config.py``, ``provision.py``, -``teardown.py``) and the profiling infrastructure (``setup_node.sh``, -``run_profile.sh``, ``parse_nccl.py``, ``torus_bench/``) -- all owned by -sibling work packages and imported/invoked here, never modified -- into one -command an operator runs to go from "nothing provisioned" to "CSVs sitting -in ``data///csv/`` and the instance torn down". - -Pipeline --------- -1. Parse CLI args into a :class:`config.Config` plus this script's own - ``--yes``/``--keep-alive``/``--dry-run``/``--ssh-cidr`` flags. -2. Print the run header, cost warning, a wall-time estimate, and the leg - plan (via :func:`legs_for`); ask for interactive confirmation unless - ``--yes`` (skipped entirely for ``--dry-run``, which spends no money). -3. Resolve the AMI, create the SSH key pair and security group, and call - ``provision.launch_instance`` -- exactly the sequence ``provision.py`` - itself runs, reusing its functions directly rather than reimplementing - any of them. If ``--dry-run``, stop here (the key pair and security - group above were still created for real; see the ``--dry-run`` flag's - help text). -4. Write the provisioning state file immediately, before waiting for SSH - (see the design comment at that call site for why). -5. Wait for the instance to become SSH-reachable, then update the state - file with the now-known public IP. -6. scp the profiling scripts and ``torus_bench/`` onto the instance. -7. ssh in to run ``setup_node.sh`` (arms the dead-man timer, builds - nccl-tests/torus_bench). -8. For each leg selected by ``--topology`` (see :func:`legs_for`), ssh in - to run ``run_profile.sh`` for the full collective sweep, then scp the - results back to ``data///``. -9. In a ``finally`` block around steps 5-8: tear the instance down (unless - ``--keep-alive``), so a crash or a failed profiling leg never leaves an - (expensive, 8x H100) instance running unattended. See the design - comment on :func:`_teardown_and_cleanup` for how a teardown failure - itself is handled without masking whatever exception was already - propagating. - -Design: no new AWS/SSH logic here ------------------------------------ -Every AWS API call in this module goes through a ``provision.py`` or -``teardown.py`` function that already exists, is already tested, and is -already documented as part of this work package's contract (see those -modules' docstrings). This module's own responsibility is narrower: -sequencing those calls correctly, building ``ssh``/``scp`` argv lists -(:func:`build_ssh_cmd`, :func:`build_scp_cmd`), and running them as -subprocesses. Keeping that boundary sharp is also what makes this module -testable without any real AWS/SSH/SCP access -- every seam it introduces -(the two ``build_*_cmd`` functions, plus the imported provisioning -functions) is a plain function that a test can monkeypatch or inspect the -return value of, per this work package's "no AWS, no network, no ssh in -tests" constraint. -""" - -from __future__ import annotations - -import argparse -import subprocess -import sys -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from config import Config -from provision import ( - _COST_WARNING, - _prompt_yes_no, - _require_boto3, - caller_ip, - ensure_key_pair, - ensure_security_group, - launch_instance, - resolve_ami, - wait_for_instance, - write_state, -) -from teardown import teardown_run - -# Design: guarded import, matching provision.py/teardown.py's own -# convention (see provision.py's module docstring for the full rationale) -# -- so `python orchestrate.py --help` keeps working even in a Python -# environment that lacks boto3, since argparse's own --help handling exits -# before main() ever reaches _require_boto3(). Importing `boto3` here as a -# module-level name of orchestrate.py's own (rather than reaching into -# `provision.boto3`) keeps this module's boto3.client(...) calls readable -# without poking at another module's internals; _require_boto3() (reused -# from provision.py, not redefined) is still what actually validates -# availability before any client is constructed, since both imports -# resolve to the same cached sys.modules entry (or both to None) in any -# given interpreter. -try: - import boto3 -except ImportError: # pragma: no cover - exercised only when boto3 truly absent - boto3 = None - -# --------------------------------------------------------------------------- -# Module-level path constants -# --------------------------------------------------------------------------- - -# Anchor every sibling-file path to this file's own directory (not the -# process cwd), matching config.py's identical _THIS_DIR convention -- so -# `python orchestrate.py` behaves the same regardless of the caller's shell -# cwd. -_THIS_DIR = Path(__file__).resolve().parent - -_SETUP_NODE_SH = _THIS_DIR / "setup_node.sh" -_RUN_PROFILE_SH = _THIS_DIR / "run_profile.sh" -_PARSE_NCCL_PY = _THIS_DIR / "parse_nccl.py" -_TORUS_BENCH_DIR = _THIS_DIR / "torus_bench" - -# Design: expose the fetched-results root as its own module-level constant -# (rather than inlining `_THIS_DIR / "data"` at the one call site) purely -# so tests can monkeypatch `orchestrate._DATA_DIR` to a tmp_path and -# guarantee the end-to-end test never writes into the real repository's -# data/ directory -- every other AWS/ssh/scp side effect in a test is -# already monkeypatched away, and this is the one remaining plain -# filesystem write main() would otherwise perform unconditionally. -_DATA_DIR = _THIS_DIR / "data" - -# run_profile.sh's argument for the FC leg: the profiling scripts -# treat "8" as "all 8 GPUs, fully connected" -- there is no logical torus -# shape to describe for that leg (see run_profile.sh's `-g 8` on the FC -# path), unlike the torus leg where is a "DxDx..." shape string. -_FC_DIMS = "8" - -_BYTES_PER_MIB = 2**20 - - -# --------------------------------------------------------------------------- -# Pure helpers (no I/O, no AWS, no subprocess) -- kept separate from main() -# specifically so they are trivially unit-testable per this work package's -# "no AWS, no network, no ssh in tests" constraint. -# --------------------------------------------------------------------------- - - -def legs_for(topology: str) -> List[str]: - """Expand a ``Config.topology`` value into the ordered list of legs to run. - - Parameters - ---------- - topology : str - Typically ``cfg.topology``, one of ``"fc"``, ``"torus"``, or - ``"both"`` (``Config.__post_init__`` already validates this, so - this function does not re-validate it -- see Notes). - - Returns - ------- - list[str] - ``["fc", "torus"]`` if ``topology == "both"``; otherwise the - single-element list ``[topology]``. - - Notes - ----- - Deliberately permissive for any value other than ``"both"``: it simply - echoes that value back as a one-element list rather than checking it - against the allowed set. Re-validating here would duplicate - ``Config.__post_init__``'s already-authoritative check for no benefit, - since every caller in this module only ever passes an already-validated - ``cfg.topology``. - - Examples - -------- - >>> legs_for("both") - ['fc', 'torus'] - >>> legs_for("fc") - ['fc'] - >>> legs_for("torus") - ['torus'] - """ - if topology == "both": - return ["fc", "torus"] - return [topology] - - -def _dims_for_leg(leg: str, torus_dims: Tuple[int, ...]) -> str: - """Compute run_profile.sh's ```` argument for one leg. - - Parameters - ---------- - leg : str - ``"fc"`` or ``"torus"``. - torus_dims : tuple[int, ...] - Per-axis torus dimensions, e.g. ``(2, 2, 2)`` (only consulted when - ``leg == "torus"``). - - Returns - ------- - str - :data:`_FC_DIMS` (``"8"``) for the FC leg; otherwise - ``torus_dims`` joined with ``"x"``, e.g. ``"2x2x2"``. - - Examples - -------- - >>> _dims_for_leg("fc", (2, 2, 2)) - '8' - >>> _dims_for_leg("torus", (2, 2, 2)) - '2x2x2' - """ - if leg == "fc": - return _FC_DIMS - return "x".join(str(d) for d in torus_dims) - - -def _estimate_wall_time_message(legs: List[str]) -> str: - """Build the human-readable wall-time estimate printed before confirmation. - - Parameters - ---------- - legs : list[str] - The leg plan, as returned by :func:`legs_for`. - - Returns - ------- - str - A one-line estimate: ~30-60 minutes when both legs are selected - (the work-package spec's own figure for a full FC+torus sweep), - or roughly half that for a single leg. - - Notes - ----- - This is a coarse, documented *estimate* for setting operator - expectations before a real-money confirmation prompt, not a measured - or SLA'd figure -- actual time depends on instance boot time, spot - availability, and the exact collective/message-size sweep configured. - """ - if len(legs) >= 2: - return ( - "Estimated wall time: ~30-60 minutes for both legs " - "(excludes instance boot/provisioning time)." - ) - return ( - "Estimated wall time: ~15-30 minutes for a single leg " - "(excludes instance boot/provisioning time)." - ) - - -def build_ssh_cmd( - key_path: Path, - user: str, - ip: str, - remote_cmd: str, - known_hosts_path: Path, -) -> List[str]: - """Build an ``ssh`` argv list to run one command on the provisioned instance. - - Parameters - ---------- - key_path : pathlib.Path - Path to the local PEM private key (as returned by - ``provision.ensure_key_pair``). - user : str - SSH login user, typically ``cfg.ssh_user``. - ip : str - Target host's public IP address. - remote_cmd : str - The full remote command line to execute, e.g. - ``"DEADMAN_MINUTES=120 bash ~/setup_node.sh"``. Passed to ``ssh`` - as a single trailing argv element; ``ssh`` hands it to the remote - login shell for interpretation, so ordinary shell syntax (env var - prefixes, multiple space-separated arguments) works as expected - without any extra quoting from this function. - known_hosts_path : pathlib.Path - Path to a per-run known-hosts file (typically - ``cfg.state_dir / "known_hosts"``). Kept as an explicit parameter - (rather than a hardcoded/global path) so this function stays pure - and independently testable -- see the module docstring's "Design: - no new AWS/SSH logic here" section. - - .. note:: - NOTE ON SPEC DEVIATION: the work-package spec lists this - function's signature as ``build_ssh_cmd(key_path, user, ip, - remote_cmd)`` -- four parameters -- but also requires the - ``-o UserKnownHostsFile=/known_hosts`` option, which - cannot be constructed without knowing ``state_dir`` from - *somewhere*. Reaching into a module-global ``Config`` from - inside this function would break the "pure helper, easy to - unit-test" property the spec explicitly asks for. Adding - ``known_hosts_path`` as an explicit fifth parameter is the - minimal change that preserves purity/testability; every - call site in this module passes ``cfg.state_dir / "known_hosts"`` - for it, matching the spec's intent. - - Returns - ------- - list[str] - argv list of the form ``["ssh", "-i", , "-o", - "StrictHostKeyChecking=accept-new", "-o", "UserKnownHostsFile=<...>", - "-o", "ConnectTimeout=30", "@", ]``, ready to - pass to ``subprocess.run``. - - Examples - -------- - >>> build_ssh_cmd(Path("/k/id.pem"), "ubuntu", "1.2.3.4", "echo hi", Path("/s/known_hosts")) - ... # doctest: +NORMALIZE_WHITESPACE - ['ssh', '-i', '/k/id.pem', '-o', 'StrictHostKeyChecking=accept-new', - '-o', 'UserKnownHostsFile=/s/known_hosts', '-o', 'ConnectTimeout=30', - 'ubuntu@1.2.3.4', 'echo hi'] - """ - return [ - "ssh", - "-i", - str(key_path), - "-o", - "StrictHostKeyChecking=accept-new", - "-o", - f"UserKnownHostsFile={known_hosts_path}", - "-o", - "ConnectTimeout=30", - f"{user}@{ip}", - remote_cmd, - ] - - -def build_scp_cmd( - key_path: Path, - sources: List[str], - dest: str, - known_hosts_path: Path, - recursive: bool = False, -) -> List[str]: - """Build an ``scp`` argv list to copy one or more paths to/from the instance. - - Parameters - ---------- - key_path : pathlib.Path - Path to the local PEM private key. - sources : list[str] - Source path(s) to copy, in whatever form ``scp`` accepts: plain - local paths for a push, or a single ``"user@ip:remote/path"`` - string for a fetch. Must be non-empty. - dest : str - Destination, again in whatever form ``scp`` accepts (a local - directory for a fetch, or ``"user@ip:remote/path"`` for a push). - known_hosts_path : pathlib.Path - Path to a per-run known-hosts file. See :func:`build_ssh_cmd`'s - docstring for why this is an explicit parameter rather than an - implicit global (the same rationale applies here). - recursive : bool, default False - If ``True``, prepend ``-r`` (needed for copying a directory, e.g. - ``torus_bench/`` or a remote ``results_/`` directory). - - Returns - ------- - list[str] - argv list of the form ``["scp", ["-r"], "-i", , "-o", - "StrictHostKeyChecking=accept-new", "-o", - "UserKnownHostsFile=<...>", "-o", "ConnectTimeout=30", *sources, - dest]``, ready to pass to ``subprocess.run``. The ``-r`` flag (when - present) is placed immediately after ``"scp"``, before every other - option, so its position is fixed and independently assertable in - tests regardless of how many sources are given. - - Raises - ------ - ValueError - If ``sources`` is empty -- an ``scp`` invocation with no source - path is never meaningful and would otherwise fail cryptically at - the OS level instead of at this argv-building step. - - Examples - -------- - >>> build_scp_cmd(Path("/k/id.pem"), ["a.sh", "b.sh"], "ubuntu@1.2.3.4:~/", Path("/s/known_hosts")) - ... # doctest: +NORMALIZE_WHITESPACE - ['scp', '-i', '/k/id.pem', '-o', 'StrictHostKeyChecking=accept-new', - '-o', 'UserKnownHostsFile=/s/known_hosts', '-o', 'ConnectTimeout=30', - 'a.sh', 'b.sh', 'ubuntu@1.2.3.4:~/'] - >>> build_scp_cmd(Path("/k/id.pem"), ["dir"], "ubuntu@1.2.3.4:~/dir", Path("/s/known_hosts"), recursive=True)[:2] - ['scp', '-r'] - """ - if not sources: - raise ValueError("build_scp_cmd requires at least one source path") - - cmd: List[str] = ["scp"] - if recursive: - cmd.append("-r") - cmd += [ - "-i", - str(key_path), - "-o", - "StrictHostKeyChecking=accept-new", - "-o", - f"UserKnownHostsFile={known_hosts_path}", - "-o", - "ConnectTimeout=30", - ] - cmd += list(sources) - cmd.append(dest) - return cmd - - -# --------------------------------------------------------------------------- -# Subprocess execution -# --------------------------------------------------------------------------- - - -def _run_streaming(cmd: List[str]) -> None: - """Run an ``ssh``/``scp`` argv list, streaming its output live. - - Parameters - ---------- - cmd : list[str] - argv list, typically from :func:`build_ssh_cmd` or - :func:`build_scp_cmd`. - - Raises - ------ - subprocess.CalledProcessError - If the subprocess exits with a non-zero status (``check=True``). - - Notes - ----- - Deliberately does not pass ``capture_output``/``stdout``/``stderr`` -- - the child process's output streams straight through to this process's - own stdout/stderr, so an operator watching a multi-minute profiling - sweep sees live progress rather than a silent hang followed by a wall - of buffered text at the end. Prints the command line first (to stdout) - so the corresponding output block is identifiable in a long combined - log. - """ - print("+ " + " ".join(cmd)) - subprocess.run(cmd, check=True) - - -# --------------------------------------------------------------------------- -# Pipeline stages -- each a thin, single-responsibility wrapper around a -# handful of _run_streaming calls, factored out of main() so that function -# stays a readable top-to-bottom sequence rather than one long body. -# --------------------------------------------------------------------------- - - -def _push_files(cfg: Config, key_path: Path, public_ip: str, known_hosts_path: Path) -> None: - """scp the profiling scripts and torus_bench/ onto the instance. - - Parameters - ---------- - cfg : Config - Run configuration; only ``cfg.ssh_user`` is consulted directly - (the rest flows through ``key_path``/``public_ip``/ - ``known_hosts_path``). - key_path : pathlib.Path - Local PEM private key path. - public_ip : str - Instance's public IP, as returned by ``provision.wait_for_instance``. - known_hosts_path : pathlib.Path - Per-run known-hosts file path. - - Notes - ----- - Two separate ``scp`` invocations, matching the two different transfer - shapes: (1) three individual files pushed non-recursively to the - remote home directory, and (2) the ``torus_bench/`` directory pushed - recursively to ``~/torus_bench`` so ``setup_node.sh`` can build it. - Side effect: two subprocess invocations over the network to the - instance. - """ - remote_home = f"{cfg.ssh_user}@{public_ip}:~/" - print("=== Pushing profiling scripts (setup_node.sh, run_profile.sh, parse_nccl.py) ===") - _run_streaming( - build_scp_cmd( - key_path, - [str(_SETUP_NODE_SH), str(_RUN_PROFILE_SH), str(_PARSE_NCCL_PY)], - remote_home, - known_hosts_path, - ) - ) - - remote_torus_dir = f"{cfg.ssh_user}@{public_ip}:~/torus_bench" - print("=== Pushing torus_bench/ ===") - _run_streaming( - build_scp_cmd( - key_path, - [str(_TORUS_BENCH_DIR)], - remote_torus_dir, - known_hosts_path, - recursive=True, - ) - ) - - -def _run_setup(cfg: Config, key_path: Path, public_ip: str, known_hosts_path: Path) -> None: - """ssh in and run ``setup_node.sh`` (arms dead-man timer, builds binaries). - - Parameters - ---------- - cfg : Config - Run configuration; ``cfg.dead_man_minutes`` is forwarded to - ``setup_node.sh`` as the ``DEADMAN_MINUTES`` environment variable. - key_path : pathlib.Path - Local PEM private key path. - public_ip : str - Instance's public IP. - known_hosts_path : pathlib.Path - Per-run known-hosts file path. - - Notes - ----- - Must run after :func:`_push_files` (``setup_node.sh`` and - ``torus_bench/`` must already be on the instance) and before any - profiling leg (``setup_node.sh`` is what builds the nccl-tests and - torus_bench binaries those legs depend on, and arms the on-instance - dead-man safety timer). Side effect: one subprocess invocation over - the network to the instance. - """ - remote_cmd = f"DEADMAN_MINUTES={cfg.dead_man_minutes} bash ~/setup_node.sh" - print("=== Running setup_node.sh ===") - _run_streaming(build_ssh_cmd(key_path, cfg.ssh_user, public_ip, remote_cmd, known_hosts_path)) - - -def _run_leg( - cfg: Config, - leg: str, - key_path: Path, - public_ip: str, - known_hosts_path: Path, - run_data_dir: Path, -) -> None: - """Run one profiling leg on the instance and fetch its results locally. - - Parameters - ---------- - cfg : Config - Run configuration (``torus_dims``, ``min_mib``/``max_mib``, - ``collectives``, ``ssh_user``). - leg : str - ``"fc"`` or ``"torus"``. - key_path : pathlib.Path - Local PEM private key path. - public_ip : str - Instance's public IP. - known_hosts_path : pathlib.Path - Per-run known-hosts file path. - run_data_dir : Path - Local directory this run's results should land under (typically - ``_DATA_DIR / cfg.run_id``); this leg's results land specifically - at ``run_data_dir / leg``. - - Notes - ----- - Side effects: one ``ssh`` subprocess invocation that runs the full - collective sweep for this leg on the instance (this is the - long-running step -- see :func:`_estimate_wall_time_message`), a - ``mkdir`` of ``run_data_dir`` (the PARENT of this leg's fetch - destination -- see the inline comment above that call for why the leaf - directory itself is deliberately left uncreated), and one ``scp -r`` - subprocess invocation that fetches the results back, creating the new - local directory tree at ``run_data_dir / leg`` itself. - - Prints the list of fetched ``*.csv`` files at the end, so an operator - watching the run can immediately confirm data landed without a - separate ``ls``. - - Raises - ------ - RuntimeError - If ``run_data_dir / leg`` already exists before the fetch -- see - the inline comment above the check for why this is treated as an - error rather than silently proceeding. - """ - dims = _dims_for_leg(leg, cfg.torus_dims) - min_bytes = cfg.min_mib * _BYTES_PER_MIB - max_bytes = cfg.max_mib * _BYTES_PER_MIB - remote_results_dir = f"~/results_{leg}" - remote_cmd = ( - f"bash ~/run_profile.sh {remote_results_dir} {leg} {min_bytes} {max_bytes} {dims} " - f"{' '.join(cfg.collectives)}" - ) - - print(f"=== Profiling leg: {leg} (dims={dims}, {cfg.min_mib}-{cfg.max_mib} MiB) ===") - _run_streaming(build_ssh_cmd(key_path, cfg.ssh_user, public_ip, remote_cmd, known_hosts_path)) - - # Design: fetch into a leaf directory (run_data_dir/leg) that must NOT - # already exist when scp runs. `scp -r user@host:~/results_fc - # ` renames the copied directory to when - # doesn't yet exist, landing its contents (csv/, raw/, - # metadata.txt) directly inside it -- exactly the data/// - # layout this work package's spec requires. Pre-creating that leaf - # directory first would instead make scp nest an extra results_fc/ - # level inside it, since scp's "copy into vs. rename to" behavior - # depends on whether the destination path already exists. - local_leg_dir = run_data_dir / leg - if local_leg_dir.exists(): - # A pre-existing leaf directory means a previous run already fetched - # results for this exact run_id+leg combination (or something else - # created the path). Silently proceeding would make scp nest an - # extra results_/ level inside the existing directory instead - # of landing csv/raw/metadata.txt directly in it (see the comment - # above), corrupting the data/// layout without any - # error -- raise instead so a re-run with a colliding run_id+leg - # fails loudly here rather than silently mis-nesting fetched data. - raise RuntimeError( - f"{local_leg_dir} already exists; a previous fetch for run_id=" - f"{cfg.run_id!r} leg={leg!r} already landed results there. " - "Refusing to scp into it again (that would nest an extra " - f"results_{leg}/ level inside the existing directory instead of " - "csv/raw/metadata.txt landing directly in it). Remove or rename " - "the existing directory, or re-run with a different --run-id." - ) - # Only the PARENT (run_data_dir) is created here -- local_leg_dir itself - # must be left absent (see the comment above) for scp's rename-vs-nest - # behavior to produce the right layout. Without this mkdir, the very - # first leg fetched under a fresh run_id would fail outright: scp cannot - # write to / if itself doesn't exist - # yet (parents=True/exist_ok=True mirrors provision.write_state's own - # "create the directory, then write into it" idiom). - run_data_dir.mkdir(parents=True, exist_ok=True) - fetch_source = f"{cfg.ssh_user}@{public_ip}:{remote_results_dir}" - print(f"=== Fetching {leg} leg results ===") - _run_streaming( - build_scp_cmd( - key_path, - [fetch_source], - str(local_leg_dir), - known_hosts_path, - recursive=True, - ) - ) - - csv_files = sorted((local_leg_dir / "csv").glob("*.csv")) - print(f"Fetched {leg} leg results to {local_leg_dir}:") - for csv_path in csv_files: - print(f" {csv_path}") - - -def _print_keep_alive_notice(cfg: Config, key_path: Path, state: Dict[str, Any]) -> None: - """Print the SSH command and a loud cost reminder for a ``--keep-alive`` run. - - Parameters - ---------- - cfg : Config - Run configuration (``ssh_user``, ``run_id``, ``dead_man_minutes``). - key_path : pathlib.Path - Local PEM private key path, included in the printed SSH command. - state : dict - This run's state dict; ``state["public_ip"]`` is used if present. - - Notes - ----- - Called from :func:`main`'s ``finally`` block in place of - :func:`_teardown_and_cleanup` when ``--keep-alive`` was passed. Does - not delete the state file (unlike the teardown path), since - ``teardown.py`` needs it later to find and tear down this - still-running instance. - """ - ip_display = state.get("public_ip") or "" - print("=" * 70) - print("--keep-alive set: instance left RUNNING. YOU ARE STILL BEING BILLED.") - print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{ip_display}") - print( - f"Dead-man shutdown deadline: ~{cfg.dead_man_minutes} minutes after " - "setup_node.sh armed it (this is a backstop independent of teardown.py)." - ) - print(f"Tear down manually with: python teardown.py --run-id {cfg.run_id}") - print("=" * 70) - - -def _teardown_and_cleanup( - ec2_client, state: Dict[str, Any], state_path: Path, cfg: Config -) -> None: - """Tear down this run's AWS resources and remove its local state file. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - state : dict - This run's state dict, as passed to ``teardown.teardown_run``. - state_path : pathlib.Path - Path to this run's ``.state/.json`` file, removed only if - teardown succeeds. - cfg : Config - Run configuration, used only to print ``cfg.run_id`` into the - follow-up messages below. - - Notes - ----- - Called from :func:`main`'s ``finally`` block, which may itself be - running while an exception from the profiling steps (e.g. a failed - ``run_profile.sh`` invocation) is already propagating out of the - surrounding ``try``. This is exactly the scenario the work-package - spec calls out: a ``finally`` block that itself raises would, under - Python's ordinary exception semantics, cause *that new* exception to - be what the caller sees instead of the original one -- effectively - masking the original failure behind a teardown failure, even though - both are independently worth surfacing. This function therefore - catches any exception ``teardown_run`` raises, prints it (and does not - delete the state file, since a failed teardown may have left resources - behind that ``teardown.py --run-id `` will still need it to - find), and simply returns -- letting whatever exception was already - active in the caller's ``try`` block continue propagating undisturbed. - """ - print(f"Tearing down run {cfg.run_id}...") - try: - teardown_run(ec2_client, state, delete_key=False) - except Exception as teardown_exc: # noqa: BLE001 - # Design/WHY deliberately broad and deliberately NOT re-raised: - # see this function's docstring Notes above. Printing to stderr - # (rather than raising) is what prevents this teardown failure - # from masking an in-flight exception from the try block this - # finally belongs to. - print(f"ERROR: teardown failed: {teardown_exc}", file=sys.stderr) - print( - "Manual cleanup required -- the instance may still be running. " - f"Try: python teardown.py --run-id {cfg.run_id}", - file=sys.stderr, - ) - return - - if state_path.exists(): - state_path.unlink() - print(f"Removed state file {state_path}.") - print("Recommended audit: python teardown.py --verify") - - -# --------------------------------------------------------------------------- -# CLI entry point -# --------------------------------------------------------------------------- - - -def main(argv: Optional[List[str]] = None) -> int: - """Parse CLI args and run the full provision -> profile -> teardown pipeline. - - Parameters - ---------- - argv : list[str] or None, default None - Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` - reads from ``sys.argv[1:]``. - - Returns - ------- - int - ``0`` on success (including a completed ``--dry-run``), ``1`` if - the user declined the interactive confirmation prompt. - - Raises - ------ - Exception - Any exception raised by provisioning, SSH/SCP subprocess failures - (``subprocess.CalledProcessError``), or waiting for the instance - propagates out of this function uncaught -- per the work-package - spec, a profiling/provisioning failure is a real failure and - should surface as one (a nonzero process exit code via - ``raise SystemExit(main())`` in ``__main__``), not be silently - downgraded to a return code. The ``finally`` block described below - still runs before that propagation completes. - - Notes - ----- - Side effects (skipped when ``--dry-run`` is passed, past the point - where the SSH key pair and security group are created -- see the - ``--dry-run`` flag's help text): creates an SSH key pair and security - group, launches an instance, writes/updates a state JSON file, scp's - profiling scripts and results to/from the instance, ssh's in to run - setup and profiling commands, and (unless ``--keep-alive``) tears the - instance down again. - - Design/WHY the state file is written immediately after - ``launch_instance`` returns, before ``wait_for_instance`` is even - called: ``wait_for_instance`` can block for several minutes (instance - boot, then polling for SSH) and can itself raise (``TimeoutError``, - ``WaiterError``). If this controller process crashes or is killed - during that wait, an instance is still running and being billed with - no local record of it unless the state file was already written - beforehand. Writing state right after launch -- with ``public_ip`` as - an explicit placeholder, filled in by a second ``write_state`` call - once ``wait_for_instance`` returns -- means ``teardown.py`` (which - also cross-checks live AWS tags, not just local state, per its own - docstring) can find and tear down this run from local state alone even - in that crash scenario, without waiting on SSH reachability first. - """ - parser = argparse.ArgumentParser( - prog="orchestrate.py", - description=( - "End-to-end NCCL correlation-study orchestration: provision one " - "p5.48xlarge instance, push profiling scripts, run the FC " - "and/or torus profiling sweep, fetch results into data//, " - "and tear the instance down." - ), - ) - Config.add_args(parser) - parser.add_argument( - "--dry-run", - action="store_true", - help=( - "Perform DryRun=True authorization checks only; launch no " - "instance and run no profiling. NOTE: a real SSH key pair and " - "security group ARE still created in AWS even with --dry-run, " - "mirroring provision.py's documented --dry-run semantics -- " - "only launch_instance's actual launch and everything after it " - "(waiting for SSH, pushing files, profiling, teardown) is " - "skipped." - ), - ) - parser.add_argument( - "--keep-alive", - action="store_true", - help=( - "Skip automatic teardown after profiling completes (or fails); " - "leave the instance running and print its SSH command instead " - "of tearing it down. The on-instance dead-man timer " - "(--dead-man-minutes) still applies regardless of this flag -- " - "it only disables orchestrate.py's own teardown call, not that " - "backstop." - ), - ) - parser.add_argument( - "--ssh-cidr", - type=str, - default=None, - help=( - "CIDR block allowed SSH access, e.g. 1.2.3.4/32. Overrides " - "auto-detected caller IP (see provision.caller_ip())." - ), - ) - parser.add_argument( - "--yes", - action="store_true", - help="Skip the interactive cost-confirmation prompt (not required for --dry-run).", - ) - args = parser.parse_args(argv) - cfg = Config.from_parsed(args) - - # Fail fast on a missing boto3 before printing anything else, matching - # provision.py's own main() -- no point asking the operator to confirm - # a cost warning for a run that cannot possibly proceed. - _require_boto3() - - legs = legs_for(cfg.topology) - print(f"=== accelforge correlation-study orchestration: run_id={cfg.run_id} ===") - print(_COST_WARNING) - print(_estimate_wall_time_message(legs)) - print(f"Legs to profile: {', '.join(legs)} (topology={cfg.topology!r})") - print( - f"Purchasing mode: {cfg.purchasing}. Instance type: {cfg.instance_type}. " - f"Region: {cfg.region}." - ) - - if not args.dry_run and not args.yes: - if not _prompt_yes_no("Proceed with provisioning and profiling? [yes/N]: "): - print("Aborted by user.") - return 1 - - ec2_client = boto3.client("ec2", region_name=cfg.region) - ssm_client = boto3.client("ssm", region_name=cfg.region) - - ssh_cidr = args.ssh_cidr - if not ssh_cidr: - ip = caller_ip() - ssh_cidr = f"{ip}/32" - print(f"SSH will be allowed from: {ssh_cidr}") - - ami_id = resolve_ami(ssm_client, cfg.ami_ssm_parameter) - print(f"Resolved AMI: {ami_id}") - - key_name = f"{cfg.tag_project}-{cfg.run_id}" - key_path = ensure_key_pair(ec2_client, key_name, cfg.key_dir) - print(f"Key pair ready: {key_name} -> {key_path}") - - sg_name = f"{cfg.tag_project}-{cfg.run_id}-sg" - sg_id = ensure_security_group(ec2_client, sg_name, ssh_cidr, cfg.tag_project, cfg.run_id) - print(f"Security group ready: {sg_id}") - - # Mirrors provision.py's main(): the dry_run flag flows all the way - # into launch_instance (which makes a real, DryRun=True API call) so a - # dry run exercises the exact same request-building code path a real - # launch would, rather than short-circuiting before this call. - launch_result = launch_instance( - ec2_client, cfg, ami_id, sg_id, key_name, dry_run=args.dry_run - ) - - if args.dry_run: - print( - f"Dry run complete (purchasing checked: {launch_result['purchasing_used']}); " - "no instance was launched, no SSH was attempted, and no profiling ran. " - "The key pair and security group above WERE created for real -- " - f"clean them up with: python teardown.py --run-id {cfg.run_id} --delete-key" - ) - return 0 - - instance_id = launch_result["instance_id"] - purchasing_used = launch_result["purchasing_used"] - print(f"Launched instance {instance_id} ({purchasing_used}).") - - # See this function's "Design/WHY" docstring Notes above for the full - # rationale: write state now, with public_ip left as an explicit - # placeholder, rather than waiting until wait_for_instance (which can - # block for minutes and can itself fail) returns. - state: Dict[str, Any] = { - "run_id": cfg.run_id, - "region": cfg.region, - "instance_id": instance_id, - "sg_id": sg_id, - "key_name": key_name, - "key_path": str(key_path), - "public_ip": None, - "purchasing_used": purchasing_used, - "ami_id": ami_id, - } - state_path = write_state(cfg.state_dir, cfg.run_id, state) - print(f"State written to: {state_path} (public_ip pending SSH reachability).") - - known_hosts_path = cfg.state_dir / "known_hosts" - run_data_dir = _DATA_DIR / cfg.run_id - - try: - public_ip = wait_for_instance(ec2_client, instance_id) - state["public_ip"] = public_ip - write_state(cfg.state_dir, cfg.run_id, state) - print(f"Instance is running and SSH-reachable at {public_ip}") - print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{public_ip}") - - _push_files(cfg, key_path, public_ip, known_hosts_path) - _run_setup(cfg, key_path, public_ip, known_hosts_path) - - for leg in legs: - _run_leg(cfg, leg, key_path, public_ip, known_hosts_path, run_data_dir) - finally: - # Teardown-in-finally: this block runs whether the try body above - # succeeded, raised (e.g. a failed run_profile.sh -> a propagating - # subprocess.CalledProcessError), or was interrupted -- so an - # (expensive, 8x H100) instance is never left running just because - # one profiling leg failed partway through. See - # _teardown_and_cleanup's docstring for how a *second* failure - # (teardown itself failing) is handled without masking whichever - # exception was already propagating out of the try body. - if args.keep_alive: - _print_keep_alive_notice(cfg, key_path, state) - else: - _teardown_and_cleanup(ec2_client, state, state_path, cfg) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/parse_nccl.py b/notebooks/astrasim2_correlation/correlation/parse_nccl.py deleted file mode 100644 index 8a4c6d41..00000000 --- a/notebooks/astrasim2_correlation/correlation/parse_nccl.py +++ /dev/null @@ -1,483 +0,0 @@ -"""Parse raw NCCL collective-communication profiling logs into tidy CSVs. - -This module turns the stdout of two profiling tools into a single, unified -CSV schema that the correlation notebook (``correlation.ipynb``) consumes: - -1. **nccl-tests** (upstream NVIDIA binaries: ``all_reduce_perf``, - ``all_gather_perf``, ``reduce_scatter_perf``, ``alltoall_perf``, - ``broadcast_perf``, ``sendrecv_perf``) run against the fully-connected - (FC) NVSwitch fabric. -2. ``torus_bench`` (a custom binary built by a sibling work package) run - against a torus-topology emulation on the same physical fabric. - -Both tools are parsed independently (:func:`parse_nccl_tests` and -:func:`parse_torus_bench`) and their results are reshaped into the shared -schema documented at :data:`UNIFIED_CSV_FIELDNAMES` before being written to -disk with :func:`rows_to_csv`. The module is also a CLI entry point (see -:func:`main`) so it can be invoked directly from ``run_profile.sh`` on the -profiling instance without any extra Python dependencies. - -Notes ------ -Stdlib-only by design: this script runs on a freshly provisioned EC2 -instance where installing a virtualenv is unwanted overhead. Only ``csv``, -``argparse``, and ``pathlib`` (plus ``sys`` for the CLI entry point) are -used, so it works under any plain ``python3`` >= 3.8. -""" - -from __future__ import annotations - -import argparse -import csv -import sys -from pathlib import Path -from typing import Union - -# Design: the unified schema is a module-level constant (rather than being -# implicit in whatever keys happen to be in the first row dict) so that -# rows_to_csv() always emits a stable, predictable column order regardless -# of which source produced the rows, and so the notebook can rely on the -# header never silently reordering itself as this module evolves. -UNIFIED_CSV_FIELDNAMES: list[str] = [ - "source", - "topology", - "dims", - "collective", - "size_bytes", - "count", - "dtype", - "time_us", - "algbw_GBps", - "busbw_GBps", - "wrong", -] - -# Minimum number of whitespace-separated tokens a nccl-tests data row must -# have before it is considered parseable. A row always carries at least -# size, count, type, redop, root (5 leading columns) plus the 8 trailing -# out-of-place/in-place metric columns = 13 tokens; 10 is used as a looser -# lower bound per the spec so that unexpected/future nccl-tests column -# layouts with slightly fewer leading columns are still accepted as long as -# the trailing-8 structure holds. -_MIN_NCCL_TESTS_TOKENS = 10 - -# Number of trailing tokens on a nccl-tests data row that carry the -# out-of-place/in-place timing results. This is the anchor of the parsing -# strategy; see parse_nccl_tests() docstring for the rationale. -_TRAILING_METRIC_TOKENS = 8 - -# Sentinel line prefix emitted by torus_bench for each data point. Chosen -# by the sibling work package specifically so it is trivial to grep/parse -# out of interleaved '#'-prefixed human-readable log noise. -_TORUS_SENTINEL_PREFIX = "TORUSBENCH," - - -def parse_nccl_tests(text: str) -> list[dict]: - """Parse the stdout of an nccl-tests collective benchmark binary. - - nccl-tests binaries (``all_reduce_perf``, ``all_gather_perf``, - ``reduce_scatter_perf``, ``alltoall_perf``, ``broadcast_perf``, - ``sendrecv_perf``) share a common output shape: a block of ``#``-prefixed - header/comment lines, followed by one data row per message size, followed - by ``#``-prefixed summary lines. The *leading* columns of a data row vary - per collective (e.g. ``redop``/``root`` are meaningless for - ``alltoall_perf`` and print as ``none``/``-1``), but the *trailing* eight - columns are always, in order: out-of-place ``time``, ``algbw``, ``busbw``, - ``#wrong``, then in-place ``time``, ``algbw``, ``busbw``, ``#wrong``. - - Parameters - ---------- - text : str - Raw stdout captured from an nccl-tests binary invocation. May - contain blank lines and ``#``-prefixed comment/header/summary lines - interleaved with data rows. - - Returns - ------- - list of dict - One dict per parsed data row, in file order, with keys: - ``size_bytes`` (int), ``count`` (int), ``dtype`` (str), - ``time_us`` (float), ``algbw_GBps`` (float), ``busbw_GBps`` (float), - ``wrong`` (str; ``"0"``, another digit string, or ``"N/A"`` when - validation was disabled for the run). Only the *out-of-place* - metrics are kept, matching the spec's trailing-token convention; - the in-place metrics are intentionally discarded since the - correlation study only needs one consistent number per size. - - Notes - ----- - Design: rather than hand-writing a distinct column layout per collective - (which would need to track every nccl-tests release), this uses a single - robust rule anchored on the *trailing* 8 tokens, which nccl-tests has - kept stable across collectives and versions even as leading columns - (redop, root) have been added/repurposed. A line is treated as a data - row only if tokens[0] and tokens[1] both parse as int -- this - distinguishes real data rows (which always start with two integers: - size in bytes, element count) from stray non-'#' lines (blank-ish - whitespace, malformed output, or future header formats) without needing - to hard-code the '#' comment convention as the *only* skip signal. - - This function does not raise on malformed input; unparseable lines are - silently skipped so that a partially-corrupt log (e.g. truncated by a - crashed run) still yields whatever valid rows it contains. - - Examples - -------- - >>> text = ( - ... "# nThread 1 nGpus 8\\n" - ... " 1048576 262144 float sum -1 " - ... "98.52 10.64 18.62 0 97.11 10.80 18.90 0\\n" - ... ) - >>> rows = parse_nccl_tests(text) - >>> rows[0]["size_bytes"], rows[0]["time_us"], rows[0]["wrong"] - (1048576, 98.52, '0') - """ - rows: list[dict] = [] - for line in text.splitlines(): - stripped = line.strip() - # Skip blank lines and '#'-prefixed header/comment/summary lines. - if not stripped or stripped.startswith("#"): - continue - - tokens = stripped.split() - if len(tokens) < _MIN_NCCL_TESTS_TOKENS: - continue - - try: - size_bytes = int(tokens[0]) - count = int(tokens[1]) - except ValueError: - # Not a data row (e.g. stray non-'#' text); skip rather than - # raise so one bad line doesn't abort parsing of an otherwise - # good log. - continue - - dtype = tokens[2] - try: - time_us = float(tokens[-_TRAILING_METRIC_TOKENS]) - algbw_gbps = float(tokens[-_TRAILING_METRIC_TOKENS + 1]) - busbw_gbps = float(tokens[-_TRAILING_METRIC_TOKENS + 2]) - except ValueError: - # The trailing columns didn't parse as floats -- not a real - # data row (defensive; shouldn't happen given the int checks - # above already filtered most non-data lines). - continue - wrong = tokens[-_TRAILING_METRIC_TOKENS + 3] - - rows.append( - { - "size_bytes": size_bytes, - "count": count, - "dtype": dtype, - "time_us": time_us, - "algbw_GBps": algbw_gbps, - "busbw_GBps": busbw_gbps, - "wrong": wrong, - } - ) - return rows - - -def parse_torus_bench(text: str) -> list[dict]: - """Parse the stdout of the custom ``torus_bench`` binary. - - ``torus_bench`` prints ``#``-prefixed human-readable header lines plus - machine-parseable sentinel lines of the exact form:: - - TORUSBENCH,,,,, - - Parameters - ---------- - text : str - Raw stdout captured from a ``torus_bench`` invocation. - - Returns - ------- - list of dict - One dict per ``TORUSBENCH,`` sentinel line, in file order, with - keys: ``collective`` (str), ``dims`` (str, e.g. ``"2x2x2"``), - ``size_bytes`` (int), ``time_us`` (float), ``wrong`` (str; one of - ``"0"`` (check passed), ``"1"`` (check failed), or ``"N/A"`` - (validation was not run for this data point)). - - Notes - ----- - Design: the sentinel line is comma-delimited (unlike nccl-tests' - whitespace-delimited columns) specifically so the sibling work package - could emit it without worrying about column-alignment padding; this - parser simply looks for the fixed ``TORUSBENCH,`` prefix and splits on - commas, ignoring every other line (including the human-readable ``#`` - header). This makes the parser forward-compatible with additional - ``#``-prefixed diagnostic lines torus_bench might add later. - - The ``check`` field's three-way encoding (``1``/``0``/``-``) is - remapped onto the same ``wrong`` vocabulary nccl-tests uses (a per-row - "wrongness" indicator string) so downstream consumers (rows_to_csv, - the notebook) can treat the ``wrong`` column uniformly across sources: - ``check == "1"`` (validation ran and passed) maps to ``wrong = "0"`` - (zero wrong elements); ``check == "-"`` (validation was skipped for - this run) maps to ``wrong = "N/A"``, mirroring nccl-tests' own "N/A" - convention for validation-disabled runs; anything else (i.e. - ``check == "0"``, validation ran and failed) maps to ``wrong = "1"``. - - Malformed sentinel lines (wrong field count, non-numeric size/time) are - silently skipped rather than raising, for the same reasons as - :func:`parse_nccl_tests`. - """ - rows: list[dict] = [] - for line in text.splitlines(): - stripped = line.strip() - if not stripped.startswith(_TORUS_SENTINEL_PREFIX): - continue - - fields = stripped.split(",") - # TORUSBENCH,,,,, = 6 fields. - if len(fields) != 6: - continue - - _, collective, dims, size_bytes_str, time_us_str, check = fields - try: - size_bytes = int(size_bytes_str) - time_us = float(time_us_str) - except ValueError: - continue - - if check == "1": - wrong = "0" - elif check == "-": - wrong = "N/A" - else: - wrong = "1" - - rows.append( - { - "collective": collective, - "dims": dims, - "size_bytes": size_bytes, - "time_us": time_us, - "wrong": wrong, - } - ) - return rows - - -def rows_to_csv(rows: list[dict], out_path: Union[str, Path]) -> None: - """Write unified-schema rows to a CSV file. - - Parameters - ---------- - rows : list of dict - Rows already reshaped into the unified schema (see - :data:`UNIFIED_CSV_FIELDNAMES` for the exact column set and order). - Each dict must contain every key in ``UNIFIED_CSV_FIELDNAMES``; - missing keys are written as empty cells by :class:`csv.DictWriter` - default behavior is NOT relied upon here -- callers are expected to - supply complete rows (see :func:`main` for how CLI callers build - them). Extra keys beyond the unified schema are rejected by - :class:`csv.DictWriter` (``extrasaction="raise"``, the default) so - schema drift is caught early rather than silently dropped. - out_path : str or pathlib.Path - Destination file path. Parent directories are NOT created by this - function; callers must ensure the directory exists. - - Returns - ------- - None - - Raises - ------ - ValueError - If a row dict contains a key not present in - :data:`UNIFIED_CSV_FIELDNAMES` (raised by the underlying - :class:`csv.DictWriter`). - OSError - If ``out_path`` cannot be opened for writing (e.g. parent directory - does not exist, permission denied). - - Notes - ----- - Opens the file with ``newline=""`` as recommended by the :mod:`csv` - module docs, so that the csv module's own line-ending handling is used - verbatim rather than being double-translated by Python's text-mode - newline translation. - """ - out_path = Path(out_path) - with out_path.open("w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter(f, fieldnames=UNIFIED_CSV_FIELDNAMES) - writer.writeheader() - writer.writerows(rows) - - -def _build_arg_parser() -> argparse.ArgumentParser: - """Construct the CLI argument parser for this module. - - Returns - ------- - argparse.ArgumentParser - Parser accepting the raw log path plus labeling/validation flags - described in the module CLI usage (see :func:`main`). - """ - parser = argparse.ArgumentParser( - prog="parse_nccl.py", - description=( - "Parse a raw nccl-tests or torus_bench profiling log into the " - "unified CSV schema consumed by the correlation notebook." - ), - ) - parser.add_argument( - "raw_log", - type=Path, - help="Path to the raw stdout log captured from the profiling binary.", - ) - parser.add_argument( - "--source", - required=True, - choices=["nccl-tests", "torus_bench"], - help="Which tool produced raw_log.", - ) - parser.add_argument( - "--collective", - required=True, - help=( - "Collective name. For --source nccl-tests this labels every " - "output row directly (the tool's own stdout does not name the " - "collective). For --source torus_bench this is instead " - "cross-checked against the collective embedded in each " - "TORUSBENCH sentinel line; a mismatch is an error." - ), - ) - parser.add_argument( - "--topology", - required=True, - choices=["fc", "torus"], - help="Fabric topology label to stamp onto every output row.", - ) - parser.add_argument( - "--dims", - default=None, - help=( - "Dimension string (e.g. '2x2x2'). For --source nccl-tests this " - "overrides the default dims label of '8' (the fixed GPU count " - "of a single NVSwitch-connected node). For --source " - "torus_bench, if given, it is cross-checked against the dims " - "embedded in each TORUSBENCH sentinel line; a mismatch is an " - "error. If omitted for torus_bench, the sentinel's own dims " - "value is used unchecked." - ), - ) - parser.add_argument( - "--out", - required=True, - type=Path, - help="Destination CSV path.", - ) - return parser - - -# Default dims label for nccl-tests rows when --dims is not supplied on the -# CLI. FC-leg runs are always against a single 8x-H100 NVSwitch node, so "8" -# (the GPU count) is the natural default; --dims exists mainly to let the -# CLI stay uniform with the torus_bench invocation and to support future -# multi-node FC runs without changing this module. -_DEFAULT_NCCL_TESTS_DIMS = "8" - - -def main(argv: Union[list, None] = None) -> None: - """CLI entry point: parse a raw log and write the unified CSV. - - Parameters - ---------- - argv : list of str, optional - Argument vector to parse in place of ``sys.argv[1:]``. Primarily - useful for testing; production invocations (from ``run_profile.sh``) - pass ``None`` and rely on ``sys.argv``. - - Returns - ------- - None - - Raises - ------ - SystemExit - Raised by :mod:`argparse` on invalid/missing arguments (exit code - 2), or explicitly via ``parser.error()`` when a ``--collective``/ - ``--dims`` value supplied on the CLI disagrees with the value - embedded in a torus_bench sentinel line (exit code 2). Also raised - implicitly if ``raw_log`` cannot be read (propagates as an - unhandled :class:`OSError`, not caught here -- a missing/unreadable - input log is a hard setup error the caller (``run_profile.sh``) - should see immediately rather than have masked). - """ - parser = _build_arg_parser() - args = parser.parse_args(argv) - - text = args.raw_log.read_text(encoding="utf-8") - - if args.source == "torus_bench": - parsed = parse_torus_bench(text) - # Design: validate CLI-supplied --collective/--dims against what - # the sentinel lines actually say, rather than trusting the CLI - # blindly. This catches operator error in run_profile.sh (e.g. a - # copy-paste mistake wiring the wrong collective's log into the - # wrong parse invocation) at parse time instead of silently - # mislabeling data that later gets combined into the notebook. - for parsed_row in parsed: - if parsed_row["collective"] != args.collective: - parser.error( - f"--collective {args.collective!r} does not match " - f"collective {parsed_row['collective']!r} found in " - f"{args.raw_log}" - ) - if args.dims is not None and parsed_row["dims"] != args.dims: - parser.error( - f"--dims {args.dims!r} does not match dims " - f"{parsed_row['dims']!r} found in {args.raw_log}" - ) - rows = [ - { - "source": "torus_bench", - "topology": args.topology, - "dims": parsed_row["dims"], - "collective": parsed_row["collective"], - "size_bytes": parsed_row["size_bytes"], - # Design: count/dtype/algbw/busbw are left empty for torus - # rows per spec -- bandwidth conventions for the torus - # topology (e.g. what counts as "algorithm bandwidth" when - # hops differ per link) are derived in the notebook from - # size_bytes/time_us/dims, not computed here, to keep this - # parser topology-agnostic. - "count": "", - "dtype": "", - "time_us": parsed_row["time_us"], - "algbw_GBps": "", - "busbw_GBps": "", - "wrong": parsed_row["wrong"], - } - for parsed_row in parsed - ] - else: - parsed = parse_nccl_tests(text) - dims = args.dims if args.dims is not None else _DEFAULT_NCCL_TESTS_DIMS - rows = [ - { - "source": "nccl-tests", - "topology": args.topology, - "dims": dims, - "collective": args.collective, - "size_bytes": parsed_row["size_bytes"], - "count": parsed_row["count"], - "dtype": parsed_row["dtype"], - "time_us": parsed_row["time_us"], - "algbw_GBps": parsed_row["algbw_GBps"], - "busbw_GBps": parsed_row["busbw_GBps"], - "wrong": parsed_row["wrong"], - } - for parsed_row in parsed - ] - - args.out.parent.mkdir(parents=True, exist_ok=True) - rows_to_csv(rows, args.out) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/notebooks/astrasim2_correlation/correlation/provision.py b/notebooks/astrasim2_correlation/correlation/provision.py deleted file mode 100644 index c6ca9fa4..00000000 --- a/notebooks/astrasim2_correlation/correlation/provision.py +++ /dev/null @@ -1,895 +0,0 @@ -"""Provision one p5.48xlarge (8x H100, NVSwitch) EC2 instance for NCCL profiling. - -This script is the "up" half of the correlation study's empirical leg: it -launches exactly one spot-first, on-demand-fallback instance, waits for it -to be SSH-reachable, and records everything needed to find/tear it down -again in a small JSON state file. See ``teardown.py`` for the "down" half -and ``README.md`` for the full runbook. - -Every function below is written to be independently importable and -testable: ``orchestrate.py`` (a sibling work package, written separately) -imports these functions directly rather than shelling out to this file, so -their signatures are part of this module's public contract and must not -change without updating that caller too. - -Design: boto3 is not a repo dependency ----------------------------------------- -accelforge's ``pyproject.toml`` does not (and per this work package's scope -must not) depend on ``boto3`` -- only this AWS-provisioning corner of one -notebook's correlation study needs it. The import below is therefore -guarded: importing this module never fails just because boto3 is missing, -so ``python provision.py --help`` keeps working in any environment. Actual -AWS calls fail fast with a clear "pip install boto3" message via -:func:`_require_boto3`, called once at the top of :func:`main` before any -client is constructed. -""" - -from __future__ import annotations - -import argparse -import json -import os -import socket -import sys -import time -import urllib.error -import urllib.request -from pathlib import Path -from typing import Optional - -from config import Config - -try: - import boto3 - from botocore.exceptions import ClientError -except ImportError: # pragma: no cover - exercised only when boto3 truly absent - boto3 = None - # Design: fall back to plain Exception as a placeholder so that - # `except ClientError:` clauses elsewhere in this module remain valid - # Python (no NameError at import time) even when boto3 is missing. - # Those clauses are only ever reached after _require_boto3() has - # already raised, so this placeholder is never actually matched in - # practice -- it exists purely to keep module import side-effect-free. - class ClientError(Exception): # type: ignore[no-redef] - pass - - -def _require_boto3() -> None: - """Raise a clear, actionable error if boto3 is not installed. - - Raises - ------ - SystemExit - Always, if ``boto3`` failed to import. The message tells the user - exactly how to fix it rather than surfacing a bare - ``ModuleNotFoundError`` traceback. - """ - if boto3 is None: - raise SystemExit( - "boto3 is required for AWS provisioning but is not installed in " - "this Python environment.\n" - "Install it with: pip install boto3" - ) - - -# ClientError codes under which "spot-then-ondemand" purchasing retries on -# demand instead of failing the whole run. All represent spot-market -# scarcity/limits rather than a request-shape problem, so retrying the same -# request as on-demand is expected to succeed. p5.48xlarge (8x H100) is a -# scarce, high-demand instance type, so hitting these in practice is not -# unusual and should not be treated as fatal when the operator has opted -# into a fallback. -_SPOT_FALLBACK_ERROR_CODES = frozenset( - { - "InsufficientInstanceCapacity", - "SpotMaxPriceTooLow", - "MaxSpotInstanceCountExceeded", - "Unsupported", - "InstanceLimitExceeded", - } -) - -_SSH_POLL_INTERVAL_S = 5.0 -_SSH_POLL_TIMEOUT_S = 5 * 60.0 - -_COST_WARNING = ( - "COST WARNING: p5.48xlarge on-demand pricing is roughly $30-55/hr " - "depending on region and current AWS pricing. VERIFY CURRENT PRICING " - "before proceeding: https://aws.amazon.com/ec2/pricing/on-demand/" -) - - -def resolve_ami(ssm_client, parameter: str) -> str: - """Resolve an AMI id from a public SSM parameter. - - Parameters - ---------- - ssm_client : botocore.client.BaseClient - A boto3 ``ssm`` client (or a stub thereof). - parameter : str - Fully-qualified SSM parameter name, e.g. - ``"/aws/service/deeplearning/ami/x86_64/.../latest/ami-id"``. - - Returns - ------- - str - The AMI id stored at ``parameter``. - - Raises - ------ - botocore.exceptions.ClientError - If ``parameter`` does not exist or the caller lacks - ``ssm:GetParameter`` permission; propagated unmodified so callers - see AWS's own error code and message. - """ - response = ssm_client.get_parameter(Name=parameter) - return response["Parameter"]["Value"] - - -def caller_ip() -> str: - """Discover the caller's public IPv4 address via checkip.amazonaws.com. - - Used by :func:`main` to scope the provisioned security group's SSH - ingress rule to just this machine, when the operator has not supplied - an explicit ``--ssh-cidr``. - - Returns - ------- - str - The caller's public IP as a dotted-quad string. - - Raises - ------ - RuntimeError - If the IP could not be discovered for any reason (network error, - timeout, or an empty response body). See Notes for why this fails - loudly rather than falling back to any sentinel value. - - Notes - ----- - Design: fails *fast* (raises) rather than falling back to a sentinel. - An earlier version of this function failed *open* to the sentinel - ``"0.0.0.0"`` on discovery failure, reasoning that a transient DNS blip - or checkip.amazonaws.com outage shouldn't hard-fail an - otherwise-working run. That reasoning had a bug: every caller turns - this return value into a CIDR via ``f"{ip}/32"``, so the sentinel - actually produced ``"0.0.0.0/32"`` -- a CIDR matching no address at - all -- which locks *everyone*, including the operator, out over SSH, - the opposite of what the old warning text claimed ("falling back to - 0.0.0.0/0", i.e. open to the world). Worse, that silent misconfiguration - was only discoverable *after* a real key pair, security group, and - instance had already been created and billing had already started. - Raising here instead is strictly better on both axes this function - cares about -- SAFE (no accidental everyone-blocked security group) and - SECURE (no accidental world-open one either) -- and it fires before any - AWS resource exists or any money is spent: both call sites - (``provision.main`` and ``orchestrate.main``) invoke this function only - when ``--ssh-cidr`` was not supplied, and always before - ``ensure_key_pair``/``ensure_security_group``/``launch_instance``. The - error message tells the operator exactly how to proceed: re-run with - ``--ssh-cidr /32``. - """ - try: - with urllib.request.urlopen("https://checkip.amazonaws.com", timeout=10) as resp: - ip = resp.read().decode("utf-8").strip() - if not ip: - raise ValueError("empty response body from checkip.amazonaws.com") - return ip - except (urllib.error.URLError, ValueError, OSError) as exc: - raise RuntimeError( - "Could not determine your public IP via checkip.amazonaws.com " - f"({exc!r}). No AWS resources have been created yet, so there is " - "nothing to clean up -- re-run with --ssh-cidr /32 to " - "supply your CIDR explicitly instead of relying on auto-detection." - ) from exc - - -def ensure_key_pair(ec2_client, key_name: str, key_dir: Path) -> Path: - """Create (or reuse) an EC2 key pair and its local PEM file. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - key_name : str - Name to give the key pair in AWS. - key_dir : pathlib.Path - Local directory to write ``.pem`` into. Created if it - does not already exist. - - Returns - ------- - pathlib.Path - Path to the local PEM file (either freshly written, or the - existing one being reused). - - Raises - ------ - RuntimeError - If AWS reports the key pair already exists (``ClientError`` code - ``InvalidKeyPair.Duplicate``) but no local PEM file is present. - AWS never returns private key material for a pre-existing key - pair, so there is no way to recover the PEM in this situation -- - the caller must delete the AWS-side key pair or choose a - different ``run_id``. - botocore.exceptions.ClientError - For any other ``create_key_pair`` failure, propagated unmodified. - - Notes - ----- - Side effect: writes a file to disk at ``/.pem`` with - ``0o600`` permissions (owner read/write only, matching what ``ssh`` - requires of private key files). - """ - key_dir.mkdir(parents=True, exist_ok=True) - key_path = key_dir / f"{key_name}.pem" - - try: - response = ec2_client.create_key_pair( - KeyName=key_name, KeyType="rsa", KeyFormat="pem" - ) - except ClientError as exc: - error_code = exc.response.get("Error", {}).get("Code", "") - if error_code == "InvalidKeyPair.Duplicate": - if key_path.exists(): - print( - f"Key pair {key_name!r} already exists in AWS and a local " - f"PEM was found at {key_path}; reusing it." - ) - return key_path - raise RuntimeError( - f"Key pair {key_name!r} already exists in AWS, but no local " - f"PEM file was found at {key_path}. AWS never returns private " - "key material for a pre-existing key pair, so it cannot be " - "recovered. Either delete the AWS-side key pair " - f"(aws ec2 delete-key-pair --key-name {key_name}) and re-run, " - "or pass a different --run-id so a fresh key pair name is used." - ) from exc - raise - - key_material = response["KeyMaterial"] - # Design: use os.open with the 0o600 mode baked into file creation - # (rather than write-then-chmod) so the PEM is never briefly readable - # at default (often world-readable) permissions between those two - # steps. - fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "w") as fh: - fh.write(key_material) - return key_path - - -def ensure_security_group( - ec2_client, group_name: str, ssh_cidr: str, tag_project: str, run_id: str -) -> str: - """Create a security group in the default VPC allowing SSH from one CIDR. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - group_name : str - Name to give the new security group. - ssh_cidr : str - CIDR block (e.g. ``"1.2.3.4/32"`` or ``"0.0.0.0/0"``) to allow - inbound TCP/22 from. - tag_project : str - Value for the ``Project`` tag on the new group. - run_id : str - Value for the ``RunId`` tag on the new group, and included in its - description. - - Returns - ------- - str - The new security group's id. - - Raises - ------ - RuntimeError - If the region has no default VPC (``describe_vpcs`` returns no - results for ``isDefault=true``). NCCL profiling on a single node - has no cross-VPC requirements, so this script deliberately does - not attempt to create or select a non-default VPC -- that is out - of scope for a short-lived profiling instance. - botocore.exceptions.ClientError - For any ``create_security_group``/``authorize_security_group_ingress``/ - ``create_tags`` failure, propagated unmodified. - """ - vpcs = ec2_client.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}]) - vpc_list = vpcs.get("Vpcs", []) - if not vpc_list: - raise RuntimeError( - "No default VPC found in this region. Create one " - "(aws ec2 create-default-vpc) or provision a VPC manually, then " - "re-run." - ) - vpc_id = vpc_list[0]["VpcId"] - - create_resp = ec2_client.create_security_group( - GroupName=group_name, - Description=f"accelforge correlation study SG for run {run_id}", - VpcId=vpc_id, - ) - sg_id = create_resp["GroupId"] - - ec2_client.authorize_security_group_ingress( - GroupId=sg_id, - IpPermissions=[ - { - "IpProtocol": "tcp", - "FromPort": 22, - "ToPort": 22, - "IpRanges": [ - { - "CidrIp": ssh_cidr, - "Description": "SSH access for accelforge correlation study", - } - ], - } - ], - ) - - ec2_client.create_tags( - Resources=[sg_id], - Tags=[ - {"Key": "Project", "Value": tag_project}, - {"Key": "RunId", "Value": run_id}, - {"Key": "Name", "Value": group_name}, - ], - ) - return sg_id - - -def _build_run_instances_kwargs( - cfg: Config, ami_id: str, sg_id: str, key_name: str, dry_run: bool, use_spot: bool -) -> dict: - """Build the ``run_instances`` kwargs shared by the spot and on-demand paths. - - Parameters - ---------- - cfg : Config - Run configuration. - ami_id : str - AMI id resolved by :func:`resolve_ami`. - sg_id : str - Security group id from :func:`ensure_security_group`. - key_name : str - Key pair name from :func:`ensure_key_pair`. - dry_run : bool - Whether to set ``DryRun=True`` on the request. - use_spot : bool - Whether to request a spot instance (adds ``InstanceMarketOptions``) - or an on-demand one. - - Returns - ------- - dict - Keyword arguments ready to pass to ``ec2_client.run_instances(**kwargs)``. - - Notes - ----- - Factored out of :func:`launch_instance` so the spot attempt and the - on-demand fallback attempt build their request the same way apart from - the one ``InstanceMarketOptions`` difference -- avoids the two paths - silently drifting apart (e.g. one attempt forgetting a tag) as this - function evolves. - """ - kwargs = { - "ImageId": ami_id, - "InstanceType": cfg.instance_type, - "KeyName": key_name, - "SecurityGroupIds": [sg_id], - "MinCount": 1, - "MaxCount": 1, - # "terminate" (not "stop") so an in-instance `shutdown` -- e.g. the - # dead-man timer armed by setup scripts -- fully releases the - # instance rather than leaving it (and its EBS billing) stopped - # but still provisioned. - "InstanceInitiatedShutdownBehavior": "terminate", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": cfg.root_volume_gb, - "VolumeType": "gp3", - "DeleteOnTermination": True, - }, - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - {"Key": "Project", "Value": cfg.tag_project}, - {"Key": "RunId", "Value": cfg.run_id}, - {"Key": "Name", "Value": f"{cfg.tag_project}-{cfg.run_id}"}, - ], - }, - { - "ResourceType": "volume", - "Tags": [ - {"Key": "Project", "Value": cfg.tag_project}, - {"Key": "RunId", "Value": cfg.run_id}, - {"Key": "Name", "Value": f"{cfg.tag_project}-{cfg.run_id}"}, - ], - }, - ], - "DryRun": dry_run, - } - if cfg.availability_zone: - kwargs["Placement"] = {"AvailabilityZone": cfg.availability_zone} - if use_spot: - kwargs["InstanceMarketOptions"] = { - "MarketType": "spot", - "SpotOptions": { - "SpotInstanceType": "one-time", - "InstanceInterruptionBehavior": "terminate", - }, - } - return kwargs - - -def launch_instance( - ec2_client, - cfg: Config, - ami_id: str, - sg_id: str, - key_name: str, - dry_run: bool = False, -) -> dict: - """Launch exactly one instance, honoring ``cfg.purchasing``. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - cfg : Config - Run configuration; ``cfg.purchasing`` selects the strategy below. - ami_id : str - AMI id resolved by :func:`resolve_ami`. - sg_id : str - Security group id from :func:`ensure_security_group`. - key_name : str - Key pair name from :func:`ensure_key_pair`. - dry_run : bool, default False - If ``True``, sets ``DryRun=True`` on every ``run_instances`` call. - AWS answers a dry run with an error either way: ``DryRunOperation`` - means the call would have succeeded, ``UnauthorizedOperation`` - means the caller lacks permission. This function treats those two - codes accordingly rather than as generic failures. - - Returns - ------- - dict - ``{"instance_id": str or None, "purchasing_used": "spot" or "ondemand"}``. - ``instance_id`` is ``None`` when ``dry_run=True`` and the - authorization check succeeded, since no instance was actually - created in that case. - - Raises - ------ - RuntimeError - If a dry run reports ``UnauthorizedOperation`` (the configured - credentials cannot launch this instance type/configuration). - botocore.exceptions.ClientError - - If ``cfg.purchasing == "spot"`` and the spot request fails for - any reason (no fallback is attempted in this mode). - - If ``cfg.purchasing == "spot-then-ondemand"`` and the spot - request fails with a code *not* in - :data:`_SPOT_FALLBACK_ERROR_CODES` (that set is deliberately - narrow -- e.g. a malformed request should fail loudly rather - than silently retrying as on-demand and masking the bug). - - If the (possibly-fallback) on-demand request itself fails. - - Notes - ----- - Purchasing strategies: - - - ``"ondemand"``: on-demand only, no spot attempt. - - ``"spot"``: spot only; any failure propagates without a fallback. - - ``"spot-then-ondemand"`` (the default): attempts spot first. If that - attempt fails with one of :data:`_SPOT_FALLBACK_ERROR_CODES` -- - capacity/limit/market conditions rather than a malformed request -- - it prints the failure and retries once as on-demand. Any other - ``ClientError`` code (e.g. a parameter validation error) propagates - immediately without a fallback attempt, since retrying on-demand - would not fix a malformed request and would only obscure the real - error. - """ - - def _run(use_spot: bool) -> dict: - kwargs = _build_run_instances_kwargs(cfg, ami_id, sg_id, key_name, dry_run, use_spot) - try: - response = ec2_client.run_instances(**kwargs) - except ClientError as exc: - code = exc.response.get("Error", {}).get("Code", "") - if code == "DryRunOperation": - # AWS's DryRun contract: this specific error code means - # "you WOULD have been authorized to make this call" -- it - # is deliberately raised as an error even on the success - # path, so seeing it here IS the successful outcome of a - # dry run, not a failure. - print("dry-run OK: authorized") - return { - "instance_id": None, - "purchasing_used": "spot" if use_spot else "ondemand", - } - if code == "UnauthorizedOperation": - raise RuntimeError( - "AWS denied the run_instances permission check " - "(UnauthorizedOperation). The configured credentials lack " - f"ec2:RunInstances (or a related) permission for " - f"{cfg.instance_type}." - ) from exc - raise - instance_id = response["Instances"][0]["InstanceId"] - return { - "instance_id": instance_id, - "purchasing_used": "spot" if use_spot else "ondemand", - } - - if cfg.purchasing == "ondemand": - return _run(use_spot=False) - - if cfg.purchasing == "spot": - return _run(use_spot=True) - - # cfg.purchasing == "spot-then-ondemand" (validated by Config.__post_init__ - # to be one of exactly these three values). - try: - return _run(use_spot=True) - except ClientError as exc: - code = exc.response.get("Error", {}).get("Code", "") - if code in _SPOT_FALLBACK_ERROR_CODES: - print(f"Spot request failed ({code}); falling back to on-demand.") - return _run(use_spot=False) - raise - - -def wait_for_instance(ec2_client, instance_id: str) -> str: - """Block until an instance is running and accepting TCP connections on port 22. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - instance_id : str - Id of the instance to wait for. - - Returns - ------- - str - The instance's public IPv4 address. - - Raises - ------ - RuntimeError - If ``describe_instances`` returns no matching instance, or the - instance has no public IP address (e.g. it landed in a subnet that - does not auto-assign one). - TimeoutError - If port 22 does not become reachable within - :data:`_SSH_POLL_TIMEOUT_S` seconds of the instance reaching the - ``running`` state. - botocore.exceptions.WaiterError - If the ``instance_running`` waiter itself times out or the - instance transitions to a terminal failure state. - - Notes - ----- - Two-stage wait, because "EC2 says running" and "sshd is accepting - connections" are different events with a real gap between them (boot, - cloud-init, driver/NCCL setup on the deep learning AMI): first the - ``instance_running`` waiter (AWS-side state), then a plain TCP connect - poll against port 22 (this script's own liveness check), every - :data:`_SSH_POLL_INTERVAL_S` seconds for up to - :data:`_SSH_POLL_TIMEOUT_S`. - """ - waiter = ec2_client.get_waiter("instance_running") - waiter.wait(InstanceIds=[instance_id]) - - describe = ec2_client.describe_instances(InstanceIds=[instance_id]) - reservations = describe.get("Reservations", []) - if not reservations or not reservations[0].get("Instances"): - raise RuntimeError( - f"describe_instances returned no data for instance {instance_id!r}" - ) - instance = reservations[0]["Instances"][0] - public_ip = instance.get("PublicIpAddress") - if not public_ip: - raise RuntimeError( - f"Instance {instance_id} is running but has no public IP address. " - "Check that its subnet auto-assigns public IPs." - ) - - _wait_for_ssh_port(public_ip) - return public_ip - - -def _wait_for_ssh_port( - host: str, - port: int = 22, - interval_s: float = _SSH_POLL_INTERVAL_S, - timeout_s: float = _SSH_POLL_TIMEOUT_S, -) -> None: - """Poll a TCP port until it accepts a connection or a timeout elapses. - - Parameters - ---------- - host : str - Hostname or IP address to connect to. - port : int, default 22 - TCP port to poll. - interval_s : float, default 5.0 - Seconds to sleep between connection attempts. - timeout_s : float, default 300.0 - Total seconds to poll before giving up. - - Raises - ------ - TimeoutError - If no connection succeeds within ``timeout_s`` seconds. - - Notes - ----- - Uses ``socket.create_connection`` (a plain TCP connect/close) rather - than an actual SSH handshake -- sufficient to confirm sshd is up - without adding a paramiko/fabric dependency for a single boolean - liveness check. - """ - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - try: - with socket.create_connection((host, port), timeout=interval_s): - return - except OSError: - pass - time.sleep(interval_s) - raise TimeoutError( - f"Timed out after {timeout_s}s waiting for {host}:{port} to accept " - "TCP connections (SSH not yet reachable)." - ) - - -def write_state(state_dir, run_id: str, state: dict) -> Path: - """Write a run's provisioning state to ``/.json``. - - Parameters - ---------- - state_dir : str or pathlib.Path - Directory to write the state file into. Created if it does not - already exist. - run_id : str - Run identifier; also the state file's basename (without ``.json``). - state : dict - JSON-serializable state to write. Expected (by ``teardown.py``) to - contain ``run_id``, ``region``, ``instance_id``, ``sg_id``, - ``key_name``, ``key_path``, ``public_ip``, ``purchasing_used``, and - ``ami_id``, but this function itself does not validate the shape - of ``state`` -- it is a thin, schema-agnostic writer so callers - (including future ones) are free to add fields. - - Returns - ------- - pathlib.Path - Path to the written state file. - - Notes - ----- - Side effect: writes ``/.json``, creating - ``state_dir`` if needed. Uses ``json.dump(..., default=str)`` so a - stray non-JSON-native value (e.g. if a caller forgets to stringify a - ``pathlib.Path``) is coerced to its string form instead of raising a - ``TypeError`` deep in a provisioning run. - """ - state_dir = Path(state_dir) - state_dir.mkdir(parents=True, exist_ok=True) - state_path = state_dir / f"{run_id}.json" - with open(state_path, "w") as fh: - json.dump(state, fh, indent=2, default=str) - return state_path - - -def _prompt_yes_no(prompt: str) -> bool: - """Ask an interactive yes/no question, returning ``True`` only for "yes". - - Parameters - ---------- - prompt : str - Text to show before the input cursor. - - Returns - ------- - bool - ``True`` if the user typed exactly ``"yes"`` (case-insensitive, - surrounding whitespace ignored); ``False`` for anything else, - including EOF/blank input. Requiring the full word "yes" (not just - "y") is a deliberate speed bump before an action that costs real - money. - """ - try: - answer = input(prompt) - except EOFError: - return False - return answer.strip().lower() == "yes" - - -def main(argv: Optional[list] = None) -> int: - """Parse CLI args and provision one instance end to end. - - Parameters - ---------- - argv : list[str] or None, default None - Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` - reads from ``sys.argv[1:]``. - - Returns - ------- - int - Process exit code: ``0`` on success (or a completed dry run), - ``1`` if the user declined the cost confirmation prompt. - - Notes - ----- - Side effects (skipped entirely when ``--dry-run`` is passed, except - for the ``DryRun=True`` API calls themselves): creates an SSH key pair - and local PEM, creates a security group, launches an instance, waits - for it to be SSH-reachable, and writes a state JSON file -- TWICE: once - immediately after launch (``public_ip`` as an explicit placeholder) and - again once ``wait_for_instance`` returns a real IP, so a crash or - interruption during the (potentially multi-minute) SSH wait still - leaves a local record of a running, billing instance. See the inline - "Design/WHY" comment at that first ``write_state`` call for the full - rationale. Prints a cost warning and requires interactive ``"yes"`` - confirmation before any of that happens, unless ``--yes`` is passed. - - This function deliberately never tears anything down itself, even on a - ``wait_for_instance`` failure -- ``provision.py``'s contract is - provision-and-leave-running (see ``README.md``'s "advanced / manual - control" section); recovery in that failure case is a printed - ``teardown.py`` command for the operator to run, not an automatic call. - """ - parser = argparse.ArgumentParser( - prog="provision.py", - description=( - "Provision one p5.48xlarge (8x H100, NVSwitch) EC2 instance, " - "spot-first with on-demand fallback, for NCCL profiling." - ), - ) - Config.add_args(parser) - parser.add_argument( - "--dry-run", - action="store_true", - help="Perform DryRun=True authorization checks only; launch nothing.", - ) - parser.add_argument( - "--ssh-cidr", - type=str, - default=None, - help=( - "CIDR block allowed SSH access, e.g. 1.2.3.4/32. Overrides " - "auto-detected caller IP (see caller_ip())." - ), - ) - parser.add_argument( - "--yes", - action="store_true", - help="Skip the interactive cost-confirmation prompt.", - ) - args = parser.parse_args(argv) - cfg = Config.from_parsed(args) - - # Fail fast on a missing boto3 before printing anything else -- no - # point asking the operator to confirm a cost warning for a run that - # cannot possibly proceed. - _require_boto3() - - print(f"=== accelforge correlation-study provisioning: run_id={cfg.run_id} ===") - print(_COST_WARNING) - print( - f"Purchasing mode: {cfg.purchasing}. Instance type: {cfg.instance_type}. " - f"Region: {cfg.region}." - ) - - if not args.dry_run and not args.yes: - if not _prompt_yes_no("Proceed with provisioning? [yes/N]: "): - print("Aborted by user.") - return 1 - - ec2_client = boto3.client("ec2", region_name=cfg.region) - ssm_client = boto3.client("ssm", region_name=cfg.region) - - ssh_cidr = args.ssh_cidr - if not ssh_cidr: - ip = caller_ip() - ssh_cidr = f"{ip}/32" - print(f"SSH will be allowed from: {ssh_cidr}") - - ami_id = resolve_ami(ssm_client, cfg.ami_ssm_parameter) - print(f"Resolved AMI: {ami_id}") - - key_name = f"{cfg.tag_project}-{cfg.run_id}" - key_path = ensure_key_pair(ec2_client, key_name, cfg.key_dir) - print(f"Key pair ready: {key_name} -> {key_path}") - - sg_name = f"{cfg.tag_project}-{cfg.run_id}-sg" - sg_id = ensure_security_group(ec2_client, sg_name, ssh_cidr, cfg.tag_project, cfg.run_id) - print(f"Security group ready: {sg_id}") - - launch_result = launch_instance( - ec2_client, cfg, ami_id, sg_id, key_name, dry_run=args.dry_run - ) - if args.dry_run: - print( - f"Dry run complete (purchasing checked: {launch_result['purchasing_used']}); " - "no instance was launched." - ) - return 0 - - instance_id = launch_result["instance_id"] - purchasing_used = launch_result["purchasing_used"] - print(f"Launched instance {instance_id} ({purchasing_used}).") - - # Design/WHY (closes an orphan-instance window): write the state file -- - # with public_ip as an explicit placeholder -- IMMEDIATELY after - # launch_instance returns, before wait_for_instance is even called. - # wait_for_instance can block for several minutes (instance boot, then - # polling for SSH) and can itself raise (TimeoutError, WaiterError, or - # simply be interrupted by Ctrl-C/a killed process). Before this fix, a - # crash in that window left an instance running and billing with NO - # local record of it at all, since write_state was only ever called - # once, after wait_for_instance returned successfully. Writing state - # now -- and rewriting it once the real public_ip is known below -- - # means teardown.py can always find and tear down this run from local - # state alone, even if this process never gets past wait_for_instance. - state = { - "run_id": cfg.run_id, - "region": cfg.region, - "instance_id": instance_id, - "sg_id": sg_id, - "key_name": key_name, - "key_path": str(key_path), - "public_ip": None, - "purchasing_used": purchasing_used, - "ami_id": ami_id, - } - state_path = write_state(cfg.state_dir, cfg.run_id, state) - print(f"State written to: {state_path} (public_ip pending SSH reachability).") - - print("Waiting for it to become reachable...") - try: - public_ip = wait_for_instance(ec2_client, instance_id) - except Exception: - # The instance is still running (and billing) regardless of why - # wait_for_instance failed -- print a loud, impossible-to-miss block - # naming exactly how to find and recover it, then re-raise - # unmodified so this failure still surfaces as a nonzero exit code - # (never silently swallowed). - print("=" * 70, file=sys.stderr) - print("ERROR: wait_for_instance failed (see traceback below).", file=sys.stderr) - print(f"Instance {instance_id} IS STILL RUNNING AND BILLING.", file=sys.stderr) - print(f"State file: {state_path}", file=sys.stderr) - print( - f"Recover with: python teardown.py --run-id {cfg.run_id} --region {cfg.region}", - file=sys.stderr, - ) - print("=" * 70, file=sys.stderr) - raise - - print(f"Instance is running and SSH-reachable at {public_ip}") - - state["public_ip"] = public_ip - state_path = write_state(cfg.state_dir, cfg.run_id, state) - - print(f"State written to: {state_path}") - print(f"Public IP: {public_ip}") - print(f"SSH command: ssh -i {key_path} {cfg.ssh_user}@{public_ip}") - print( - "Remember: tear this down when done " - f"(python teardown.py --run-id {cfg.run_id})." - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/run_profile.sh b/notebooks/astrasim2_correlation/correlation/run_profile.sh deleted file mode 100644 index 372f580e..00000000 --- a/notebooks/astrasim2_correlation/correlation/run_profile.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# -# run_profile.sh -- run one profiling leg (FC or torus) for one or more -# NCCL collectives on a provisioned 8x-H100 instance, and parse each raw -# log into the unified CSV schema consumed by the correlation notebook. -# -# This script runs ON the EC2 instance (not locally); it has no GPUs to -# talk to when checked out anywhere else, so it is verified here by -# `bash -n` (syntax check) only -- see the WP2 report for details. -# -# Usage: -# ./run_profile.sh \ -# [...] -# -# Env overrides: -# NCCL_TESTS_DIR Path to a built nccl-tests checkout. Default: $HOME/nccl-tests -# TORUS_BENCH_BIN Path to the built torus_bench binary. Default: $HOME/torus_bench/torus_bench -# PARSE_NCCL Path to parse_nccl.py. Default: the copy next to this script. -# WARMUP nccl-tests/torus_bench warmup iteration count. Default: 5 -# ITERS nccl-tests/torus_bench measured iteration count. Default: 20 -# -# Design: fail fast and loud (set -euo pipefail) rather than silently -# continuing past a failed collective run or a failed parse -- a partial, -# uncaught failure here would otherwise show up much later as a confusing -# gap in the correlation notebook's data rather than as a build/run error -# on the instance where it's cheap to diagnose. -set -euo pipefail - -# --------------------------------------------------------------------------- -# Argument parsing -# --------------------------------------------------------------------------- -if [[ $# -lt 6 ]]; then - echo "Usage: $0 [...]" >&2 - exit 1 -fi - -results_dir="$1"; shift -topology="$1"; shift -min_bytes="$1"; shift -max_bytes="$1"; shift -dims="$1"; shift -# Remaining positional args are the list of collectives to profile in this leg. -collectives=("$@") - -if [[ "$topology" != "fc" && "$topology" != "torus" ]]; then - echo "ERROR: must be 'fc' or 'torus', got '$topology'" >&2 - exit 1 -fi - -# --------------------------------------------------------------------------- -# Environment / defaults -# --------------------------------------------------------------------------- -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -NCCL_TESTS_DIR="${NCCL_TESTS_DIR:-$HOME/nccl-tests}" -TORUS_BENCH_BIN="${TORUS_BENCH_BIN:-$HOME/torus_bench/torus_bench}" -# Design: PARSE_NCCL defaults to the copy sitting next to this script -# (rather than requiring it on PATH or hard-coding an absolute install -# path) so the pair of files can be scp'd to the instance as a unit and -# just work. -PARSE_NCCL="${PARSE_NCCL:-$SCRIPT_DIR/parse_nccl.py}" -WARMUP="${WARMUP:-5}" -ITERS="${ITERS:-20}" - -RAW_DIR="$results_dir/raw" -CSV_DIR="$results_dir/csv" -mkdir -p "$RAW_DIR" "$CSV_DIR" - -# --------------------------------------------------------------------------- -# metadata.txt: written exactly once per invocation (not once per -# collective), since it captures machine/software state that doesn't -# change across the collectives loop below. -# --------------------------------------------------------------------------- -metadata_file="$results_dir/metadata.txt" -{ - echo "=== date ===" - date -u - echo - echo "=== uname -a ===" - uname -a - echo - echo "=== nvidia-smi --query-gpu=name,driver_version --format=csv ===" - nvidia-smi --query-gpu=name,driver_version --format=csv - echo - echo "=== nvidia-smi topo -m ===" - nvidia-smi topo -m - echo - echo "=== nccl-tests git rev ===" - # Best-effort: nccl-tests may not be a git checkout (e.g. if a tarball - # was scp'd instead), so a failed `git rev-parse` here must not abort - # the whole script under `set -e`. - git -C "$NCCL_TESTS_DIR" rev-parse HEAD 2>/dev/null || echo "unknown (not a git checkout or NCCL_TESTS_DIR missing)" -} > "$metadata_file" -echo "Wrote $metadata_file" - -# --------------------------------------------------------------------------- -# FC leg: stock nccl-tests binaries. -# --------------------------------------------------------------------------- -run_fc_leg() { - local collective="$1" - local bin="$NCCL_TESTS_DIR/build/${collective}_perf" - local raw_log="$RAW_DIR/fc_${collective}.log" - local out_csv="$CSV_DIR/fc_${collective}.csv" - - if [[ ! -x "$bin" ]]; then - echo "ERROR: nccl-tests binary not found: $bin" >&2 - echo " Build nccl-tests first, e.g. via setup_node.sh, or:" >&2 - echo " make -j -C \"$NCCL_TESTS_DIR\" MPI=0 CUDA_HOME=\"\${CUDA_HOME:-/usr/local/cuda}\"" >&2 - exit 1 - fi - - echo "=== FC leg: $collective ===" - "$bin" -b "$min_bytes" -e "$max_bytes" -f 2 -g 8 -w "$WARMUP" -n "$ITERS" -c 1 | tee "$raw_log" - python3 "$PARSE_NCCL" "$raw_log" \ - --source nccl-tests \ - --collective "$collective" \ - --topology fc \ - --dims "$dims" \ - --out "$out_csv" - - # Fail loudly on a header-only CSV: parse_nccl.py can exit 0 while emitting zero data rows - # (e.g. every swept size hit a divisibility skip for this dims/size combination), which - # would otherwise look identical to a real, successful leg -- a silent data gap discovered - # only much later in the correlation notebook, rather than here, where the raw log needed - # to diagnose it is still on disk and cheap to inspect. - csv_lines="$(wc -l < "$out_csv")" - if [[ "$csv_lines" -le 1 ]]; then - echo "ERROR: $out_csv has no data rows (header-only or empty) -- see $raw_log" >&2 - exit 1 - fi - echo "Wrote $out_csv" -} - -# --------------------------------------------------------------------------- -# Torus leg: custom torus_bench binary (built by the sibling work package). -# --------------------------------------------------------------------------- -run_torus_leg() { - local collective="$1" - local raw_log="$RAW_DIR/torus_${collective}.log" - local out_csv="$CSV_DIR/torus_${collective}.csv" - - if [[ ! -x "$TORUS_BENCH_BIN" ]]; then - echo "ERROR: torus_bench binary not found: $TORUS_BENCH_BIN" >&2 - echo " Build torus_bench first (see setup_node.sh), e.g.:" >&2 - echo " make -C \"\$TORUS_BENCH_DIR\" torus_bench" >&2 - exit 1 - fi - - echo "=== Torus leg: $collective ===" - "$TORUS_BENCH_BIN" --collective "$collective" --dims "$dims" \ - -b "$min_bytes" -e "$max_bytes" -f 2 -w "$WARMUP" -n "$ITERS" --check \ - | tee "$raw_log" - python3 "$PARSE_NCCL" "$raw_log" \ - --source torus_bench \ - --collective "$collective" \ - --topology torus \ - --dims "$dims" \ - --out "$out_csv" - - # See run_fc_leg's identical check above for the full rationale: a header-only CSV here - # (e.g. every swept size hit torus_bench's own divisibility SKIP for this dims/size - # combination) must fail the script loudly rather than silently passing as "done". - csv_lines="$(wc -l < "$out_csv")" - if [[ "$csv_lines" -le 1 ]]; then - echo "ERROR: $out_csv has no data rows (header-only or empty) -- see $raw_log" >&2 - exit 1 - fi - echo "Wrote $out_csv" -} - -# --------------------------------------------------------------------------- -# Main loop: one leg (selected by $topology), all requested collectives. -# --------------------------------------------------------------------------- -for collective in "${collectives[@]}"; do - if [[ "$topology" == "fc" ]]; then - run_fc_leg "$collective" - else - run_torus_leg "$collective" - fi -done - -echo "Done. Results in $results_dir" diff --git a/notebooks/astrasim2_correlation/correlation/setup_node.sh b/notebooks/astrasim2_correlation/correlation/setup_node.sh deleted file mode 100644 index ec407e99..00000000 --- a/notebooks/astrasim2_correlation/correlation/setup_node.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# -# setup_node.sh -- idempotent one-time (but safe-to-rerun) setup for an -# EC2 p5.48xlarge (8x H100, NVSwitch) profiling instance: arms a dead-man -# shutdown, verifies the GPU fabric is visible, builds nccl-tests and -# torus_bench if not already built, and prints a versions summary. -# -# This script runs ON the EC2 instance; it is verified here by `bash -n` -# (syntax check) only -- see the WP2 report for details. -# -# Env: -# DEADMAN_MINUTES Minutes until the dead-man shutdown fires. Default: 120 -# NCCL_TESTS_DIR Where to clone/build nccl-tests. Default: $HOME/nccl-tests -# TORUS_BENCH_DIR Where torus_bench is scp'd/built. Default: $HOME/torus_bench -# CUDA_HOME CUDA toolkit root used to build nccl-tests. Default: /usr/local/cuda -set -euo pipefail - -DEADMAN_MINUTES="${DEADMAN_MINUTES:-120}" -NCCL_TESTS_DIR="${NCCL_TESTS_DIR:-$HOME/nccl-tests}" -TORUS_BENCH_DIR="${TORUS_BENCH_DIR:-$HOME/torus_bench}" -CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" - -# --------------------------------------------------------------------------- -# Step 1: arm the dead-man switch FIRST, before anything else can fail or -# hang. -# -# Design/WHY this must be first: the instance is launched with -# shutdown-behavior=terminate, so a `shutdown -P` here is what actually -# terminates (not just stops) the instance and stops billing. If setup -# were to fail, hang (e.g. a stuck `make`, a stalled git clone over a flaky -# network), or if the orchestrating controller process on the caller's side -# dies/loses connectivity, this is the only backstop that guarantees the -# (expensive, 8x H100) instance doesn't run forever. Arming it before any -# other step -- including the GPU sanity check below, which could itself -# hang on a broken driver -- ensures the cost cap applies unconditionally -# from the very start of setup, not only after setup "succeeds". -# --------------------------------------------------------------------------- -echo "Arming dead-man shutdown: instance will terminate in ${DEADMAN_MINUTES} minutes unless this script (or a later run of it) is used to push it back further." -# Cancel any already-pending shutdown before arming a new one: issuing a second `shutdown` -# while one is already pending errors on some systemd versions, and cancel-then-rearm also -# makes a re-run of this script push the deadline BACK (rather than erroring or stacking), -# which is the desired semantics for a legitimate re-setup (e.g. extending a long-running -# sweep with a fresh DEADMAN_MINUTES). `|| true` because there being no pending shutdown to -# cancel (the common case, e.g. this script's first run) is not an error. -sudo shutdown -c 2>/dev/null || true -sudo shutdown -P "+${DEADMAN_MINUTES}" - -# --------------------------------------------------------------------------- -# Step 2: verify nvidia-smi works and all 8 GPUs are visible. Fail loudly -# (rather than proceeding to build against a broken/partial driver) since -# every downstream profiling run depends on this. -# --------------------------------------------------------------------------- -if ! command -v nvidia-smi >/dev/null 2>&1; then - echo "ERROR: nvidia-smi not found on PATH. Is the NVIDIA driver installed?" >&2 - exit 1 -fi - -if ! nvidia-smi >/dev/null; then - echo "ERROR: nvidia-smi is present but failed to run. Driver/GPU problem?" >&2 - exit 1 -fi - -gpu_count="$(nvidia-smi -L | wc -l)" -if [[ "$gpu_count" -ne 8 ]]; then - echo "ERROR: expected 8 GPUs (p5.48xlarge), found $gpu_count. Aborting." >&2 - exit 1 -fi -echo "OK: nvidia-smi reports $gpu_count GPUs." - -# --------------------------------------------------------------------------- -# Step 3: build nccl-tests if it isn't already built. -# -# Idempotent: only clones if NCCL_TESTS_DIR doesn't exist yet, and only -# (re)builds if the all_reduce_perf binary is missing -- a rerun of this -# script after a successful first run is a fast no-op here. -# --------------------------------------------------------------------------- -if [[ -x "$NCCL_TESTS_DIR/build/all_reduce_perf" ]]; then - echo "OK: nccl-tests already built at $NCCL_TESTS_DIR." -else - if [[ ! -d "$NCCL_TESTS_DIR" ]]; then - echo "Cloning nccl-tests into $NCCL_TESTS_DIR ..." - git clone https://github.com/NVIDIA/nccl-tests "$NCCL_TESTS_DIR" - fi - echo "Building nccl-tests (MPI=0, CUDA_HOME=$CUDA_HOME) ..." - make -j -C "$NCCL_TESTS_DIR" MPI=0 CUDA_HOME="$CUDA_HOME" -fi - -# --------------------------------------------------------------------------- -# Step 4: build torus_bench if it isn't already built. -# -# TORUS_BENCH_DIR is scp'd onto the instance by the orchestrator (a sibling -# work package owns the torus_bench source); if it hasn't landed yet, warn -# and continue rather than failing -- the FC leg (nccl-tests) can still run -# without it, and setup_node.sh may legitimately run before the orchestrator -# has finished copying torus_bench over. -# --------------------------------------------------------------------------- -torus_bench_bin="$TORUS_BENCH_DIR/torus_bench" -if [[ -x "$torus_bench_bin" ]]; then - echo "OK: torus_bench already built at $torus_bench_bin." -elif [[ -d "$TORUS_BENCH_DIR" ]]; then - echo "Building torus_bench ..." - make -C "$TORUS_BENCH_DIR" torus_bench -else - echo "WARNING: $TORUS_BENCH_DIR not found (expected to be scp'd there by the orchestrator)." >&2 - echo " Skipping torus_bench build; the FC leg can still run without it." >&2 -fi - -# --------------------------------------------------------------------------- -# Step 5: print a versions summary for the run's metadata/provenance. -# Every lookup here is best-effort (guarded so a missing tool doesn't abort -# the script under `set -e`), since this is diagnostic output, not a hard -# requirement. -# --------------------------------------------------------------------------- -echo "=== Versions ===" - -if command -v nvcc >/dev/null 2>&1; then - nvcc --version | tail -n 1 -else - echo "nvcc: not found on PATH" -fi - -nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | sed 's/^/driver_version: /' - -# NCCL version discovery: prefer asking a Python-visible torch build (most -# accurate for the environment that will actually run collectives), and -# fall back to scanning the linker cache for libnccl if torch isn't -# importable. Both are best-effort. -if python3 -c "import torch; print(torch.cuda.nccl.version())" 2>/dev/null; then - : -elif ldconfig -p | grep -qi libnccl; then - echo "libnccl found via ldconfig:" - ldconfig -p | grep -i libnccl -else - echo "NCCL version: could not be determined (no torch, no libnccl in ldconfig cache)" -fi - -echo "setup_node.sh complete." diff --git a/notebooks/astrasim2_correlation/correlation/teardown.py b/notebooks/astrasim2_correlation/correlation/teardown.py deleted file mode 100644 index e533125a..00000000 --- a/notebooks/astrasim2_correlation/correlation/teardown.py +++ /dev/null @@ -1,850 +0,0 @@ -"""Tear down AWS resources created by ``provision.py``. - -This script is the "down" half of the correlation study's empirical leg -(see ``provision.py`` for the "up" half and ``README.md`` for the full -runbook). It is designed to be safe to run more than once, and to work -even when its own local state is missing or stale, because it discovers -targets two ways and reconciles them: - -1. Local state files under ``.state/*.json``, written by - ``provision.py``'s ``write_state`` -- the fast, detailed path, since a - state file already has the security group id and key pair name without - any extra API calls. -2. A live ``describe_instances`` search filtered on the ``Project`` tag - (and ``RunId`` tag, when ``--run-id`` is given) -- the authoritative - path, since it reflects what AWS actually has running right now even if - a state file was deleted, never written (a crash mid-provision), or the - run was started from a different machine/checkout. - -Every function below is written to be independently importable and -testable, matching ``provision.py``'s convention; see that module's -docstring for why boto3 is imported guarded rather than as a hard -dependency. -""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from pathlib import Path -from typing import Any, Dict, List, Optional - -from config import Config - -try: - import boto3 - from botocore.exceptions import ClientError -except ImportError: # pragma: no cover - exercised only when boto3 truly absent - boto3 = None - - # See provision.py's identical placeholder for why this exists: keeps - # `except ClientError:` clauses valid Python even without boto3 - # installed, without ever actually being reachable (main() always - # calls _require_boto3() first). - class ClientError(Exception): # type: ignore[no-redef] - pass - - -def _require_boto3() -> None: - """Raise a clear, actionable error if boto3 is not installed. - - Raises - ------ - SystemExit - Always, if ``boto3`` failed to import. See ``provision._require_boto3`` - for the identical rationale; kept as a separate copy here (rather - than importing it from ``provision``) so this module has no - import-time dependency on ``provision.py`` at all. - """ - if boto3 is None: - raise SystemExit( - "boto3 is required for AWS teardown but is not installed in this " - "Python environment.\n" - "Install it with: pip install boto3" - ) - - -# Instance states worth discovering/tearing down. Deliberately excludes -# "shutting-down" and "terminated": those instances are already on their -# way out or gone and do not need (and, for "terminated", cannot receive) -# a terminate_instances call. -_ACTIVE_STATES = ("pending", "running", "stopping", "stopped") - -# EC2 does not release the ENI-to-security-group association the instant -# terminate_instances returns (or the instance_terminated waiter is -# satisfied); delete_security_group can fail with DependencyViolation for a -# short window afterward while that teardown finishes propagating. Retrying -# with a fixed backoff is expected to succeed within a few attempts rather -# than being a genuine, permanent conflict. -_SG_DELETE_MAX_RETRIES = 5 -_SG_DELETE_RETRY_SLEEP_S = 5.0 - - -def find_tagged_instances( - ec2_client, tag_project: str, run_id: Optional[str] = None -) -> List[dict]: - """Find EC2 instances tagged for this study, optionally scoped to one run. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - tag_project : str - Value the ``Project`` tag must match. - run_id : str or None, default None - If given, additionally require the ``RunId`` tag to match this - value. If ``None``, instances from every run under ``tag_project`` - are returned. - - Returns - ------- - list[dict] - Raw ``Instance`` dicts (the ``Reservations[].Instances[]`` shape - returned by ``describe_instances``), for instances currently in - one of :data:`_ACTIVE_STATES`. Empty list if none match. - - Notes - ----- - Paginates via ``NextToken`` manually (rather than - ``ec2_client.get_paginator(...)``) so this function works identically - against a plain client and a ``botocore.stub.Stubber``-wrapped one used - in tests, without needing the Stubber to understand paginator internals. - """ - filters = [ - {"Name": "tag:Project", "Values": [tag_project]}, - {"Name": "instance-state-name", "Values": list(_ACTIVE_STATES)}, - ] - if run_id: - filters.append({"Name": "tag:RunId", "Values": [run_id]}) - - instances: List[dict] = [] - kwargs: Dict[str, Any] = {"Filters": filters} - while True: - response = ec2_client.describe_instances(**kwargs) - for reservation in response.get("Reservations", []): - instances.extend(reservation.get("Instances", [])) - next_token = response.get("NextToken") - if not next_token: - break - kwargs["NextToken"] = next_token - return instances - - -def _delete_security_group_with_retry( - ec2_client, - sg_id: str, - max_retries: int = _SG_DELETE_MAX_RETRIES, - retry_sleep_s: float = _SG_DELETE_RETRY_SLEEP_S, -) -> None: - """Delete a security group, retrying on ``DependencyViolation``. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - sg_id : str - Security group id to delete. - max_retries : int, default 5 - Maximum number of ``delete_security_group`` attempts. - retry_sleep_s : float, default 5.0 - Seconds to sleep between retries. - - Raises - ------ - RuntimeError - If every attempt fails with ``DependencyViolation`` (the ENI - association never cleared in time). - botocore.exceptions.ClientError - For any ``ClientError`` code other than ``DependencyViolation`` or - ``InvalidGroup.NotFound``, propagated immediately without retry. - - Notes - ----- - See the module-level comment on :data:`_SG_DELETE_MAX_RETRIES` for why - ``DependencyViolation`` specifically is retried rather than treated as - fatal on the first failure. - """ - last_exc: Optional[ClientError] = None - for attempt in range(1, max_retries + 1): - try: - ec2_client.delete_security_group(GroupId=sg_id) - print(f"Deleted security group {sg_id}.") - return - except ClientError as exc: - code = exc.response.get("Error", {}).get("Code", "") - if code == "InvalidGroup.NotFound": - print(f"Security group {sg_id} already gone.") - return - if code != "DependencyViolation": - raise - last_exc = exc - if attempt < max_retries: - print( - f"delete_security_group({sg_id}) hit DependencyViolation " - f"(attempt {attempt}/{max_retries}); retrying in " - f"{retry_sleep_s}s..." - ) - time.sleep(retry_sleep_s) - raise RuntimeError( - f"Failed to delete security group {sg_id} after {max_retries} attempts " - "due to a persistent DependencyViolation." - ) from last_exc - - -def teardown_run(ec2_client, state: dict, delete_key: bool) -> None: - """Terminate one run's instance and clean up its security group and key. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - state : dict - Per-run state, either loaded from a ``.state/*.json`` file (see - ``provision.write_state``) or synthesized from a live - ``describe_instances`` result (see ``_derive_state_from_instance``). - Recognized keys: ``instance_id``, ``sg_id``, ``key_name``, - ``key_path``. All are optional -- a missing key simply skips that - cleanup step, so a partially-populated state (e.g. derived from - AWS alone, with no known ``key_path``) still tears down whatever it - can. - delete_key : bool - If ``True``, also delete the AWS-side key pair (and the local PEM, - if ``state["key_path"]`` is known and exists on disk). - - Raises - ------ - RuntimeError - If security group deletion exhausts its retries (see - :func:`_delete_security_group_with_retry`). - botocore.exceptions.ClientError - For any AWS failure other than the specific "already gone" codes - this function is written to tolerate (``InvalidInstanceID.NotFound`` - for the instance, ``InvalidGroup.NotFound`` for the security - group), propagated unmodified. - botocore.exceptions.WaiterError - If the ``instance_terminated`` waiter times out or the instance - reaches an unexpected terminal state. - - Notes - ----- - Order matters: instance termination is started and waited on *before* - security group deletion is attempted, because the security group - cannot be deleted while an instance's network interface still - references it (see :data:`_SG_DELETE_MAX_RETRIES`'s docstring). - """ - instance_id = state.get("instance_id") - if instance_id: - try: - ec2_client.terminate_instances(InstanceIds=[instance_id]) - except ClientError as exc: - code = exc.response.get("Error", {}).get("Code", "") - if code != "InvalidInstanceID.NotFound": - raise - print( - f"Instance {instance_id} already gone " - "(InvalidInstanceID.NotFound); continuing teardown." - ) - else: - print( - f"Termination requested for {instance_id}; waiting for it to " - "fully terminate..." - ) - waiter = ec2_client.get_waiter("instance_terminated") - waiter.wait(InstanceIds=[instance_id]) - print(f"Instance {instance_id} terminated.") - - sg_id = state.get("sg_id") - if sg_id: - _delete_security_group_with_retry(ec2_client, sg_id) - - if delete_key: - key_name = state.get("key_name") - if key_name: - try: - ec2_client.delete_key_pair(KeyName=key_name) - print(f"Deleted AWS key pair {key_name}.") - except ClientError as exc: - # Best-effort: a stale/already-deleted key pair should not - # block the rest of teardown, but it is still reported, not - # silently dropped. - print(f"WARNING: failed to delete AWS key pair {key_name}: {exc}") - - key_path = state.get("key_path") - if key_path: - local_path = Path(key_path) - if local_path.exists(): - local_path.unlink() - print(f"Deleted local PEM {local_path}.") - elif key_name: - print( - f"No local PEM path known for key pair {key_name!r} (this run's " - "state was derived from AWS alone, not a local state file); " - "only the AWS-side key pair was deleted. If a PEM for it exists " - "on this or another machine, remove it manually." - ) - - -def _derive_state_from_instance(instance: dict) -> dict: - """Reconstruct a minimal teardown ``state`` dict from a live instance. - - Used when a tagged instance is discovered via :func:`find_tagged_instances` - but has no matching local ``.state/*.json`` file (deleted, never - written, or written on a different machine). EC2 instance descriptions - already carry everything ``teardown_run`` needs except the local PEM - path, which cannot be recovered this way. - - Parameters - ---------- - instance : dict - One ``Instance`` dict as returned by ``describe_instances``. - - Returns - ------- - dict - ``{"run_id", "instance_id", "sg_id", "key_name", "key_path"}``, - with ``key_path`` always ``None`` (see above) and ``run_id`` taken - from the instance's ``RunId`` tag, falling back to the instance id - itself if that tag is somehow missing. - """ - tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])} - security_groups = instance.get("SecurityGroups", []) - return { - "run_id": tags.get("RunId", instance["InstanceId"]), - "instance_id": instance["InstanceId"], - "sg_id": security_groups[0]["GroupId"] if security_groups else None, - "key_name": instance.get("KeyName"), - "key_path": None, - } - - -def _state_dir_files(state_dir: Path, run_id: Optional[str]) -> List[Path]: - """List local state files relevant to this teardown invocation. - - Parameters - ---------- - state_dir : pathlib.Path - Directory containing ``.json`` state files. - run_id : str or None - If given, look only for ``/.json``. If - ``None``, return every ``*.json`` file in ``state_dir``. - - Returns - ------- - list[pathlib.Path] - Matching, existing file paths, sorted for deterministic output. - Empty list if ``state_dir`` does not exist or nothing matches. - """ - if not state_dir.exists(): - return [] - if run_id: - candidate = state_dir / f"{run_id}.json" - return [candidate] if candidate.exists() else [] - return sorted(state_dir.glob("*.json")) - - -def _load_state_file(path: Path) -> dict: - """Load one state JSON file. - - Parameters - ---------- - path : pathlib.Path - Path to a ``.state/.json`` file. - - Returns - ------- - dict - The parsed JSON content. - - Raises - ------ - OSError - If ``path`` cannot be opened. - json.JSONDecodeError - If ``path`` does not contain valid JSON. - """ - with open(path, "r") as fh: - return json.load(fh) - - -def _prompt_yes_no(prompt: str) -> bool: - """Ask an interactive yes/no question, returning ``True`` only for "yes". - - Parameters - ---------- - prompt : str - Text to show before the input cursor. - - Returns - ------- - bool - ``True`` only if the user typed exactly ``"yes"`` - (case-insensitive); ``False`` otherwise, including on EOF. - """ - try: - answer = input(prompt) - except EOFError: - return False - return answer.strip().lower() == "yes" - - -def _verify(ec2_client, tag_project: str, run_id: Optional[str]) -> int: - """Audit for any still-running tagged instances, without tearing anything down. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof). - tag_project : str - Value the ``Project`` tag must match. - run_id : str or None - If given, scope the audit to just this run. - - Returns - ------- - int - ``0`` (and prints ``"no running instances"``) if nothing tagged - remains in :data:`_ACTIVE_STATES`; ``1`` (and prints a table) if - anything does. Intended for use as a CI/cron safety check after a - teardown, so a stuck resource is caught rather than silently - left running and accruing cost. - """ - instances = find_tagged_instances(ec2_client, tag_project, run_id) - if not instances: - print("no running instances") - return 0 - - print("Tagged instances still present:") - print(f"{'InstanceId':<21} {'State':<12} {'RunId'}") - for instance in instances: - tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])} - state_name = instance.get("State", {}).get("Name", "unknown") - print(f"{instance['InstanceId']:<21} {state_name:<12} {tags.get('RunId', '?')}") - return 1 - - -# --------------------------------------------------------------------------- -# Region resolution (Fix 4a) -# --------------------------------------------------------------------------- -# -# Design/WHY: before this fix, teardown.py always defaulted --region to -# "us-east-1" and used exactly that one region for every discovery/teardown -# call, regardless of what region a run's own state file recorded. A run -# provisioned in any other region (e.g. because capacity/quota forced a -# different --region at provision time) was therefore invisible to -# `teardown.py --all` and to a bare `teardown.py --run-id ` unless the -# operator remembered to pass --region explicitly every time -- silently -# leaving that run's instance running and billing. The fix: --region now -# defaults to None (so this module can tell "the user explicitly asked for -# us-east-1" apart from "the user said nothing"), and region resolution -# follows this priority, per run: -# 1. an explicit --region flag always wins (it is an explicit override of -# whatever a state file might say, e.g. for recovering a run whose -# state file was hand-edited or lost a region field); -# 2. otherwise, a matching local state file's own recorded "region" field -# is authoritative (this is what makes --all correctly span multiple -# regions in one invocation); -# 3. otherwise (no --region, no matching/region-bearing state file -- -# e.g. an instance discovered via live AWS tags alone, with no local -# state at all), fall back to Config's own default region -# ("us-east-1"), matching this module's pre-fix behavior for that one -# case. -# resolve_regions() is a pure function (no AWS calls, no I/O) precisely so -# this priority logic is unit-testable on its own, per the work-package -# spec's explicit ask. - - -def resolve_regions(args, states: Dict[str, dict]) -> Dict[str, List[str]]: - """Determine which AWS region(s) to operate in, and which run_ids live in each. - - See the "Region resolution" design comment immediately above this - function for the full priority rationale (explicit ``--region`` flag, - then a state file's own recorded region, then Config's default). - - Parameters - ---------- - args : argparse.Namespace or any object with ``.region`` and ``.run_id`` - Only ``args.region`` (str or None) and ``args.run_id`` (str or - None) are consulted; duck-typed so a test can pass a minimal stand-in - without building a full parsed CLI namespace. - states : dict[str, dict] - Every locally known state dict, keyed by run_id, as loaded from - ``.state/*.json`` files -- NOT pre-filtered to ``args.run_id``; this - function does that scoping itself. - - Returns - ------- - dict[str, list[str]] - Mapping of region -> list of run_ids (drawn from `states`) resolved - to that region. - - - If ``args.run_id`` is set: exactly one key (the resolved region - for that one run), mapping to ``[args.run_id]``. - - Otherwise (``--all`` or a bare ``--verify``): one key per distinct - region recorded across every entry in `states` (grouped), PLUS - the default/fallback region (``args.region`` if given, else - Config's default) as its own key even if no state file happens to - record it -- so a caller iterating this mapping's keys always - still searches that region too, for tag-only discovery of - instances with no matching local state file at all (e.g. a crash - before ``provision.write_state`` ever ran). - - Notes - ----- - Pure function: makes no AWS calls and performs no I/O, so it is - directly unit-testable without a Stubber or any boto3 client. - - Examples - -------- - >>> import argparse - >>> args = argparse.Namespace(region=None, run_id=None) - >>> resolve_regions(args, {"run-a": {"region": "us-west-2"}}) - {'us-east-1': [], 'us-west-2': ['run-a']} - """ - default_region = args.region or Config().region - - if args.run_id: - state = states.get(args.run_id) - region = args.region or (state.get("region") if state else None) or default_region - return {region: [args.run_id]} - - regions: Dict[str, List[str]] = {default_region: []} - for run_id, state in states.items(): - region = args.region or state.get("region") or default_region - regions.setdefault(region, []).append(run_id) - return regions - - -def _client_for_region(clients: Dict[str, Any], region: str): - """Return a cached boto3 ``ec2`` client for `region`, creating it on first use. - - Parameters - ---------- - clients : dict[str, botocore.client.BaseClient] - Mutable cache, keyed by region name; mutated in place on a cache - miss. Callers own the dict's lifetime (typically one per - :func:`main` invocation). - region : str - AWS region name to build (or reuse) a client for. - - Returns - ------- - botocore.client.BaseClient - A boto3 ``ec2`` client bound to `region`, reused across calls that - pass the same `region` and the same `clients` dict. - - Notes - ----- - Design: ``--all`` may need to talk to several regions in one - invocation (one per distinct region recorded across this study's state - files -- see :func:`resolve_regions`). Building a client lazily, keyed - by region, keeps the number of client objects (and any connection-pool - overhead) proportional to the number of *distinct regions* actually - involved in one run, rather than the number of runs or the number of - times a region happens to be revisited. - """ - if region not in clients: - clients[region] = boto3.client("ec2", region_name=region) - return clients[region] - - -def _teardown_one_region( - ec2_client, - tag_project: str, - region: str, - run_id_filter: Optional[str], - file_states_by_run: Dict[str, dict], - file_paths_by_run: Dict[str, Path], - delete_key: bool, - yes: bool, -) -> int: - """Discover, stale-clean, confirm, and tear down every matching resource in one region. - - Factored out of :func:`main` so :func:`main` itself only needs to - resolve regions (:func:`resolve_regions`) and loop over them; this is - also what makes the stale-state cleanup path (Fix 4b) directly - unit-testable with a single ``botocore.stub.Stubber``-wrapped client, - without needing to drive the CLI/argparse layer or monkeypatch - ``boto3.client`` at all. - - Parameters - ---------- - ec2_client : botocore.client.BaseClient - A boto3 ``ec2`` client (or a stub thereof) already bound to - `region`. - tag_project : str - Value the ``Project`` tag must match (``Config().tag_project``). - region : str - The AWS region `ec2_client` is bound to; used only in printed - messages (this function performs no region validation of its own). - run_id_filter : str or None - If given, :func:`find_tagged_instances` is scoped to just this one - run (mirrors the original single-``--run-id`` behavior); ``None`` - discovers every tagged run in `region`. - file_states_by_run : dict[str, dict] - Locally known state dicts for the run_ids :func:`resolve_regions` - assigned to this region, keyed by run_id. - file_paths_by_run : dict[str, pathlib.Path] - The corresponding ``.state/.json`` paths, same keys as - `file_states_by_run`. - delete_key : bool - Forwarded to every :func:`teardown_run` call in this region. - yes : bool - If ``False``, prompts for interactive confirmation before tearing - down any LIVE target found in this region. Stale-state cleanup - (see Notes) is never gated on this prompt: it only reclaims - resources this function has already determined are orphaned (no - matching live instance), not anything newly discovered as - still-in-use. - - Returns - ------- - int - ``0`` if every teardown/cleanup in this region succeeded (including - "nothing to do here"); ``1`` if any individual teardown/cleanup - raised, or if the user declined confirmation for this region's live - targets. - - Notes - ----- - Order of operations, and why (Fix 4b): (1) live AWS discovery via - :func:`find_tagged_instances`, reconciled against `file_states_by_run` - exactly as the pre-fix module-level code did; (2) any state file in - `file_states_by_run` with NO matching live instance is "stale" -- - rather than just unlinking its state file (the pre-fix behavior), it is - now routed through :func:`teardown_run` FIRST. ``teardown_run``'s - terminate step already tolerates ``InvalidInstanceID.NotFound``, so a - truly-dead instance is a safe no-op there -- but its security group and - (optionally) key pair do NOT disappear just because the instance is - gone, e.g. via the on-instance dead-man timer terminating it out from - under a local state file that was never cleaned up. Routing through - ``teardown_run`` first closes that SG/key-pair leak; the state file is - removed only if that call did NOT raise, so a failed stale cleanup - leaves the state file in place for a future retry instead of silently - losing track of the leak. (3) only THEN are any remaining LIVE targets - confirmed and torn down, matching the pre-fix confirmation UX. - """ - file_states_by_instance: Dict[str, dict] = { - s["instance_id"]: s for s in file_states_by_run.values() if s.get("instance_id") - } - aws_instances = find_tagged_instances(ec2_client, tag_project, run_id_filter) - - targets: List[dict] = [] - for instance in aws_instances: - instance_id = instance["InstanceId"] - if instance_id in file_states_by_instance: - targets.append(file_states_by_instance[instance_id]) - else: - targets.append(_derive_state_from_instance(instance)) - - exit_code = 0 - - # --- Stale-state cleanup: see the "Order of operations" note above. --- - live_run_ids = {t.get("run_id") for t in targets if t.get("run_id")} - for run_id_key in set(file_states_by_run) - live_run_ids: - print( - f"State file for run {run_id_key!r} (region {region}) has no matching live AWS " - "instance; routing it through teardown to reclaim any leftover security group/key " - "pair before removing the state file." - ) - try: - teardown_run(ec2_client, file_states_by_run[run_id_key], delete_key=delete_key) - except Exception as exc: # noqa: BLE001 - # Design: deliberately broad, matching the identical rationale - # on the live-target teardown loop below -- one stale run's - # cleanup failure must not abort the rest of a --all batch. - print(f"ERROR cleaning up stale run {run_id_key!r}: {exc}", file=sys.stderr) - exit_code = 1 - continue - state_path = file_paths_by_run.get(run_id_key) - if state_path and state_path.exists(): - state_path.unlink() - print(f"Removed state file {state_path}.") - - if not targets: - if not file_states_by_run: - print(f"No matching resources found in {region}.") - return exit_code - - print(f"The following will be torn down in {region}:") - for t in targets: - print( - f" run_id={t.get('run_id', '?')} instance_id={t.get('instance_id', '?')} " - f"sg_id={t.get('sg_id', '?')}" - ) - - if not yes and not _prompt_yes_no("Proceed with teardown? [yes/N]: "): - print("Aborted by user.") - return 1 - - for t in targets: - try: - teardown_run(ec2_client, t, delete_key=delete_key) - except Exception as exc: # noqa: BLE001 - # Design: deliberately broad. One run's teardown failure should - # not abort the rest of a --all batch; report it and keep - # going, then reflect the failure in the process exit code - # rather than swallowing it silently. - print(f"ERROR tearing down run {t.get('run_id', '?')}: {exc}", file=sys.stderr) - exit_code = 1 - continue - - run_id_key = t.get("run_id") - state_path = file_paths_by_run.get(run_id_key) - if state_path and state_path.exists(): - state_path.unlink() - print(f"Removed state file {state_path}.") - - return exit_code - - -def main(argv: Optional[list] = None) -> int: - """Parse CLI args and tear down matching resources. - - Parameters - ---------- - argv : list[str] or None, default None - Argument list, as passed to ``argparse``'s ``parse_args``. ``None`` - reads from ``sys.argv[1:]``. - - Returns - ------- - int - Process exit code. For ``--verify``: ``0`` if every audited region - is clean, ``1`` if tagged instances remain in any of them. - Otherwise: ``0`` on a fully successful teardown across every region - involved (or nothing to do anywhere), ``1`` if the user declined - confirmation in any region or if any individual run's - teardown/stale-cleanup raised. - - Notes - ----- - ``--tag-project`` is deliberately not a flag here (this CLI's flag set - is fixed by the work-package spec to - ``[--region] [--run-id | --all] [--verify] [--delete-key] [--yes]``): - the ``Project`` tag value used for discovery is always - ``Config().tag_project`` (i.e. ``Config``'s dataclass default, - ``"accelforge-correlation"``). A run provisioned with a *custom* - ``--tag-project`` cannot be found by this CLI and must instead be torn - down by calling :func:`find_tagged_instances`/:func:`teardown_run` - directly with that project value -- both are plain importable - functions for exactly this reason. - - ``--region`` now defaults to ``None`` (Fix 4a), NOT ``"us-east-1"``: see - the "Region resolution" design comment above :func:`resolve_regions` - for the full rationale and priority order. This function may therefore - talk to *more than one* region in a single invocation (e.g. ``--all`` - spanning every region this study's local state knows about, or - ``--verify`` auditing all of them) -- :func:`_client_for_region` keeps - one cached boto3 client per distinct region actually needed. - """ - parser = argparse.ArgumentParser( - prog="teardown.py", - description="Tear down accelforge correlation-study AWS resources.", - ) - parser.add_argument( - "--region", - type=str, - default=None, - help=( - "AWS region. If omitted, each targeted run's own state-file-recorded " - "region is used when known, else Config's default region (us-east-1); " - "an explicit --region always overrides both. See resolve_regions()." - ), - ) - target = parser.add_mutually_exclusive_group() - target.add_argument("--run-id", type=str, default=None, help="Tear down only this run.") - target.add_argument( - "--all", - action="store_true", - help=( - "Tear down every accelforge-correlation-tagged run this study's local " - "state knows about, across every region those runs were provisioned in." - ), - ) - parser.add_argument( - "--verify", - action="store_true", - help="Audit only: report any still-running tagged instances and exit non-zero if any remain.", - ) - parser.add_argument( - "--delete-key", - action="store_true", - help="Also delete the AWS key pair and local PEM.", - ) - parser.add_argument("--yes", action="store_true", help="Skip interactive confirmation.") - args = parser.parse_args(argv) - - if not args.verify and not args.run_id and not args.all: - parser.error("one of --run-id, --all, or --verify is required") - - _require_boto3() - - # Design: instantiate a plain Config() rather than duplicating its - # tag_project/state_dir default logic here. See the "Notes" above on - # why --tag-project is not a teardown.py flag. - defaults = Config() - tag_project = defaults.tag_project - state_dir = defaults.state_dir - - # Load EVERY locally known state file up front, regardless of scope -- - # resolve_regions() needs the full set to group by region (Fix 4a); - # scope filtering (--run-id vs --all/--verify) happens inside - # resolve_regions() and the per-region loop below, not here. - all_states_by_run: Dict[str, dict] = {} - all_paths_by_run: Dict[str, Path] = {} - for path in _state_dir_files(state_dir, None): - try: - loaded = _load_state_file(path) - except (OSError, json.JSONDecodeError) as exc: - print(f"WARNING: could not read state file {path}: {exc}; skipping.") - continue - run_id_key = loaded.get("run_id", path.stem) - all_states_by_run[run_id_key] = loaded - all_paths_by_run[run_id_key] = path - - region_map = resolve_regions(args, all_states_by_run) - clients: Dict[str, Any] = {} - - if args.verify: - # Fix 4c: audit every region resolve_regions() knows about (not - # only the --region flag/fallback), so a run provisioned in a - # different region than the operator happens to be thinking about - # is not silently skipped by an audit meant to catch exactly that. - exit_code = 0 - for region in sorted(region_map): - print(f"--- Verifying region {region} ---") - client = _client_for_region(clients, region) - if _verify(client, tag_project, args.run_id) != 0: - exit_code = 1 - return exit_code - - exit_code = 0 - for region in sorted(region_map): - ec2_client = _client_for_region(clients, region) - run_ids_here = region_map[region] - file_states_by_run = { - rid: all_states_by_run[rid] for rid in run_ids_here if rid in all_states_by_run - } - file_paths_by_run = { - rid: all_paths_by_run[rid] for rid in run_ids_here if rid in all_paths_by_run - } - region_exit_code = _teardown_one_region( - ec2_client, - tag_project, - region, - args.run_id, - file_states_by_run, - file_paths_by_run, - args.delete_key, - args.yes, - ) - if region_exit_code != 0: - exit_code = 1 - - return exit_code - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/notebooks/astrasim2_correlation/correlation/tests/__init__.py b/notebooks/astrasim2_correlation/correlation/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py b/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py deleted file mode 100644 index 79c73050..00000000 --- a/notebooks/astrasim2_correlation/correlation/tests/test_orchestrate.py +++ /dev/null @@ -1,580 +0,0 @@ -"""Tests for orchestrate.py. - -Like ``test_provision_teardown.py``, these tests never touch real AWS, -never touch the network, and never spawn a real ``ssh``/``scp`` process. -Where ``test_provision_teardown.py`` achieves that with -``botocore.stub.Stubber`` (because it exercises ``provision.py``'s and -``teardown.py``'s own boto3-request-building code directly), this module -takes a different, coarser-grained approach: ``orchestrate.py`` mostly -*sequences* those already-tested functions rather than building its own -boto3 requests, so the provisioning functions themselves -(``resolve_ami``, ``ensure_key_pair``, ``ensure_security_group``, -``launch_instance``, ``wait_for_instance``), ``teardown.teardown_run``, -``orchestrate.caller_ip``, ``orchestrate.boto3``, and -``orchestrate.subprocess.run`` are all monkeypatched with lightweight -fakes that record what they were called with. This exercises -``orchestrate.py``'s own sequencing/argv-building logic -- the part this -work package is actually responsible for -- without needing Stubber -responses for calls this module never makes itself. - -Import strategy ------------------ -Mirrors ``test_provision_teardown.py`` exactly: ``correlation/`` has no -``__init__.py`` (deliberately not a package), so it is inserted onto -``sys.path`` explicitly before importing ``config``/``orchestrate``, -rather than relying on pytest's own rootdir-insertion behavior. -""" - -from __future__ import annotations - -import json -import subprocess -import sys -from pathlib import Path -from typing import Any, Dict, List - -import pytest - -# boto3 is not (and must not become) an accelforge package dependency; skip -# this whole module rather than error if it is not installed in the -# environment running the tests. orchestrate.py's own import of boto3 is -# guarded the same way provision.py's/teardown.py's is, but the tests below -# exercise real code paths that assume boto3 (and its stub-friendly -# botocore internals) are present, matching test_provision_teardown.py's -# identical importorskip. -boto3 = pytest.importorskip("boto3") - -_CORRELATION_DIR = Path(__file__).resolve().parent.parent -if str(_CORRELATION_DIR) not in sys.path: - sys.path.insert(0, str(_CORRELATION_DIR)) - -import config # noqa: E402 -import orchestrate # noqa: E402 - -Config = config.Config - - -# --------------------------------------------------------------------------- -# legs_for -# --------------------------------------------------------------------------- - - -def test_legs_for_both_returns_fc_then_torus(): - """"both" expands to the FC leg followed by the torus leg, in that order.""" - assert orchestrate.legs_for("both") == ["fc", "torus"] - - -def test_legs_for_fc_returns_single_leg(): - assert orchestrate.legs_for("fc") == ["fc"] - - -def test_legs_for_torus_returns_single_leg(): - assert orchestrate.legs_for("torus") == ["torus"] - - -# --------------------------------------------------------------------------- -# build_ssh_cmd / build_scp_cmd -# --------------------------------------------------------------------------- - - -def test_build_ssh_cmd_exact_argv(tmp_path): - """build_ssh_cmd's argv matches the exact option set/order the spec requires.""" - key_path = tmp_path / "keys" / "run.pem" - known_hosts_path = tmp_path / ".state" / "known_hosts" - - cmd = orchestrate.build_ssh_cmd(key_path, "ubuntu", "203.0.113.9", "echo hi", known_hosts_path) - - assert cmd == [ - "ssh", - "-i", - str(key_path), - "-o", - "StrictHostKeyChecking=accept-new", - "-o", - f"UserKnownHostsFile={known_hosts_path}", - "-o", - "ConnectTimeout=30", - "ubuntu@203.0.113.9", - "echo hi", - ] - - -def test_build_scp_cmd_non_recursive_push_argv(tmp_path): - """A multi-source, non-recursive push builds argv with sources then dest, no -r.""" - key_path = tmp_path / "keys" / "run.pem" - known_hosts_path = tmp_path / ".state" / "known_hosts" - sources = ["setup_node.sh", "run_profile.sh", "parse_nccl.py"] - dest = "ubuntu@203.0.113.9:~/" - - cmd = orchestrate.build_scp_cmd(key_path, sources, dest, known_hosts_path) - - assert cmd == [ - "scp", - "-i", - str(key_path), - "-o", - "StrictHostKeyChecking=accept-new", - "-o", - f"UserKnownHostsFile={known_hosts_path}", - "-o", - "ConnectTimeout=30", - "setup_node.sh", - "run_profile.sh", - "parse_nccl.py", - "ubuntu@203.0.113.9:~/", - ] - assert "-r" not in cmd - - -def test_build_scp_cmd_recursive_flag_is_first_positional_after_scp(tmp_path): - """recursive=True inserts -r immediately after the program name, before -i.""" - key_path = tmp_path / "keys" / "run.pem" - known_hosts_path = tmp_path / ".state" / "known_hosts" - - cmd = orchestrate.build_scp_cmd( - key_path, - ["ubuntu@203.0.113.9:~/results_fc"], - "/local/data/run/fc", - known_hosts_path, - recursive=True, - ) - - assert cmd[0] == "scp" - assert cmd[1] == "-r" - assert cmd[2] == "-i" - assert cmd[-2] == "ubuntu@203.0.113.9:~/results_fc" - assert cmd[-1] == "/local/data/run/fc" - - -def test_build_scp_cmd_rejects_empty_sources(tmp_path): - with pytest.raises(ValueError): - orchestrate.build_scp_cmd(tmp_path / "k.pem", [], "dest", tmp_path / "known_hosts") - - -# --------------------------------------------------------------------------- -# End-to-end orchestration (main()), everything monkeypatched -# --------------------------------------------------------------------------- - - -class _FakeBoto3: - """Stand-in for the `boto3` module, only supplying `.client(...)`. - - Design: monkeypatched onto `orchestrate.boto3` specifically (not the - real, globally-shared `boto3` module) so this fake never leaks into - any other module's view of boto3. Returns a plain sentinel string - rather than a real client, since every function orchestrate.py passes - that "client" to (resolve_ami, ensure_key_pair, ...) is itself - monkeypatched below and never actually calls a botocore method on it. - """ - - @staticmethod - def client(service_name: str, region_name: str = None): - return f"fake-{service_name}-client[{region_name}]" - - -@pytest.fixture -def orchestrate_fakes(tmp_path, monkeypatch): - """Monkeypatch every AWS/network/subprocess seam orchestrate.py has. - - Returns - ------- - dict - ``{"calls": list of (name, args) tuples recording every fake - invocation in order, "run_cmds": list of argv lists recorded by - the fake subprocess.run, "teardown_calls": list of state dicts - teardown_run was called with}``. - - Notes - ----- - Also points ``cfg``'s ``--key-dir``/``--state-dir`` and - ``orchestrate._DATA_DIR`` at ``tmp_path`` subdirectories (see - ``orchestrate._DATA_DIR``'s module docstring comment for why that - constant exists) so a full ``orchestrate.main(...)`` run in these - tests writes real state/CSV files only under pytest's ephemeral - ``tmp_path``, never into the real repository tree. - """ - calls: List[tuple] = [] - run_cmds: List[List[str]] = [] - teardown_calls: List[Dict[str, Any]] = [] - - monkeypatch.setattr(orchestrate, "boto3", _FakeBoto3) - monkeypatch.setattr(orchestrate, "caller_ip", lambda: (_ for _ in ()).throw( - AssertionError("caller_ip() should never be called when --ssh-cidr is passed") - )) - - def fake_resolve_ami(ssm_client, parameter): - calls.append(("resolve_ami", parameter)) - return "ami-fake0123456789" - - def fake_ensure_key_pair(ec2_client, key_name, key_dir): - calls.append(("ensure_key_pair", key_name)) - key_dir.mkdir(parents=True, exist_ok=True) - return key_dir / f"{key_name}.pem" - - def fake_ensure_security_group(ec2_client, group_name, ssh_cidr, tag_project, run_id): - calls.append(("ensure_security_group", group_name, ssh_cidr)) - return "sg-fake0123456789" - - def fake_launch_instance(ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False): - calls.append(("launch_instance", dry_run)) - if dry_run: - return {"instance_id": None, "purchasing_used": "ondemand"} - return {"instance_id": "i-fake0123456789", "purchasing_used": "spot"} - - def fake_wait_for_instance(ec2_client, instance_id): - # Ordering assertion baked into the fake itself (rather than only - # checked after main() returns): the state file must already - # exist, with this instance_id recorded, by the time - # wait_for_instance is called -- this is the exact "state written - # before the SSH wait" ordering the work-package spec requires. - # See orchestrate.main's docstring "Design/WHY" note. - state_path = Path(_last_state_dir[0]) / f"{_last_run_id[0]}.json" - assert state_path.exists(), "state file must be written before wait_for_instance is called" - written = json.loads(state_path.read_text()) - assert written["instance_id"] == instance_id - assert written["public_ip"] is None - calls.append(("wait_for_instance", instance_id)) - return "203.0.113.9" - - def fake_teardown_run(ec2_client, state, delete_key): - teardown_calls.append(dict(state)) - calls.append(("teardown_run", state.get("instance_id"), delete_key)) - - def fake_run(cmd, check=True, **kwargs): - run_cmds.append(list(cmd)) - return subprocess.CompletedProcess(cmd, 0) - - # _last_state_dir / _last_run_id let fake_wait_for_instance locate the - # state file without needing main()'s local `cfg` in scope; populated - # by the test itself right before calling orchestrate.main(...). - _last_state_dir: List[Path] = [None] - _last_run_id: List[str] = [None] - - monkeypatch.setattr(orchestrate, "resolve_ami", fake_resolve_ami) - monkeypatch.setattr(orchestrate, "ensure_key_pair", fake_ensure_key_pair) - monkeypatch.setattr(orchestrate, "ensure_security_group", fake_ensure_security_group) - monkeypatch.setattr(orchestrate, "launch_instance", fake_launch_instance) - monkeypatch.setattr(orchestrate, "wait_for_instance", fake_wait_for_instance) - monkeypatch.setattr(orchestrate, "teardown_run", fake_teardown_run) - monkeypatch.setattr(orchestrate.subprocess, "run", fake_run) - - data_dir = tmp_path / "data" - monkeypatch.setattr(orchestrate, "_DATA_DIR", data_dir) - - return { - "calls": calls, - "run_cmds": run_cmds, - "teardown_calls": teardown_calls, - "state_dir_holder": _last_state_dir, - "run_id_holder": _last_run_id, - "data_dir": data_dir, - } - - -def _base_argv(tmp_path, run_id: str, extra: List[str] = None) -> List[str]: - """Shared CLI args for the end-to-end tests below. - - Parameters - ---------- - tmp_path : pathlib.Path - pytest's per-test temp directory; key-dir/state-dir are pointed - here so no test ever touches the real correlation/keys or - correlation/.state directories. - run_id : str - Deterministic run id so tests can locate the state file/data - directory by name instead of discovering a generated one. - extra : list[str] or None - Additional argv to append (e.g. ``["--keep-alive"]``). - - Returns - ------- - list[str] - argv suitable for ``orchestrate.main(...)``. Always includes - ``--ssh-cidr`` explicitly so ``caller_ip()`` (a real network call) - is never reached, and ``--yes`` so no interactive prompt blocks - the test. - """ - argv = [ - "--yes", - "--topology", - "both", - "--run-id", - run_id, - "--key-dir", - str(tmp_path / "keys"), - "--state-dir", - str(tmp_path / "state"), - "--ssh-cidr", - "203.0.113.5/32", - "--collectives", - "all_reduce,alltoall", - "--min-mib", - "1", - "--max-mib", - "2", - "--torus-dims", - "2x2x2", - ] - if extra: - argv += extra - return argv - - -def test_main_end_to_end_happy_path(tmp_path, orchestrate_fakes, monkeypatch): - """Full main() run: state ordering, per-leg profiling, fetch, and teardown. - - Asserts, per the work-package spec's test #3: - - the state file is written after launch (before wait_for_instance is - called -- enforced inside the fake_wait_for_instance itself) and - contains instance_id, - - the setup ssh command is executed with DEADMAN_MINUTES set, - - exactly one run_profile.sh invocation per leg, with the correct - dims ("8" then "2x2x2") and byte bounds (1 MiB / 2 MiB here), - - one scp fetch per leg, - - teardown_run is called exactly once, at the end, - - the state file is removed afterward. - """ - run_id = "test-run-e2e" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - - exit_code = orchestrate.main(_base_argv(tmp_path, run_id)) - - assert exit_code == 0 - - # --- state file lifecycle ------------------------------------------------- - state_path = state_dir / f"{run_id}.json" - assert not state_path.exists(), "state file must be removed after a successful teardown" - - # --- setup command ---------------------------------------------------------- - run_cmds = orchestrate_fakes["run_cmds"] - setup_cmds = [c for c in run_cmds if "setup_node.sh" in c[-1]] - assert len(setup_cmds) == 1 - assert "DEADMAN_MINUTES=120" in setup_cmds[0][-1] - assert setup_cmds[0][0] == "ssh" - - # --- one run_profile.sh invocation per leg, correct dims/bytes ----------- - profile_cmds = [c for c in run_cmds if "run_profile.sh" in c[-1]] - assert len(profile_cmds) == 2 - fc_cmd, torus_cmd = profile_cmds[0][-1], profile_cmds[1][-1] - assert "results_fc fc 1048576 2097152 8 all_reduce alltoall" in fc_cmd - assert "results_torus torus 1048576 2097152 2x2x2 all_reduce alltoall" in torus_cmd - - # --- fetch per leg ------------------------------------------------------ - # Recursive scp fetch argv shape (per build_scp_cmd): [..., source, dest], - # so the source (a "user@ip:~/results_" string) is always the - # second-to-last element. - scp_fetch_cmds = [c for c in run_cmds if c[0] == "scp" and "-r" in c and "results_" in c[-2]] - fetch_sources = {c[-2] for c in scp_fetch_cmds} - assert any("results_fc" in s for s in fetch_sources) - assert any("results_torus" in s for s in fetch_sources) - - # --- push commands happened before setup/profiling ----------------------- - # A push argv is a flat list of local file-path elements followed by a - # remote dest string, so membership needs a substring scan across - # elements rather than an exact-element containment check (the pushed - # sources are full absolute paths, not the bare "setup_node.sh"). - def _any_elem_contains(cmd: List[str], needle: str) -> bool: - return any(needle in elem for elem in cmd) - - push_cmds = [c for c in run_cmds if c[0] == "scp" and _any_elem_contains(c, "setup_node.sh")] - assert len(push_cmds) == 1 - torus_push_cmds = [c for c in run_cmds if c[0] == "scp" and "-r" in c and "torus_bench" in c[-1]] - assert len(torus_push_cmds) == 1 - - # --- ordering: push -> setup -> (profile -> fetch) x legs ---------------- - def _first_index(predicate): - return next(i for i, c in enumerate(run_cmds) if predicate(c)) - - push_idx = _first_index(lambda c: c[0] == "scp" and _any_elem_contains(c, "setup_node.sh")) - setup_idx = _first_index(lambda c: c[0] == "ssh" and "setup_node.sh" in c[-1]) - fc_profile_idx = _first_index(lambda c: c[0] == "ssh" and "run_profile.sh" in c[-1] and " fc " in c[-1]) - assert push_idx < setup_idx < fc_profile_idx - - # --- teardown called exactly once, at the very end ----------------------- - assert len(orchestrate_fakes["teardown_calls"]) == 1 - assert orchestrate_fakes["teardown_calls"][0]["instance_id"] == "i-fake0123456789" - call_names = [c[0] for c in orchestrate_fakes["calls"]] - assert call_names[-1] == "teardown_run" - - -# --------------------------------------------------------------------------- -# Fix 1 (BLOCKER): _run_leg must create the fetch's PARENT directory (not the -# leaf) before scp runs, and must refuse to fetch into an already-existing -# leaf directory. -# --------------------------------------------------------------------------- - - -def test_main_creates_run_data_dir_before_fetch_scp(tmp_path, orchestrate_fakes, monkeypatch): - """run_data_dir (the fetch's parent) exists by the time each leg's fetch scp runs. - - Regression test for Fix 1: without ``run_data_dir.mkdir(...)`` before the - fetch, `scp -r user@host:~/results_fc /fc` would fail - outright the first time a run_id is used, since its parent directory - would not exist yet. This test captures os-path existence from *inside* - the fake ``subprocess.run`` at the exact moment a fetch command is - recorded, per the work-package spec's test #1. - """ - run_id = "test-run-mkdir" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - data_dir = orchestrate_fakes["data_dir"] - - run_data_dir_existed_at_fetch: List[bool] = [] - - def fake_run(cmd, check=True, **kwargs): - orchestrate_fakes["run_cmds"].append(list(cmd)) - # A fetch is a recursive scp whose source (second-to-last argv - # element, per build_scp_cmd's argv shape) names a remote - # results_ path -- distinguishes it from the (also recursive) - # torus_bench/ push, whose source is a local path instead. - if cmd[0] == "scp" and "-r" in cmd and "results_" in cmd[-2]: - run_data_dir_existed_at_fetch.append((data_dir / run_id).exists()) - return subprocess.CompletedProcess(cmd, 0) - - monkeypatch.setattr(orchestrate.subprocess, "run", fake_run) - - exit_code = orchestrate.main(_base_argv(tmp_path, run_id)) - - assert exit_code == 0 - # One fetch per leg (fc, torus); run_data_dir must already exist at both. - assert len(run_data_dir_existed_at_fetch) == 2 - assert all(run_data_dir_existed_at_fetch) - - -def test_run_leg_raises_if_leaf_dir_already_exists(tmp_path, monkeypatch): - """_run_leg refuses to re-fetch into an already-existing run_data_dir/leg. - - Direct unit test of :func:`orchestrate._run_leg` (rather than a full - ``main()`` run) per the work-package spec's test #2, exercising the - RuntimeError in isolation. ``_run_streaming`` is monkeypatched to a - recording no-op so no real ssh/scp subprocess is ever attempted; the - remote profiling ssh command runs (it happens before the leaf-dir check - in ``_run_leg``'s body), but the fetch scp must never be reached. - """ - run_cmds: List[List[str]] = [] - monkeypatch.setattr(orchestrate, "_run_streaming", lambda cmd: run_cmds.append(cmd)) - - cfg = Config(run_id="test-run-leaf-exists", torus_dims=(2, 2, 2), collectives="all_reduce") - run_data_dir = tmp_path / "data" / cfg.run_id - local_leg_dir = run_data_dir / "fc" - local_leg_dir.mkdir(parents=True) - - with pytest.raises(RuntimeError, match="already exists"): - orchestrate._run_leg( - cfg, - "fc", - tmp_path / "key.pem", - "203.0.113.9", - tmp_path / "known_hosts", - run_data_dir, - ) - - # Only the remote profiling ssh command (which precedes the leaf-dir - # check in _run_leg's body) ran; the fetch scp was never attempted. - assert len(run_cmds) == 1 - assert run_cmds[0][0] == "ssh" - - -def test_main_dry_run_stops_before_launch_and_ssh(tmp_path, orchestrate_fakes): - """--dry-run creates key+SG+a dry launch, but never waits/pushes/profiles/tears down.""" - run_id = "test-run-dry" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - - exit_code = orchestrate.main(_base_argv(tmp_path, run_id, extra=["--dry-run"])) - - assert exit_code == 0 - call_names = [c[0] for c in orchestrate_fakes["calls"]] - assert "ensure_key_pair" in call_names - assert "ensure_security_group" in call_names - assert "launch_instance" in call_names - assert "wait_for_instance" not in call_names - assert "teardown_run" not in call_names - assert orchestrate_fakes["run_cmds"] == [] - assert not (state_dir / f"{run_id}.json").exists() - - -def test_main_profiling_failure_still_tears_down_and_propagates(tmp_path, orchestrate_fakes, monkeypatch): - """A failed profiling subprocess still triggers teardown, and the error propagates.""" - run_id = "test-run-fail" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - - def failing_run(cmd, check=True, **kwargs): - orchestrate_fakes["run_cmds"].append(list(cmd)) - if "run_profile.sh" in cmd[-1]: - raise subprocess.CalledProcessError(returncode=1, cmd=cmd) - return subprocess.CompletedProcess(cmd, 0) - - monkeypatch.setattr(orchestrate.subprocess, "run", failing_run) - - with pytest.raises(subprocess.CalledProcessError): - orchestrate.main(_base_argv(tmp_path, run_id)) - - # Teardown must still have run despite the propagating exception. - assert len(orchestrate_fakes["teardown_calls"]) == 1 - assert orchestrate_fakes["teardown_calls"][0]["instance_id"] == "i-fake0123456789" - # And the state file was still cleaned up by that successful teardown. - assert not (state_dir / f"{run_id}.json").exists() - - -def test_main_teardown_failure_does_not_mask_original_exception(tmp_path, orchestrate_fakes, monkeypatch): - """If teardown_run ALSO raises, the original profiling exception still propagates.""" - run_id = "test-run-double-fail" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - - def failing_run(cmd, check=True, **kwargs): - orchestrate_fakes["run_cmds"].append(list(cmd)) - if "run_profile.sh" in cmd[-1]: - raise subprocess.CalledProcessError(returncode=1, cmd=cmd) - return subprocess.CompletedProcess(cmd, 0) - - def failing_teardown(ec2_client, state, delete_key): - orchestrate_fakes["teardown_calls"].append(dict(state)) - raise RuntimeError("simulated teardown failure (e.g. DependencyViolation exhausted)") - - monkeypatch.setattr(orchestrate.subprocess, "run", failing_run) - monkeypatch.setattr(orchestrate, "teardown_run", failing_teardown) - - # The ORIGINAL exception (CalledProcessError from profiling) must be - # what propagates, not the teardown's RuntimeError -- this is the - # "don't mask the original exception" behavior _teardown_and_cleanup - # documents. - with pytest.raises(subprocess.CalledProcessError): - orchestrate.main(_base_argv(tmp_path, run_id)) - - assert len(orchestrate_fakes["teardown_calls"]) == 1 - # Teardown failed, so the state file must NOT have been removed -- - # teardown.py --run-id needs it to find leftover resources later. - assert (state_dir / f"{run_id}.json").exists() - - -def test_main_keep_alive_skips_teardown_and_keeps_state_file(tmp_path, orchestrate_fakes, capsys): - """--keep-alive leaves the instance up: no teardown_run call, state file remains.""" - run_id = "test-run-keep-alive" - state_dir = tmp_path / "state" - orchestrate_fakes["state_dir_holder"][0] = state_dir - orchestrate_fakes["run_id_holder"][0] = run_id - - exit_code = orchestrate.main(_base_argv(tmp_path, run_id, extra=["--keep-alive"])) - - assert exit_code == 0 - assert orchestrate_fakes["teardown_calls"] == [] - call_names = [c[0] for c in orchestrate_fakes["calls"]] - assert "teardown_run" not in call_names - - state_path = state_dir / f"{run_id}.json" - assert state_path.exists() - written = json.loads(state_path.read_text()) - assert written["public_ip"] == "203.0.113.9" - - out = capsys.readouterr().out - assert "STILL BEING BILLED" in out - assert "ssh -i" in out diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py b/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py deleted file mode 100644 index 322e682b..00000000 --- a/notebooks/astrasim2_correlation/correlation/tests/test_parse_nccl.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Tests for parse_nccl.py: the nccl-tests / torus_bench log parser. - -Fixtures under ``tests/fixtures/`` hold representative raw stdout captured -from the two profiling tools (see the ``correlation`` package's WP2 spec -for the exact log formats). These tests exercise the parsing functions -directly (:func:`parse_nccl.parse_nccl_tests`, -:func:`parse_nccl.parse_torus_bench`), the CSV writer -(:func:`parse_nccl.rows_to_csv`), and the CLI entry point end-to-end via -:mod:`subprocess`, using ``sys.executable`` so the tests run under whatever -interpreter is running pytest itself (matching how ``run_profile.sh`` -invokes this script with a plain ``python3``). -""" - -from __future__ import annotations - -import csv -import subprocess -import sys -from pathlib import Path - -import pytest - -from parse_nccl import ( - UNIFIED_CSV_FIELDNAMES, - parse_nccl_tests, - parse_torus_bench, - rows_to_csv, -) - -# Design: resolve fixture/script paths relative to this test file (not the -# CWD) so the suite passes regardless of where pytest is invoked from, per -# the spec's instruction to load fixtures with pathlib relative to the test -# file. -TESTS_DIR = Path(__file__).resolve().parent -FIXTURES_DIR = TESTS_DIR / "fixtures" -PARSE_NCCL_SCRIPT = TESTS_DIR.parent / "parse_nccl.py" - -FC_ALL_REDUCE_LOG = FIXTURES_DIR / "fc_all_reduce.log" -FC_ALLTOALL_LOG = FIXTURES_DIR / "fc_alltoall.log" -TORUS_ALL_REDUCE_LOG = FIXTURES_DIR / "torus_all_reduce.log" - - -def test_parse_nccl_tests_all_reduce(): - """fc_all_reduce.log parses to 2 rows with the expected first-row values. - - Exercises the common case: a well-formed all_reduce_perf log with a - standard 13-token data row (size, count, type, redop, root, then the - trailing-8 out-of-place/in-place metrics). - """ - text = FC_ALL_REDUCE_LOG.read_text(encoding="utf-8") - rows = parse_nccl_tests(text) - - assert len(rows) == 2 - first = rows[0] - assert first["size_bytes"] == 1048576 - assert first["count"] == 262144 - assert first["dtype"] == "float" - assert first["time_us"] == pytest.approx(98.52) - assert first["algbw_GBps"] == pytest.approx(10.64) - assert first["busbw_GBps"] == pytest.approx(18.62) - assert first["wrong"] == "0" - - -def test_parse_nccl_tests_alltoall_na_wrong(): - """fc_alltoall.log parses to 2 rows and exercises the 'N/A' #wrong path. - - alltoall_perf prints redop="none" and root="-1" instead of a real - reduction op/root -- this test confirms the trailing-8-token rule - parses those rows correctly regardless, and that an "N/A" out-of-place - #wrong value (validation disabled for that data point) is preserved - as the literal string "N/A" rather than raising or being coerced to a - number. - """ - text = FC_ALLTOALL_LOG.read_text(encoding="utf-8") - rows = parse_nccl_tests(text) - - assert len(rows) == 2 - assert rows[0]["wrong"] == "N/A" - assert rows[0]["size_bytes"] == 1048576 - assert rows[0]["time_us"] == pytest.approx(120.44) - # Second row is a normal (non-N/A) row, confirming N/A handling on row 0 - # didn't leak into subsequent parsing. - assert rows[1]["wrong"] == "0" - assert rows[1]["size_bytes"] == 2097152 - - -def test_parse_torus_bench_all_reduce(): - """torus_all_reduce.log parses to 3 rows with the expected first row. - - Exercises the comma-delimited TORUSBENCH sentinel format and the - check-field-to-wrong-string remapping (check "1" -> wrong "0"). - """ - text = TORUS_ALL_REDUCE_LOG.read_text(encoding="utf-8") - rows = parse_torus_bench(text) - - assert len(rows) == 3 - first = rows[0] - assert first["collective"] == "all_reduce" - assert first["dims"] == "2x2x2" - assert first["size_bytes"] == 1048576 - assert first["time_us"] == pytest.approx(142.11) - assert first["wrong"] == "0" - - -def test_rows_to_csv_round_trip(tmp_path): - """rows_to_csv() writes the exact unified header, in order, and empty - algbw/busbw cells for torus rows round-trip as empty strings. - - Builds one nccl-tests-shaped row and one torus_bench-shaped row (as - main() would produce them) and confirms the on-disk CSV, when read back - with csv.DictReader, has fieldnames matching UNIFIED_CSV_FIELDNAMES - exactly (order included) and that the torus row's bandwidth columns are - empty rather than "None" or some other stand-in. - """ - rows = [ - { - "source": "nccl-tests", - "topology": "fc", - "dims": "8", - "collective": "all_reduce", - "size_bytes": 1048576, - "count": 262144, - "dtype": "float", - "time_us": 98.52, - "algbw_GBps": 10.64, - "busbw_GBps": 18.62, - "wrong": "0", - }, - { - "source": "torus_bench", - "topology": "torus", - "dims": "2x2x2", - "collective": "all_reduce", - "size_bytes": 1048576, - "count": "", - "dtype": "", - "time_us": 142.11, - "algbw_GBps": "", - "busbw_GBps": "", - "wrong": "0", - }, - ] - out_path = tmp_path / "unified.csv" - - rows_to_csv(rows, out_path) - - with out_path.open(newline="", encoding="utf-8") as f: - reader = csv.DictReader(f) - assert reader.fieldnames == UNIFIED_CSV_FIELDNAMES - read_rows = list(reader) - - assert len(read_rows) == 2 - assert read_rows[1]["source"] == "torus_bench" - assert read_rows[1]["algbw_GBps"] == "" - assert read_rows[1]["busbw_GBps"] == "" - - -def _run_cli(*args: str) -> subprocess.CompletedProcess: - """Invoke parse_nccl.py's CLI as a subprocess. - - Parameters - ---------- - *args : str - Arguments to pass after the script path, e.g. the raw log path and - ``--source``/``--out``/etc. flags. - - Returns - ------- - subprocess.CompletedProcess - Result of the invocation, with stdout/stderr captured as text. - - Notes - ----- - Design: uses ``sys.executable`` (not a hard-coded ``python3``) so the - subprocess runs under the exact interpreter executing the test suite, - matching pytest's own environment rather than risking a PATH mismatch. - """ - return subprocess.run( - [sys.executable, str(PARSE_NCCL_SCRIPT), *args], - capture_output=True, - text=True, - ) - - -def test_cli_nccl_tests_end_to_end(tmp_path): - """CLI parses an nccl-tests log to a CSV with the expected row count.""" - out_path = tmp_path / "fc_all_reduce.csv" - result = _run_cli( - str(FC_ALL_REDUCE_LOG), - "--source", - "nccl-tests", - "--collective", - "all_reduce", - "--topology", - "fc", - "--out", - str(out_path), - ) - - assert result.returncode == 0, result.stderr - assert out_path.exists() - - with out_path.open(newline="", encoding="utf-8") as f: - read_rows = list(csv.DictReader(f)) - assert len(read_rows) == 2 - assert all(row["source"] == "nccl-tests" for row in read_rows) - assert all(row["dims"] == "8" for row in read_rows) - - -def test_cli_torus_bench_end_to_end(tmp_path): - """CLI parses a torus_bench log to a CSV with the expected row count.""" - out_path = tmp_path / "torus_all_reduce.csv" - result = _run_cli( - str(TORUS_ALL_REDUCE_LOG), - "--source", - "torus_bench", - "--collective", - "all_reduce", - "--topology", - "torus", - "--dims", - "2x2x2", - "--out", - str(out_path), - ) - - assert result.returncode == 0, result.stderr - assert out_path.exists() - - with out_path.open(newline="", encoding="utf-8") as f: - read_rows = list(csv.DictReader(f)) - assert len(read_rows) == 3 - assert all(row["source"] == "torus_bench" for row in read_rows) - assert all(row["algbw_GBps"] == "" for row in read_rows) - - -def test_cli_torus_bench_collective_mismatch_errors(tmp_path): - """CLI exits non-zero when --collective disagrees with the log content. - - Bonus coverage beyond the spec's 6 mandated cases: confirms the - validate-CLI-against-sentinel behavior documented in parse_nccl.main() - actually triggers a hard failure rather than silently mislabeling data, - since a silent mismatch here would corrupt the correlation notebook's - inputs without any visible signal. - """ - out_path = tmp_path / "should_not_be_created.csv" - result = _run_cli( - str(TORUS_ALL_REDUCE_LOG), - "--source", - "torus_bench", - "--collective", - "all_gather", # deliberately wrong; fixture is all_reduce - "--topology", - "torus", - "--out", - str(out_path), - ) - - assert result.returncode != 0 - assert not out_path.exists() - - -@pytest.mark.parametrize( - "log_text", - [ - "# just a comment\n\n# another comment\n", - "hello world\n", - "# nThread 1 nGpus 8\nhello world\n# trailing comment\n", - ], -) -def test_parse_nccl_tests_skips_malformed_lines(log_text): - """Comment-only, blank, and garbage lines are skipped without raising.""" - assert parse_nccl_tests(log_text) == [] - - -@pytest.mark.parametrize( - "log_text", - [ - "# torus_bench collective=all_reduce dims=2x2x2\n", - "hello world\n", - "# header\nhello world\nTORUSBENCH,incomplete,field\n", - ], -) -def test_parse_torus_bench_skips_malformed_lines(log_text): - """Comment-only, blank, garbage, and malformed sentinel lines are - skipped without raising (including a TORUSBENCH, line with too few - comma-separated fields).""" - assert parse_torus_bench(log_text) == [] diff --git a/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py b/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py deleted file mode 100644 index 0e8caad2..00000000 --- a/notebooks/astrasim2_correlation/correlation/tests/test_provision_teardown.py +++ /dev/null @@ -1,802 +0,0 @@ -"""Tests for config.py, provision.py, and teardown.py. - -These tests never touch real AWS: every boto3 client call is intercepted -by ``botocore.stub.Stubber``, which asserts on the exact request -parameters and returns a canned response (or raises a canned -``ClientError``) instead of making a network call. This lets the tests -exercise real botocore request-building and error-handling code paths -- -including exact parameter shape assertions -- with zero network access and -zero AWS credentials, matching this work package's hard constraint that -nothing may call AWS. - -Import strategy ------------------ -``config.py``/``provision.py``/``teardown.py`` live directly under -``correlation/`` (one level above this ``tests/`` package), and -``correlation/`` is deliberately *not* a Python package (no -``__init__.py`` there -- see the work-package spec). ``provision.py`` and -``teardown.py`` both do a plain ``from config import Config``, which only -resolves if ``correlation/`` is on ``sys.path``. We therefore insert that -directory onto ``sys.path`` explicitly before importing any of the three -modules under test, rather than relying on pytest's own rootdir-insertion -behavior (which happens to also achieve this here, but only for the -specific "prepend" import mode and package-marker layout currently in -place -- an explicit insert is robust to either changing). -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -import pytest - -# boto3 is not (and must not become) an accelforge package dependency; skip -# this whole module rather than error if it is not installed in the -# environment running the tests. -boto3 = pytest.importorskip("boto3") - -from botocore.exceptions import ClientError # noqa: E402 (after importorskip) -from botocore.stub import Stubber # noqa: E402 - -_CORRELATION_DIR = Path(__file__).resolve().parent.parent -if str(_CORRELATION_DIR) not in sys.path: - sys.path.insert(0, str(_CORRELATION_DIR)) - -import config # noqa: E402 -import provision # noqa: E402 -import teardown # noqa: E402 - -Config = config.Config - - -def _make_client(service_name: str): - """Build a boto3 client with dummy credentials, safe for Stubber use. - - Parameters - ---------- - service_name : str - E.g. ``"ec2"`` or ``"ssm"``. - - Returns - ------- - botocore.client.BaseClient - A real boto3 client object, never used to make a real network - call in these tests (every call site below is wrapped in a - ``Stubber`` context). - - Notes - ----- - Design: ``Stubber`` intercepts the HTTP send step, but botocore still - runs its normal request-signing step first, which raises - ``NoCredentialsError`` if no credentials are configured anywhere - (env vars, profile, instance metadata, ...). This test environment - intentionally has none, so every client is built with harmless dummy - static credentials purely to satisfy the signer -- these are never - sent anywhere, since Stubber never performs a real HTTP request. - """ - return boto3.client( - service_name, - region_name="us-east-1", - aws_access_key_id="testing", - aws_secret_access_key="testing", - ) - - -# --------------------------------------------------------------------------- -# resolve_ami -# --------------------------------------------------------------------------- - - -def test_resolve_ami_returns_stubbed_parameter_value(): - """resolve_ami extracts Parameter.Value from the SSM response.""" - ssm_client = _make_client("ssm") - stubber = Stubber(ssm_client) - parameter_name = ( - "/aws/service/deeplearning/ami/x86_64/" - "base-oss-nvidia-driver-gpu-ubuntu-22.04/latest/ami-id" - ) - stubber.add_response( - "get_parameter", - { - "Parameter": { - "Name": parameter_name, - "Value": "ami-0123456789abcdef0", - "Type": "String", - } - }, - {"Name": parameter_name}, - ) - - with stubber: - result = provision.resolve_ami(ssm_client, parameter_name) - - stubber.assert_no_pending_responses() - assert result == "ami-0123456789abcdef0" - - -# --------------------------------------------------------------------------- -# launch_instance -# --------------------------------------------------------------------------- - - -def test_launch_instance_spot_then_ondemand_falls_back_on_capacity_error(): - """A spot InsufficientInstanceCapacity error triggers an on-demand retry. - - Also asserts (per the work-package spec) that the first, failing - request was a spot request -- i.e. it carried InstanceMarketOptions -- - both directly (inspecting the kwargs dict) and indirectly (via - Stubber's expected_params, which would fail the test if - launch_instance's real spot request didn't match). - - Design (Fix 10): the FIRST call's expected_params is an independently - hardcoded literal dict, not derived from - ``provision._build_run_instances_kwargs`` -- deriving it from the same - helper the code under test calls would make this test circular (a bug - in that helper's request-shape would go undetected, since the test's - expectation and the code's actual request would drift together). The - literal below pins the exact request shape against the work-package - spec instead of against the code's own helper. The second (on-demand - fallback) call keeps using the helper-derived ``ondemand_kwargs`` for - convenience, since its shape isn't the focus of this particular test. - """ - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - cfg = Config(purchasing="spot-then-ondemand", run_id="test-run") - ami_id = "ami-0123456789abcdef0" - sg_id = "sg-0123456789abcdef0" - key_name = "accelforge-correlation-test-run" - - # Independently hardcoded, per the work-package spec's exact field list - # -- see the docstring above for why this must NOT be derived from - # provision._build_run_instances_kwargs. - spot_kwargs_literal = { - "ImageId": ami_id, - "InstanceType": "p5.48xlarge", - "KeyName": key_name, - "SecurityGroupIds": [sg_id], - "MinCount": 1, - "MaxCount": 1, - "InstanceInitiatedShutdownBehavior": "terminate", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 200, - "VolumeType": "gp3", - "DeleteOnTermination": True, - }, - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - {"Key": "Project", "Value": "accelforge-correlation"}, - {"Key": "RunId", "Value": "test-run"}, - {"Key": "Name", "Value": "accelforge-correlation-test-run"}, - ], - }, - { - "ResourceType": "volume", - "Tags": [ - {"Key": "Project", "Value": "accelforge-correlation"}, - {"Key": "RunId", "Value": "test-run"}, - {"Key": "Name", "Value": "accelforge-correlation-test-run"}, - ], - }, - ], - "DryRun": False, - "InstanceMarketOptions": { - "MarketType": "spot", - "SpotOptions": { - "SpotInstanceType": "one-time", - "InstanceInterruptionBehavior": "terminate", - }, - }, - } - # Kept helper-derived for the fallback call, per the spec ("if convenient"). - ondemand_kwargs = provision._build_run_instances_kwargs( - cfg, ami_id, sg_id, key_name, dry_run=False, use_spot=False - ) - - # The spec's explicit ask: the FIRST request must have carried - # InstanceMarketOptions (spot), the fallback must not. - assert "InstanceMarketOptions" in spot_kwargs_literal - assert "InstanceMarketOptions" not in ondemand_kwargs - - stubber.add_client_error( - "run_instances", - service_error_code="InsufficientInstanceCapacity", - service_message="There is no Spot capacity available.", - expected_params=spot_kwargs_literal, - ) - stubber.add_response( - "run_instances", - {"Instances": [{"InstanceId": "i-0123456789abcdef0"}]}, - expected_params=ondemand_kwargs, - ) - - with stubber: - result = provision.launch_instance( - ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False - ) - - stubber.assert_no_pending_responses() - assert result == { - "instance_id": "i-0123456789abcdef0", - "purchasing_used": "ondemand", - } - - -def test_launch_instance_spot_only_does_not_fall_back(): - """purchasing="spot" propagates the ClientError instead of retrying on-demand.""" - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - cfg = Config(purchasing="spot", run_id="test-run") - ami_id = "ami-0123456789abcdef0" - sg_id = "sg-0123456789abcdef0" - key_name = "accelforge-correlation-test-run" - - spot_kwargs = provision._build_run_instances_kwargs( - cfg, ami_id, sg_id, key_name, dry_run=False, use_spot=True - ) - # Only one response is ever queued: if launch_instance incorrectly - # attempted a second (fallback) call, Stubber itself would raise for - # having no more queued responses, which is not a ClientError -- so - # pytest.raises(ClientError) below would fail loudly in that case too. - stubber.add_client_error( - "run_instances", - service_error_code="InsufficientInstanceCapacity", - service_message="There is no Spot capacity available.", - expected_params=spot_kwargs, - ) - - with stubber: - with pytest.raises(ClientError): - provision.launch_instance( - ec2_client, cfg, ami_id, sg_id, key_name, dry_run=False - ) - - stubber.assert_no_pending_responses() - - -def test_launch_instance_dry_run_success_does_not_raise(capsys): - """A DryRunOperation error is treated as a successful authorization check.""" - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - cfg = Config(purchasing="ondemand", run_id="test-run") - ami_id = "ami-0123456789abcdef0" - sg_id = "sg-0123456789abcdef0" - key_name = "accelforge-correlation-test-run" - - ondemand_kwargs = provision._build_run_instances_kwargs( - cfg, ami_id, sg_id, key_name, dry_run=True, use_spot=False - ) - stubber.add_client_error( - "run_instances", - service_error_code="DryRunOperation", - service_message="Request would have succeeded, but DryRun flag is set.", - expected_params=ondemand_kwargs, - ) - - with stubber: - result = provision.launch_instance( - ec2_client, cfg, ami_id, sg_id, key_name, dry_run=True - ) - - stubber.assert_no_pending_responses() - assert result["purchasing_used"] == "ondemand" - # No instance was actually created during a dry run. - assert result["instance_id"] is None - assert "dry-run OK" in capsys.readouterr().out - - -# --------------------------------------------------------------------------- -# ensure_security_group -# --------------------------------------------------------------------------- - - -def test_ensure_security_group_authorizes_requested_cidr(): - """ensure_security_group wires the given ssh_cidr into the ingress rule.""" - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - vpc_id = "vpc-0123456789abcdef0" - sg_id = "sg-0123456789abcdef0" - ssh_cidr = "203.0.113.5/32" - group_name = "accelforge-correlation-test-run-sg" - run_id = "test-run" - tag_project = "accelforge-correlation" - - stubber.add_response( - "describe_vpcs", - {"Vpcs": [{"VpcId": vpc_id, "IsDefault": True}]}, - {"Filters": [{"Name": "isDefault", "Values": ["true"]}]}, - ) - stubber.add_response( - "create_security_group", - {"GroupId": sg_id}, - { - "GroupName": group_name, - "Description": f"accelforge correlation study SG for run {run_id}", - "VpcId": vpc_id, - }, - ) - # The parameter that matters most here: expected_params pins the exact - # CidrIp ensure_security_group must send, so the test fails loudly if - # the wrong CIDR (or the wrong port) were ever authorized. - stubber.add_response( - "authorize_security_group_ingress", - {}, - { - "GroupId": sg_id, - "IpPermissions": [ - { - "IpProtocol": "tcp", - "FromPort": 22, - "ToPort": 22, - "IpRanges": [ - { - "CidrIp": ssh_cidr, - "Description": "SSH access for accelforge correlation study", - } - ], - } - ], - }, - ) - stubber.add_response( - "create_tags", - {}, - { - "Resources": [sg_id], - "Tags": [ - {"Key": "Project", "Value": tag_project}, - {"Key": "RunId", "Value": run_id}, - {"Key": "Name", "Value": group_name}, - ], - }, - ) - - with stubber: - result = provision.ensure_security_group( - ec2_client, group_name, ssh_cidr, tag_project, run_id - ) - - stubber.assert_no_pending_responses() - assert result == sg_id - - -# --------------------------------------------------------------------------- -# teardown.find_tagged_instances -# --------------------------------------------------------------------------- - - -def test_find_tagged_instances_returns_ids_from_both_reservations(): - """find_tagged_instances flattens across multiple Reservations entries.""" - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - tag_project = "accelforge-correlation" - expected_filters = { - "Filters": [ - {"Name": "tag:Project", "Values": [tag_project]}, - { - "Name": "instance-state-name", - "Values": ["pending", "running", "stopping", "stopped"], - }, - ] - } - stubber.add_response( - "describe_instances", - { - "Reservations": [ - { - "Instances": [ - { - "InstanceId": "i-aaaa000000000001", - "State": {"Name": "running"}, - "Tags": [{"Key": "RunId", "Value": "run-a"}], - } - ] - }, - { - "Instances": [ - { - "InstanceId": "i-bbbb000000000002", - "State": {"Name": "pending"}, - "Tags": [{"Key": "RunId", "Value": "run-b"}], - } - ] - }, - ] - }, - expected_filters, - ) - - with stubber: - result = teardown.find_tagged_instances(ec2_client, tag_project) - - stubber.assert_no_pending_responses() - ids = {instance["InstanceId"] for instance in result} - assert ids == {"i-aaaa000000000001", "i-bbbb000000000002"} - - -# --------------------------------------------------------------------------- -# teardown security-group delete retry -# --------------------------------------------------------------------------- - - -def test_delete_security_group_retries_past_dependency_violation(monkeypatch): - """A DependencyViolation is retried (not fatal) and eventually succeeds.""" - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - sg_id = "sg-0123456789abcdef0" - - stubber.add_client_error( - "delete_security_group", - service_error_code="DependencyViolation", - service_message="resource sg-0123456789abcdef0 has a dependent object", - expected_params={"GroupId": sg_id}, - ) - stubber.add_response("delete_security_group", {}, {"GroupId": sg_id}) - - sleep_calls = [] - # Patch time.sleep as seen through teardown's own `import time`, so the - # retry loop does not actually block the test suite for - # _SG_DELETE_RETRY_SLEEP_S seconds. - monkeypatch.setattr(teardown.time, "sleep", lambda seconds: sleep_calls.append(seconds)) - - with stubber: - teardown._delete_security_group_with_retry(ec2_client, sg_id) - - stubber.assert_no_pending_responses() - assert len(sleep_calls) == 1 - - -# --------------------------------------------------------------------------- -# Fix 3 (MAJOR): caller_ip() fails fast instead of fail-open -# --------------------------------------------------------------------------- - - -def test_caller_ip_raises_on_discovery_failure(monkeypatch): - """caller_ip() raises RuntimeError (mentioning --ssh-cidr) on any discovery failure. - - Regression test for Fix 3: the old behavior returned the sentinel - "0.0.0.0" on failure, which every caller turned into the CIDR - "0.0.0.0/32" -- unreachable by anyone, including the operator -- only - after real AWS resources already existed and were already billing. - Failing fast here means the error surfaces before any of that happens. - """ - - def fake_urlopen(*args, **kwargs): - raise provision.urllib.error.URLError("simulated DNS failure") - - monkeypatch.setattr(provision.urllib.request, "urlopen", fake_urlopen) - - with pytest.raises(RuntimeError, match="--ssh-cidr"): - provision.caller_ip() - - -# --------------------------------------------------------------------------- -# Fix 2 (MAJOR): provision.main() closes the orphan-instance window -# --------------------------------------------------------------------------- - - -def test_provision_main_writes_state_before_wait_and_reports_recovery_on_failure( - tmp_path, monkeypatch, capsys -): - """provision.main() writes state (public_ip=None) before wait_for_instance runs, - and on a wait_for_instance failure prints a loud recovery block (instance - id, STILL RUNNING AND BILLING, state file path, exact recovery command) - and re-raises rather than silently losing track of a running instance. - - Every AWS-touching seam provision.main() has (resolve_ami, - ensure_key_pair, ensure_security_group, launch_instance, - wait_for_instance, and boto3.client itself) is monkeypatched with a - lightweight fake, mirroring test_orchestrate.py's ``orchestrate_fakes`` - approach -- this exercises provision.main()'s own sequencing/state-file - logic (what Fix 2 changed) without touching real AWS or needing a - Stubber response sequence for calls this test never lets happen for - real. - """ - - class _FakeBoto3: - @staticmethod - def client(service_name, region_name=None): - return f"fake-{service_name}-client[{region_name}]" - - monkeypatch.setattr(provision, "boto3", _FakeBoto3) - monkeypatch.setattr(provision, "resolve_ami", lambda ssm, param: "ami-fake0123456789") - - def fake_ensure_key_pair(ec2, key_name, key_dir): - key_dir.mkdir(parents=True, exist_ok=True) - return key_dir / f"{key_name}.pem" - - monkeypatch.setattr(provision, "ensure_key_pair", fake_ensure_key_pair) - monkeypatch.setattr( - provision, - "ensure_security_group", - lambda ec2, name, cidr, tag_project, run_id: "sg-fake0123456789", - ) - monkeypatch.setattr( - provision, - "launch_instance", - lambda ec2, cfg, ami_id, sg_id, key_name, dry_run=False: { - "instance_id": "i-fake0123456789", - "purchasing_used": "spot", - }, - ) - - state_dir = tmp_path / "state" - run_id = "test-run-orphan-window" - - def failing_wait_for_instance(ec2, instance_id): - # By the time wait_for_instance is called, state must already be on - # disk with public_ip still the None placeholder -- exactly the - # ordering Fix 2 requires (state written BEFORE the SSH wait, not - # only after it succeeds). - state_path = state_dir / f"{run_id}.json" - assert state_path.exists(), "state file must exist before wait_for_instance is called" - written = json.loads(state_path.read_text()) - assert written["instance_id"] == instance_id - assert written["public_ip"] is None - raise TimeoutError("simulated SSH-reachability timeout") - - monkeypatch.setattr(provision, "wait_for_instance", failing_wait_for_instance) - - argv = [ - "--yes", - "--run-id", - run_id, - "--key-dir", - str(tmp_path / "keys"), - "--state-dir", - str(state_dir), - "--ssh-cidr", - "203.0.113.5/32", - ] - - with pytest.raises(TimeoutError): - provision.main(argv) - - # The state file must survive the failure -- teardown.py needs it to - # find and tear down the still-running instance later. - state_path = state_dir / f"{run_id}.json" - assert state_path.exists() - - err = capsys.readouterr().err - assert "i-fake0123456789" in err - assert "STILL RUNNING AND BILLING" in err - assert str(state_path) in err - assert f"python teardown.py --run-id {run_id} --region" in err - - -# --------------------------------------------------------------------------- -# Fix 4a (MAJOR): teardown.resolve_regions -- pure region-resolution logic -# --------------------------------------------------------------------------- -# -# These tests make no AWS calls and use no Stubber, per resolve_regions()'s -# own design (a pure function of `args` and a plain dict of loaded state -# files) -- `args` is a minimal argparse.Namespace stand-in, not a fully -# parsed CLI invocation, since resolve_regions() only ever consults -# `.region` and `.run_id`. - - -def test_resolve_regions_explicit_flag_wins_over_state_region(): - """An explicit --region always overrides a state file's own recorded region.""" - args = argparse.Namespace(region="us-west-2", run_id="run-a") - states = {"run-a": {"region": "eu-central-1"}} - - assert teardown.resolve_regions(args, states) == {"us-west-2": ["run-a"]} - - -def test_resolve_regions_uses_state_file_region_when_no_flag(): - """With no --region, a single --run-id's own recorded region is used.""" - args = argparse.Namespace(region=None, run_id="run-a") - states = {"run-a": {"region": "ap-southeast-2"}} - - assert teardown.resolve_regions(args, states) == {"ap-southeast-2": ["run-a"]} - - -def test_resolve_regions_falls_back_to_config_default_when_unknown(): - """With no --region and no matching state file, Config's default region is used.""" - args = argparse.Namespace(region=None, run_id="run-with-no-state-file") - states: dict = {} - - assert teardown.resolve_regions(args, states) == { - Config().region: ["run-with-no-state-file"] - } - - -def test_resolve_regions_all_groups_by_recorded_region_and_keeps_default(): - """--all (run_id=None) groups every known run by its own region, plus the default.""" - args = argparse.Namespace(region=None, run_id=None) - states = { - "run-a": {"region": "us-west-2"}, - "run-b": {"region": "us-west-2"}, - "run-c": {"region": "eu-central-1"}, - "run-d": {}, # no recorded region at all -> falls back to the default - } - - result = teardown.resolve_regions(args, states) - - assert set(result["us-west-2"]) == {"run-a", "run-b"} - assert result["eu-central-1"] == ["run-c"] - # The default/fallback region (Config's own default) is always present - # as a key, even though only run-d actually landed there via fallback, - # so a caller iterating this mapping's keys always still searches it - # for tag-only discovery of state-less instances. - assert "run-d" in result[Config().region] - - -def test_resolve_regions_all_explicit_flag_overrides_every_run(): - """An explicit --region with --all overrides every individual run's own region.""" - args = argparse.Namespace(region="us-east-2", run_id=None) - states = { - "run-a": {"region": "us-west-2"}, - "run-b": {"region": "eu-central-1"}, - } - - result = teardown.resolve_regions(args, states) - - assert result == {"us-east-2": ["run-a", "run-b"]} - - -# --------------------------------------------------------------------------- -# Fix 4b (MAJOR): a stale state file (no matching live instance) is routed -# through teardown_run -- SG/key-pair cleanup included -- BEFORE its local -# state file is removed, instead of just being unlinked. -# --------------------------------------------------------------------------- - - -def test_teardown_one_region_stale_state_calls_delete_security_group_before_removing_file( - tmp_path, -): - """A stale run's security group is deleted before its state file disappears. - - Regression test for Fix 4b: the pre-fix behavior just unlinked a stale - state file without ever touching AWS, leaking the security group (and, - with --delete-key, the key pair) of any run whose instance died out - from under it -- e.g. via the on-instance dead-man timer firing -- before - teardown.py was ever run. Stubber's queued ``delete_security_group`` - response is only consumed if ``_teardown_one_region``'s internal - ``teardown_run`` call actually invokes it (``stubber.assert_no_pending_ - responses()`` below fails the test otherwise); combined with the state - file only being removed from disk AFTER that call returns without - raising, these two assertions together establish the required "SG - deleted before state file disappears" ordering. - """ - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - run_id = "stale-run" - instance_id = "i-0123456789abcdef0" - sg_id = "sg-0123456789abcdef0" - tag_project = "accelforge-correlation" - - state = { - "run_id": run_id, - "region": "us-east-1", - "instance_id": instance_id, - "sg_id": sg_id, - "key_name": "accelforge-correlation-stale-run", - "key_path": None, - } - state_path = tmp_path / f"{run_id}.json" - state_path.write_text(json.dumps(state)) - - # Discovery: no live instance matches this run -- makes it "stale". - stubber.add_response( - "describe_instances", - {"Reservations": []}, - { - "Filters": [ - {"Name": "tag:Project", "Values": [tag_project]}, - { - "Name": "instance-state-name", - "Values": ["pending", "running", "stopping", "stopped"], - }, - ] - }, - ) - # teardown_run's terminate step: the instance is already gone (e.g. the - # dead-man timer fired) -- tolerated as InvalidInstanceID.NotFound, per - # teardown_run's own existing contract. - stubber.add_client_error( - "terminate_instances", - service_error_code="InvalidInstanceID.NotFound", - service_message=f"The instance ID '{instance_id}' does not exist", - expected_params={"InstanceIds": [instance_id]}, - ) - # The assertion this test exists for: delete_security_group MUST be - # called (Stubber raises on assert_no_pending_responses() otherwise). - stubber.add_response("delete_security_group", {}, {"GroupId": sg_id}) - - with stubber: - exit_code = teardown._teardown_one_region( - ec2_client, - tag_project, - "us-east-1", - None, - {run_id: state}, - {run_id: state_path}, - delete_key=False, - yes=True, - ) - - stubber.assert_no_pending_responses() - assert exit_code == 0 - # The state file is gone only now that teardown_run (including - # delete_security_group) has already completed successfully. - assert not state_path.exists() - - -def test_teardown_one_region_stale_state_keeps_file_if_teardown_run_raises(tmp_path, monkeypatch): - """If the stale-cleanup teardown_run call itself raises, the state file survives. - - Complements the happy-path stale-cleanup test above: a failed cleanup - must not lose track of the leak by removing the state file anyway. - """ - ec2_client = _make_client("ec2") - stubber = Stubber(ec2_client) - - run_id = "stale-run-cleanup-fails" - sg_id = "sg-0123456789abcdef0" - tag_project = "accelforge-correlation" - - # No instance_id: teardown_run skips straight to security-group - # deletion, which is the call we make fail here. - state = {"run_id": run_id, "region": "us-east-1", "sg_id": sg_id, "key_name": None} - state_path = tmp_path / f"{run_id}.json" - state_path.write_text(json.dumps(state)) - - # Avoid actually sleeping between retries (see - # test_delete_security_group_retries_past_dependency_violation's - # identical rationale for patching teardown's own `import time`). - monkeypatch.setattr(teardown.time, "sleep", lambda seconds: None) - - stubber.add_response( - "describe_instances", - {"Reservations": []}, - { - "Filters": [ - {"Name": "tag:Project", "Values": [tag_project]}, - { - "Name": "instance-state-name", - "Values": ["pending", "running", "stopping", "stopped"], - }, - ] - }, - ) - # A persistent DependencyViolation exhausts _delete_security_group_with_retry's - # retries and surfaces as a RuntimeError from teardown_run. - for _ in range(teardown._SG_DELETE_MAX_RETRIES): - stubber.add_client_error( - "delete_security_group", - service_error_code="DependencyViolation", - service_message="resource has a dependent object", - expected_params={"GroupId": sg_id}, - ) - - with stubber: - exit_code = teardown._teardown_one_region( - ec2_client, - tag_project, - "us-east-1", - None, - {run_id: state}, - {run_id: state_path}, - delete_key=False, - yes=True, - ) - - stubber.assert_no_pending_responses() - assert exit_code == 1 - assert state_path.exists() diff --git a/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile b/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile deleted file mode 100644 index a53fd392..00000000 --- a/notebooks/astrasim2_correlation/correlation/torus_bench/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -CUDA_HOME ?= /usr/local/cuda -NCCL_HOME ?= -NVCC ?= $(CUDA_HOME)/bin/nvcc -GXX ?= g++ -ARCH ?= -arch=sm_90 - -.PHONY: clean - -NCCL_INC := $(if $(NCCL_HOME),-I$(NCCL_HOME)/include) -NCCL_LIB := $(if $(NCCL_HOME),-L$(NCCL_HOME)/lib) - -# Default target (first in this file): the GPU/NCCL executor. Requires nvcc plus a CUDA -# toolkit and NCCL headers/libs; NOT buildable in this development environment (no nvcc here -# -- see the work-package report). Provided so the GPU target compiles cleanly, by construction, -# on the target AWS p5.48xlarge instance, where CUDA_HOME/NCCL_HOME should be set as needed. -torus_bench: torus_bench.cu - $(NVCC) -O3 $(ARCH) $(NCCL_INC) $(NCCL_LIB) -o $@ $< -lnccl - -# Simulator target: forces g++ to treat the .cu file as plain C++ (-x c++), not CUDA source. -# This is the local, GPU-free test vehicle for the schedule-building, edge-assertion, and -# per-collective reduction/routing logic -- see the file header of torus_bench.cu for the full -# schedule-as-data rationale. This is the target exercised by the acceptance criteria. -torus_bench_sim: torus_bench.cu - $(GXX) -x c++ -std=c++17 -O2 -Wall -Wextra -DTORUS_SIM -o $@ $< - -clean: - rm -f torus_bench torus_bench_sim diff --git a/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu b/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu deleted file mode 100644 index ed17c0b9..00000000 --- a/notebooks/astrasim2_correlation/correlation/torus_bench/torus_bench.cu +++ /dev/null @@ -1,1681 +0,0 @@ -// torus_bench.cu -// -// Single-process multi-GPU NCCL benchmark that runs collective algorithms restricted to the -// EDGES OF A LOGICAL TORUS, on hardware that is physically fully-connected (one AWS -// p5.48xlarge, 8x H100 over NVSwitch). -// -// SCIENTIFIC PURPOSE (drives the whole design) -// --------------------------------------------- -// This benchmark exists to correlate an analytical torus-network model against real -// measurements. The measurement is only valid if EVERY inter-GPU transfer travels between -// logical torus NEIGHBORS -- that constraint IS the experiment. A single misrouted transfer -// (e.g. a "shortcut" the NVSwitch fabric would happily allow but the torus topology would not) -// silently invalidates the correlation. We therefore do not trust ourselves to hand-write -// per-collective CUDA/NCCL call sequences and eyeball their correctness; instead: -// -// SCHEDULE-AS-DATA: pure host code (no CUDA, no GPU) builds an explicit, fully materialized -// step-by-step transfer schedule (`Schedule` = vector, each Step a set of concurrent -// `Xfer`s followed by local reduce/copy `LocalOp`s). A single choke point -- the edge -// assertion inside build_schedule() -- inspects every Xfer the schedule will ever contain and -// aborts the program if any of them is not a torus-neighbor transfer. This assertion is the -// scientific guarantee of this file and must never be disabled, in either build. -// -// TWO EXECUTORS, ONE SCHEDULE: the identical Schedule produced by build_schedule() is handed -// to either of two interchangeable executors with matching function signatures: -// - a host-memory SIMULATOR (compiled here, in this environment, with no GPU/nvcc -// available -- this is the local test vehicle and is exercised by the acceptance -// criteria), and -// - an NCCL/CUDA executor (built on the target GPU instance; cannot be compiled or run in -// this environment since no nvcc/CUDA toolkit is installed here -- see the project report -// for confirmation of this constraint). -// Both executors are driven by the exact same schedule-building and CLI code; only the -// "how do I actually move these bytes" implementation differs, selected at compile time via -// the TORUS_SIM macro. Compiling with `g++ -x c++ -DTORUS_SIM` yields the simulator binary; -// compiling with `nvcc` (TORUS_SIM undefined) yields the GPU binary. All CUDA/NCCL-specific -// code is fenced with `#ifndef TORUS_SIM` so a plain C++ compiler never sees CUDA syntax. -// -// TOPOLOGY CONVENTION -// -------------------- -// dims = [d_0, ..., d_{K-1}] is a K-dimensional torus with product(dims) = N ranks. Rank <-> -// coordinate mapping is row-major with the LAST dimension fastest-varying (i.e. like a C array -// of shape `dims`). neighbor(r, dim, +-1) wraps around (mod dims[dim]). For an extent-2 -// dimension, +1 and -1 land on the SAME neighbor -- the code below computes this generically via -// modular arithmetic and never special-cases extent-2 dimensions. -// -// DOCUMENTATION CONVENTIONS USED IN THIS FILE -// --------------------------------------------- -// Each function has a comment block with: a one-line summary, Parameters, Returns, and (where -// relevant) Preconditions/Notes -- the C++ analogue of NumPy-style docstrings. Each per-collective -// schedule-builder additionally documents, phase by phase, the data-placement INVARIANT that -// phase establishes; this is the load-bearing correctness argument for that collective and is -// exactly what a reader needs to convince themselves the algorithm is right. -// -// DERIVATION NOTE ON reduce_scatter (see build_reduce_scatter() below for full detail): the -// planning spec for this work package proposed a tentative per-step slot formula for -// reduce_scatter and then explicitly flagged uncertainty about it ("hold on, mirror the -// all_gather invariant exactly"). That tentative formula, worked through by hand and confirmed -// with a throwaway Python simulation across dims=[8],[2,4],[4,2],[2,2,2], places each fully -// reduced slot one hop short of its destination rank. The corrected formula and direction -// (documented at build_reduce_scatter) were derived from a time-reversal argument against the -// (spec-verified-correct) all_gather formula and independently confirmed by brute-force -// simulation before being encoded here; the C++ simulator's --check flag re-verifies this at -// runtime for every size in the sweep. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// ===================================================================================== -// Buffer identifiers and collective enum -// ===================================================================================== - -// Logical per-rank buffer roles used by every schedule builder and both executors. Both -// executors allocate storage per (rank, BufId) according to buffer_sizes() below. -enum BufId : int { - BUF_SEND = 0, - BUF_RECV = 1, - BUF_TMP = 2, - BUF_WORK_A = 3, - BUF_WORK_B = 4, - NUM_BUFS = 5 -}; - -enum class Collective { ALL_REDUCE, ALL_GATHER, REDUCE_SCATTER, ALLTOALL, BROADCAST, SENDRECV }; - -// Returns the canonical CLI/output-format name for a collective. -// -// Parameters -// ---------- -// c : Collective -// -// Returns -// ------- -// const char* -- a string literal, e.g. "all_reduce". Never null. -const char* collective_name(Collective c) { - switch (c) { - case Collective::ALL_REDUCE: return "all_reduce"; - case Collective::ALL_GATHER: return "all_gather"; - case Collective::REDUCE_SCATTER: return "reduce_scatter"; - case Collective::ALLTOALL: return "alltoall"; - case Collective::BROADCAST: return "broadcast"; - case Collective::SENDRECV: return "sendrecv"; - } - return "?"; -} - -// Parses a --collective CLI argument. -// -// Parameters -// ---------- -// s : const std::string& -- one of the six recognized collective names. -// out : Collective& -- set on success; left unmodified on failure. -// -// Returns -// ------- -// bool -- true if `s` was recognized. -bool parse_collective(const std::string& s, Collective& out) { - if (s == "all_reduce") { out = Collective::ALL_REDUCE; return true; } - if (s == "all_gather") { out = Collective::ALL_GATHER; return true; } - if (s == "reduce_scatter") { out = Collective::REDUCE_SCATTER; return true; } - if (s == "alltoall") { out = Collective::ALLTOALL; return true; } - if (s == "broadcast") { out = Collective::BROADCAST; return true; } - if (s == "sendrecv") { out = Collective::SENDRECV; return true; } - return false; -} - -// Returns a human-readable name for a BufId, used only in --check diagnostic output. -const char* buf_name(int b) { - switch (b) { - case BUF_SEND: return "BUF_SEND"; - case BUF_RECV: return "BUF_RECV"; - case BUF_TMP: return "BUF_TMP"; - case BUF_WORK_A: return "BUF_WORK_A"; - case BUF_WORK_B: return "BUF_WORK_B"; - } - return "?"; -} - -// ===================================================================================== -// Schedule IR (pure data -- no CUDA dependency anywhere in this section) -// ===================================================================================== - -// One point-to-point transfer between two DIFFERENT ranks. Every Xfer that will ever be -// constructed by any builder in this file must satisfy is_torus_neighbor(src, dst) -- see -// check_edge_or_abort() and the final validation pass in build_schedule(). src == dst is -// forbidden by construction: same-rank data movement must be expressed as a LocalOp instead. -struct Xfer { - int src, dst; - int src_buf, dst_buf; - size_t src_off, dst_off, bytes; -}; - -// A same-rank operation: either a byte-for-byte copy (add == false) or an elementwise -// float accumulate dst[i] += src[i] (add == true), applied over `bytes` bytes (i.e. -// bytes/sizeof(float) floats -- bytes must be a multiple of 4 whenever add == true). -struct LocalOp { - int rank; - int src_buf; - size_t src_off; - int dst_buf; - size_t dst_off; - size_t bytes; - bool add; -}; - -// One synchronization step of a Schedule: all `xfers` are considered to execute concurrently -// (reading pre-step state), and only once every Xfer in the step has landed do the `post` -// LocalOps run (also concurrently with each other, since by construction no two post ops of -// the same step touch overlapping (rank,buf,offset) ranges). -struct Step { - std::vector xfers; - std::vector post; -}; - -using Schedule = std::vector; - -// ===================================================================================== -// Torus topology helpers -// ===================================================================================== - -// Returns N = product(dims), the total rank count of the torus. -inline int num_ranks(const std::vector& dims) { - int n = 1; - for (int d : dims) n *= d; - return n; -} - -// Converts a rank id to its torus coordinates. -// -// Parameters -// ---------- -// r : int -- rank id, 0 <= r < product(dims). -// dims : const std::vector& -- torus extents, dims[K-1] is the fastest-varying axis. -// -// Returns -// ------- -// std::vector -- coordinates, one per dimension, coords[j] in [0, dims[j]). -inline std::vector coords_of(int r, const std::vector& dims) { - std::vector c(dims.size()); - for (int j = (int)dims.size() - 1; j >= 0; --j) { - c[j] = r % dims[j]; - r /= dims[j]; - } - return c; -} - -// Inverse of coords_of(): converts torus coordinates back to a rank id (row-major, last -// dimension fastest). -inline int rank_of(const std::vector& c, const std::vector& dims) { - int r = 0; - for (size_t j = 0; j < dims.size(); ++j) r = r * dims[j] + c[j]; - return r; -} - -// Returns the rank reached from `r` by moving one hop of `delta` (+1 or -1) along dimension -// `dim`, wrapping around (mod dims[dim]). For an extent-2 dimension, delta=+1 and delta=-1 -// necessarily return the same rank -- this falls out of the modular arithmetic below with no -// special-casing, matching the spec's requirement that extent-2 degeneracy not be hard-coded. -inline int neighbor(int r, int dim, int delta, const std::vector& dims) { - std::vector c = coords_of(r, dims); - int e = dims[dim]; - c[dim] = ((c[dim] + delta) % e + e) % e; - return rank_of(c, dims); -} - -// Determines whether two ranks are torus neighbors: their coordinates must differ in exactly -// one dimension, and in that dimension by +-1 modulo the dimension's extent. -// -// Notes -// ----- -// For an extent-2 dimension this is trivially satisfied by any pair that differs there (both -// possible non-zero differences, 1 and (extent-1)=1, coincide), which is the intended behavior. -inline bool is_torus_neighbor(int a, int b, const std::vector& dims) { - std::vector ca = coords_of(a, dims), cb = coords_of(b, dims); - int diff_dim = -1, diff_count = 0; - for (size_t j = 0; j < dims.size(); ++j) { - if (ca[j] != cb[j]) { diff_dim = (int)j; ++diff_count; } - } - if (diff_count != 1) return false; - int e = dims[diff_dim]; - int d = ((ca[diff_dim] - cb[diff_dim]) % e + e) % e; - return d == 1 || d == e - 1; -} - -// Aborts the program with a file:line diagnostic if (src,dst) is not a torus edge, or if -// src == dst. This is THE scientific guarantee of this benchmark (see file header) and must -// remain active in both builds. -// -// Design decision: we use an explicit check + std::abort() rather than assert() from -// , because assert() compiles to a no-op under -DNDEBUG and we do not control every -// build environment this file might eventually be compiled in (e.g. a release-mode CI flag). -// An explicit check is unconditionally active regardless of optimization/NDEBUG flags. -inline void check_edge_or_abort(int src, int dst, const std::vector& dims) { - if (src == dst) { - std::fprintf(stderr, - "%s:%d: EDGE ASSERTION FAILED: Xfer has src==dst (rank %d); same-rank " - "movement must be expressed as a LocalOp, not an Xfer\n", - __FILE__, __LINE__, src); - std::abort(); - } - if (!is_torus_neighbor(src, dst, dims)) { - std::fprintf(stderr, - "%s:%d: EDGE ASSERTION FAILED: rank %d -> rank %d is not a torus " - "neighbor for the given dims; this transfer would not exist on the " - "logical torus and must not be scheduled\n", - __FILE__, __LINE__, src, dst); - std::abort(); - } -} - -// Enumerates every combination of coordinate values across dimensions [0, d), invoking `cb` -// once per combination with a coordinate vector `v` whose entries at indices >= d are left -// exactly as passed in (typically already pinned to a specific rank's own coordinates). -// -// Parameters -// ---------- -// d : int -- number of leading dimensions (0..d-1) to enumerate freely; if d == 0, `cb` is -// invoked exactly once, with `v` unchanged (the "no free dimensions" case). -// dims : const std::vector& -- torus extents. -// v : std::vector -- base coordinate vector (taken by value since we mutate indices < d -// during enumeration; entries at indices >= d are the caller's fixed values). -// cb : const std::function&)>& -- invoked once per combination. -// -// Notes -// ----- -// Enumeration order (dimension 0 varies slowest, in this implementation) is an arbitrary -// choice: every caller in this file treats each combination as an independent, order-agnostic -// unit of work (one Xfer/LocalOp pair per combination), so the traversal order used to reach -// the same combination SET has no effect on correctness. -inline void for_each_free_combo(int d, const std::vector& dims, std::vector v, - const std::function&)>& cb) { - std::function rec = [&](int j) { - if (j == d) { cb(v); return; } - for (int val = 0; val < dims[j]; ++val) { - v[j] = val; - rec(j + 1); - } - }; - rec(0); -} - -// ===================================================================================== -// Per-(collective,S) buffer sizing -// ===================================================================================== - -// Computes the per-rank byte size of each of the five logical buffers a given collective needs -// for total-collective-size S. Both executors call this identically to allocate storage (heap -// arrays for the simulator, cudaMalloc for the NCCL build) -- see file header for buffer roles. -// -// Parameters -// ---------- -// c : Collective -// dims : const std::vector& -- torus extents; N = product(dims). -// S : size_t -- total collective size in bytes for this sweep point. -// -// Returns -// ------- -// std::array -- indexed by BufId; unused buffers are size 0. -// -// Preconditions -// ------------- -// S must be divisible by N and by 4*N*N (whole-float shard/chunk boundaries); callers are -// expected to have already applied the sweep-level divisibility skip check (see main()) before -// calling this. -inline std::array buffer_sizes(Collective c, const std::vector& dims, - size_t S) { - int N = num_ranks(dims); - size_t m = S / (size_t)N; // per-rank shard, m = S/N, per the CLI spec's convention. - int K = (int)dims.size(); - std::array sz{}; - sz.fill(0); - switch (c) { - case Collective::SENDRECV: - sz[BUF_SEND] = m; - sz[BUF_RECV] = (size_t)K * m; // one m-byte region per dimension. - break; - case Collective::BROADCAST: - sz[BUF_SEND] = m; // only meaningful on root; allocated uniformly for simplicity. - sz[BUF_RECV] = m; - break; - case Collective::ALL_GATHER: - sz[BUF_SEND] = m; - sz[BUF_RECV] = (size_t)N * m; // N slots of m bytes each. - break; - case Collective::REDUCE_SCATTER: - sz[BUF_SEND] = (size_t)N * m; // N slots of m bytes each (destined for each rank). - sz[BUF_RECV] = m; - sz[BUF_WORK_A] = (size_t)N * m; // running accumulator, one slot per destination. - sz[BUF_TMP] = (size_t)N * m; // staging area for incoming adds. - break; - case Collective::ALL_REDUCE: - // Composed internally of reduce_scatter(shard=m/N) followed by all_gather - // (shard=m/N); see build_all_reduce() for the full derivation. Both sub-phases' - // internal buffer needs (N*(m/N) == m) collapse to a uniform m bytes here. - sz[BUF_SEND] = m; - sz[BUF_RECV] = m; - sz[BUF_WORK_A] = m; - sz[BUF_TMP] = m; - break; - case Collective::ALLTOALL: - sz[BUF_SEND] = m; // N chunks of c=m/N bytes each. - sz[BUF_RECV] = m; - sz[BUF_WORK_A] = 2 * m; // ping-pong in-flight staging; sized generously (2x) per - sz[BUF_WORK_B] = 2 * m; // spec, occupancy is asserted at schedule-build time. - break; - } - return sz; -} - -// ===================================================================================== -// Schedule builders (pure host code -- no CUDA dependency anywhere in this section) -// ===================================================================================== - -// Builds the schedule for `sendrecv`: one step per torus dimension, every rank exchanges its -// full BUF_SEND vector with its +1 neighbor in that dimension. -// -// Invariant per step d: after step d, every rank's BUF_RECV region [d*m, (d+1)*m) holds the -// data that neighbor(r, d, -1) sent -- i.e. its BUF_SEND contents -- since neighbor(r,d,-1)'s -// own send in this same step targets exactly rank r (send direction is always +1, and -// neighbor(neighbor(r,d,-1), d, +1) == r by construction of neighbor()). -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N is exchanged per dimension. -// -// Returns -// ------- -// Schedule -- K steps, N Xfers each, no LocalOps. -inline Schedule build_sendrecv(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - Schedule sched; - for (int d = 0; d < K; ++d) { - Step step; - for (int r = 0; r < N; ++r) { - int dst = neighbor(r, d, +1, dims); - check_edge_or_abort(r, dst, dims); - step.xfers.push_back({r, dst, BUF_SEND, BUF_RECV, 0, (size_t)d * m, m}); - } - sched.push_back(std::move(step)); - } - return sched; -} - -// Builds the schedule for `broadcast` from root rank 0. -// -// Algorithm: dimension-by-dimension forward-chain propagation. A host-side has_data[N] tracks, -// at schedule-BUILD time (not at run time), which ranks are already known to hold the -// broadcast data after each step; this bookkeeping exists purely to decide which Xfers to -// generate and is not part of the executed Schedule itself. -// -// Invariant: at the start of dimension d's phase, the set of ranks with has_data[r] == true is -// exactly the set of ranks that agree with rank 0 on every coordinate j >= d (this holds for -// d == 0 trivially: only rank 0 itself). Each of dimension d's (extent_d - 1) repeats extends -// every already-seeded "line" one hop further in the +1 direction; after all extent_d - 1 -// repeats, every rank agreeing with rank 0 on coordinates j > d has data (regardless of its -// coordinate d), establishing the invariant for phase d+1. After all K dimensions, every rank -// has data. -// -// Design decision: root's own copy is written by a dedicated step-0 LocalOp (BUF_SEND -> -// BUF_RECV) with NO concurrent Xfers in that same step. This lets every subsequent send (even -// root's very first "real" send) source uniformly from BUF_RECV: had root's first send shared -// step 0 with the LocalOp, it would need to special-case sourcing from BUF_SEND instead, since -// a Step's post-LocalOps run strictly after that step's Xfers land. -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N is the broadcast vector size. -// -// Returns -// ------- -// Schedule -- 1 (LocalOp-only) + sum_d(extent_d - 1) steps. -inline Schedule build_broadcast(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - Schedule sched; - { - Step step0; - step0.post.push_back({0, BUF_SEND, 0, BUF_RECV, 0, m, false}); - sched.push_back(std::move(step0)); - } - std::vector has_data(N, 0); - has_data[0] = 1; - for (int d = 0; d < K; ++d) { - int E = dims[d]; - for (int s = 0; s < E - 1; ++s) { - Step step; - std::vector new_has = has_data; // conditions evaluated against pre-step state. - for (int r = 0; r < N; ++r) { - if (!has_data[r]) continue; - int dst = neighbor(r, d, +1, dims); - if (has_data[dst]) continue; - check_edge_or_abort(r, dst, dims); - step.xfers.push_back({r, dst, BUF_RECV, BUF_RECV, 0, 0, m}); - new_has[dst] = 1; - } - has_data = new_has; - sched.push_back(std::move(step)); - } - } - return sched; -} - -// Builds the schedule for `all_gather`. -// -// Algorithm: standard ring all-gather, generalized to a mixed-radix torus by processing one -// dimension at a time (ascending order). Step 0 seeds each rank's own slot via a LocalOp. -// -// Invariant: at the START of dimension d's phase, rank r holds exactly the slots -// {u : u_j == r_j for all j >= d} (dimensions below d are already fully "free" -- rank r holds -// every value there -- while dimensions >= d are still pinned to r's own coordinate). This -// holds trivially at d == 0 (only slot r itself, u_j == r_j for ALL j) and, vacuously, means -// every rank holds every slot once d reaches K (all dimensions processed). -// -// Each phase d performs (extent_d - 1) ring-relay steps in the +1 direction: at step s, rank r -// forwards the slot batch it received on the PREVIOUS step (or, at s=1, the batch it started -// the phase with) to its +1 neighbor. This is the classic "forward only what you just received" -// ring relay, which avoids redundant retransmission and completes dimension d's expansion in -// exactly extent_d - 1 hops. -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N is each rank's own shard size. -// -// Returns -// ------- -// Schedule -- 1 + sum_d(extent_d - 1) steps. -inline Schedule build_all_gather(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - Schedule sched; - { - Step step0; - for (int r = 0; r < N; ++r) - step0.post.push_back({r, BUF_SEND, 0, BUF_RECV, (size_t)r * m, m, false}); - sched.push_back(std::move(step0)); - } - for (int d = 0; d < K; ++d) { - int E = dims[d]; - for (int s = 1; s <= E - 1; ++s) { - Step step; - for (int r = 0; r < N; ++r) { - std::vector base = coords_of(r, dims); - int dst = neighbor(r, d, +1, dims); - check_edge_or_abort(r, dst, dims); - base[d] = ((base[d] - (s - 1)) % E + E) % E; - for_each_free_combo(d, dims, base, [&](const std::vector& v) { - int vslot = rank_of(v, dims); - size_t off = (size_t)vslot * m; - step.xfers.push_back({r, dst, BUF_RECV, BUF_RECV, off, off, m}); - }); - } - sched.push_back(std::move(step)); - } - } - return sched; -} - -// Builds the schedule for `reduce_scatter`. -// -// DERIVATION NOTE (see also file header): the planning spec's own tentative formula for this -// collective was flagged mid-sentence as unreliable ("hold on, mirror the all_gather invariant -// exactly"). This implementation instead derives the per-step slot set and DIRECTION from a -// time-reversal argument against the (independently correct, spec-supplied) all_gather formula, -// and that derivation was confirmed by an exhaustive brute-force simulation (dims=[8],[2,4], -// [4,2],[2,2,2]) before being encoded here. Summary of the correction: the naive "mirror -// all_gather with the same (r_d - (s-1)) mod E slot formula and +1 direction" places each fully -// reduced slot ONE HOP SHORT of its destination rank (rank k's slot ends up fully summed at -// rank k-1, not rank k). The fix is both a different slot formula AND a different direction: -// sends go in the -1 direction, and at step t the sender's slot index is (r_d + t) mod E, not -// (r_d - (s-1)) mod E. -// -// Algorithm: work happens in BUF_WORK_A (an N-slot accumulator seeded from BUF_SEND at step 0). -// Dimensions are processed in DESCENDING order (K-1 down to 0), matching the spec. -// -// Invariant: at the END of dimension d's phase, rank r holds partial sums ONLY for slots -// {v : v_j == r_j for all j >= d} (dimensions below d are not yet reduced -- rank r's held -// slots still range over every value there -- while dimensions >= d have been fully reduced -// down to r's own coordinate). This is exactly the all_gather invariant with the phase-transition -// direction reversed: all_gather EXPANDS what a rank holds as d increases from 0 to K; -// reduce_scatter CONTRACTS what a rank holds as d decreases from K-1 to 0, so by symmetry it is -// stated as an END-of-phase (rather than start-of-phase) condition. -// -// Correctness of the per-step formula for a single dimension's ring (extent E, direction -1, -// slot index v_d = (r_d + t) mod E at step t = 1..E-1): consider dimension d as an isolated ring -// (the free dimensions jd ride along unchanged, in parallel, for -// every value). For a fixed target slot-index k (a value of v_d), the ring of E nodes must sum -// together each node's local contribution for chunk k, ending up entirely at node k. At reduce -// step t, the node currently entrusted with chunk k's running partial sum is node (k - t) mod E -// (t=1: node k-1 sends its OWN local value for chunk k to node k-2, which adds it in; t=2: node -// k-2, now holding a 2-term partial sum, forwards it to node k-3; ...; t=E-1: node (k - -// (E-1)) mod E == (k+1) mod E, holding an (E-1)-term partial sum -- every node except k itself -// -- forwards it to node k, which adds in its own remaining term to complete the sum of all E -// contributions). Restating "node (k - t) mod E sends chunk k to node (k - t - 1) mod E" from -// the SENDER's own coordinate r = (k - t) mod E gives: sender r sends chunk k = (r + t) mod E to -// receiver (r - 1) mod E == neighbor(r, d, -1). This is exactly the formula used below. -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N, BUF_SEND holds N slots of m bytes each. -// -// Returns -// ------- -// Schedule -- 1 (seed) + sum_d(extent_d - 1) (reduce) + 1 (final copy) steps. -inline Schedule build_reduce_scatter(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - Schedule sched; - { - Step step0; - for (int r = 0; r < N; ++r) - step0.post.push_back({r, BUF_SEND, 0, BUF_WORK_A, 0, (size_t)N * m, false}); - sched.push_back(std::move(step0)); - } - for (int d = K - 1; d >= 0; --d) { - int E = dims[d]; - for (int t = 1; t <= E - 1; ++t) { - Step step; - for (int r = 0; r < N; ++r) { - std::vector cr = coords_of(r, dims); - // Direction is -1 (see derivation above); this is the corrected direction, - // NOT the +1 direction all_gather uses. - int dst = neighbor(r, d, -1, dims); - check_edge_or_abort(r, dst, dims); - std::vector base = cr; - base[d] = ((cr[d] + t) % E + E) % E; - for_each_free_combo(d, dims, base, [&](const std::vector& v) { - int vslot = rank_of(v, dims); - size_t off = (size_t)vslot * m; - // Stage into the receiver's BUF_TMP at the same v*m offset (distinct - // offsets across combos within this step fall out automatically since each - // combo yields a distinct vslot), then a post LocalOp adds it into the - // receiver's running accumulator at that same slot. - step.xfers.push_back({r, dst, BUF_WORK_A, BUF_TMP, off, off, m}); - step.post.push_back({dst, BUF_TMP, off, BUF_WORK_A, off, m, true}); - }); - } - sched.push_back(std::move(step)); - } - } - { - Step stepf; - for (int r = 0; r < N; ++r) - stepf.post.push_back({r, BUF_WORK_A, (size_t)r * m, BUF_RECV, 0, m, false}); - sched.push_back(std::move(stepf)); - } - return sched; -} - -// Builds the schedule for `all_reduce` by composing the (unmodified) reduce_scatter and -// all_gather builders over sub-shards of size m/N, per the spec. -// -// Design: build_reduce_scatter(dims, m) is called with reduce_scatter's OWN "S" parameter set -// to all_reduce's per-rank vector size m (not to S itself). Internally, reduce_scatter then -// computes its own m_rs = m/N == the desired sub-shard size, and its own N*m_rs == m exactly -// matches the size of all_reduce's per-rank input buffer -- so all_reduce's existing BUF_SEND -// content can be fed to reduce_scatter completely unmodified, with no data rearrangement, -// because a flat m-byte vector split into N contiguous m/N-byte pieces is precisely -// reduce_scatter's own "N contiguous slots" input convention. Symmetrically, -// build_all_gather(dims, m) produces an N*(m/N) == m byte output, exactly all_reduce's needed -// result size. -// -// A single bridging LocalOp copies reduce_scatter's own (small, m/N-byte) output out of its -// BUF_RECV into BUF_SEND, where the reused all_gather schedule's own step 0 expects to find its -// input. This reuse is safe because reduce_scatter only ever READS BUF_SEND once, in its own -// step 0; by the time its schedule finishes, BUF_SEND is dead and free to reuse as scratch. -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N is each rank's full input/output vector size. -// -// Returns -// ------- -// Schedule -- reduce_scatter(dims,m)'s steps, then 1 bridging step, then all_gather(dims,m)'s -// steps. -inline Schedule build_all_reduce(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - size_t m = S / (size_t)N; - size_t shard = m / (size_t)N; - - Schedule sched = build_reduce_scatter(dims, m); - { - Step bridge; - for (int r = 0; r < N; ++r) - bridge.post.push_back({r, BUF_RECV, 0, BUF_SEND, 0, shard, false}); - sched.push_back(std::move(bridge)); - } - Schedule ag = build_all_gather(dims, m); - for (auto& step : ag) sched.push_back(std::move(step)); - return sched; -} - -// Builds the schedule for `alltoall` using dimension-ordered minimal routing. -// -// Each of the N*(N-1) non-self chunks (u,v), u != v, starts at rank u (BUF_SEND offset v*c, -// c = m/N) and must reach rank v (BUF_RECV offset u*c). Self chunks (u == v) never move; they -// are resolved by a single step-0 LocalOp per rank. -// -// Routing: dimensions are processed in ascending order. Within dimension d's phase, every chunk -// whose current holder disagrees with its target on coordinate d takes one hop per Step, in the -// direction (+1 or -1) that minimizes remaining distance around that dimension's ring; the -// phase repeats until no chunk needs to move in dimension d (bounded by extent_d - 1 -// iterations, the maximum possible remaining distance, with an explicit abort if that bound is -// ever exceeded -- it should not be, since minimal-direction hops need at most -// floor(extent_d/2) <= extent_d - 1 of them). -// -// In-flight buffer management: a chunk not currently at BUF_SEND or BUF_RECV lives in one of -// two ping-pong work buffers, BUF_WORK_A/BUF_WORK_B, alternating buffers each time it takes a -// hop (this guarantees a chunk's read-from buffer this step always differs from its -// write-to buffer this step, so simple sequential-then-concurrent memory semantics -- -// read-all-then-write-all within a Step -- can never alias a chunk's own old and new copies of -// itself). -// -// Slot allocation within a work buffer is a genuine per-step host-side bump allocator (a -// closed-form offset formula is not safe in general: two chunks that share a source rank and -// happen to have the same destination coordinate on every already-matched dimension travel -// together and can simultaneously occupy the same intermediate rank, so slots must be assigned, -// not computed). New allocations for step t are chosen to avoid every slot occupied at the -// START of step t (including slots about to be vacated this same step, since those are still -// being read from concurrently); old slots are freed only once the step's Xfers have been -// fully built, so freed capacity becomes available starting with the NEXT step, never the -// current one. Total occupancy per (rank, buffer) is asserted <= the buffer's slot capacity -// (2*N slots of c bytes each, matching the 2*m-byte buffer size from buffer_sizes()); the spec -// notes this should stay near N by symmetry for uniform all-to-all, which is also confirmed by -// a max-occupancy check in the throwaway Python simulation used to validate this algorithm -// before writing it here (observed max was well under half the budget for all tested dims). -// -// A chunk that arrives at its destination (all dimensions matched) is routed directly into -// BUF_RECV instead of a work buffer and is marked done, removing it from further consideration. -// -// Parameters -// ---------- -// dims : const std::vector& -// S : size_t -- total collective bytes; m = S/N per rank, c = m/N per chunk. -// -// Returns -// ------- -// Schedule -- 1 (self-chunk) step, followed by one step per routing hop actually taken. -inline Schedule build_alltoall(const std::vector& dims, size_t S) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - size_t c = m / (size_t)N; - Schedule sched; - { - Step step0; - for (int r = 0; r < N; ++r) - step0.post.push_back({r, BUF_SEND, (size_t)r * c, BUF_RECV, (size_t)r * c, c, false}); - sched.push_back(std::move(step0)); - } - - // Host-side (schedule-build-time only) bookkeeping of each in-flight chunk's current - // position. buf: -1 == still/again at BUF_SEND (never true after the first hop, but used - // as the initial state so the first hop's ping-pong toggle lands on BUF_WORK_A); 0 == - // BUF_WORK_A; 1 == BUF_WORK_B. slot is meaningful only when buf >= 0. - struct A2AChunk { - int u, v, holder; - int buf; - int slot; - bool done; - }; - std::vector chunks; - chunks.reserve((size_t)N * (N - 1)); - for (int u = 0; u < N; ++u) - for (int v = 0; v < N; ++v) - if (u != v) chunks.push_back({u, v, u, -1, -1, false}); - - const int CAP_SLOTS = 2 * N; // 2*m bytes / c bytes-per-slot, matching buffer_sizes(). - std::vector, 2>> occupied(N); - for (int r = 0; r < N; ++r) { - occupied[r][0].assign(CAP_SLOTS, 0); - occupied[r][1].assign(CAP_SLOTS, 0); - } - auto alloc_slot = [&](int rank, int buf) -> int { - for (int s = 0; s < CAP_SLOTS; ++s) { - if (!occupied[rank][buf][s]) { - occupied[rank][buf][s] = 1; - return s; - } - } - std::fprintf(stderr, - "%s:%d: alltoall work-buffer overflow at rank %d buf %d (occupancy " - "would exceed 2*m bytes)\n", - __FILE__, __LINE__, rank, buf); - std::abort(); - return -1; // unreachable - }; - - for (int d = 0; d < K; ++d) { - int E = dims[d]; - int iter = 0; - while (true) { - std::vector movers; - for (size_t i = 0; i < chunks.size(); ++i) { - if (chunks[i].done) continue; - std::vector cr = coords_of(chunks[i].holder, dims); - std::vector cv = coords_of(chunks[i].v, dims); - if (cr[d] != cv[d]) movers.push_back(i); - } - if (movers.empty()) break; - ++iter; - if (iter > E - 1) { - std::fprintf(stderr, - "%s:%d: alltoall routing failed to converge in dimension %d " - "within %d iterations\n", - __FILE__, __LINE__, d, E - 1); - std::abort(); - } - - struct Move { - size_t chunk_idx; - int dst_rank; - int new_buf, new_slot; - bool arrives; - }; - std::vector moves; - moves.reserve(movers.size()); - // Pass 1: decide direction/destination and allocate NEW slots against the - // occupancy snapshot as of the start of this step (see design note above: old - // slots are deliberately not freed until pass 3, so a slot being read from this - // step is never handed out as someone else's new landing spot this same step). - for (size_t i : movers) { - A2AChunk& ch = chunks[i]; - std::vector cr = coords_of(ch.holder, dims); - std::vector cv = coords_of(ch.v, dims); - int distp = ((cv[d] - cr[d]) % E + E) % E; - int distm = ((cr[d] - cv[d]) % E + E) % E; - int dirn = (distp <= distm) ? +1 : -1; - int dst = neighbor(ch.holder, d, dirn, dims); - check_edge_or_abort(ch.holder, dst, dims); - std::vector cdst = coords_of(dst, dims); - bool arrives = true; - for (int j = 0; j < K; ++j) - if (cdst[j] != cv[j]) { arrives = false; break; } - int new_buf = -1, new_slot = -1; - if (!arrives) { - new_buf = (ch.buf == 0) ? 1 : 0; // ping-pong: SEND or B -> A; A -> B. - new_slot = alloc_slot(dst, new_buf); - } - moves.push_back({i, dst, new_buf, new_slot, arrives}); - } - // Pass 2: emit Xfers from each chunk's OLD position to its NEW position. - Step step; - for (const Move& mv : moves) { - A2AChunk& ch = chunks[mv.chunk_idx]; - int src_buf = (ch.buf < 0) ? (int)BUF_SEND : (ch.buf == 0 ? (int)BUF_WORK_A : (int)BUF_WORK_B); - size_t src_off = (ch.buf < 0) ? (size_t)ch.v * c : (size_t)ch.slot * c; - int dst_buf; - size_t dst_off; - if (mv.arrives) { - dst_buf = BUF_RECV; - dst_off = (size_t)ch.u * c; - } else { - dst_buf = (mv.new_buf == 0) ? (int)BUF_WORK_A : (int)BUF_WORK_B; - dst_off = (size_t)mv.new_slot * c; - } - step.xfers.push_back({ch.holder, mv.dst_rank, src_buf, dst_buf, src_off, dst_off, c}); - } - // Pass 3: now that this step's Xfers are fully built, free vacated slots and - // update chunk bookkeeping for the next iteration. - for (const Move& mv : moves) { - A2AChunk& ch = chunks[mv.chunk_idx]; - if (ch.buf >= 0) occupied[ch.holder][ch.buf][ch.slot] = 0; - ch.holder = mv.dst_rank; - if (mv.arrives) { - ch.done = true; - ch.buf = -1; - ch.slot = -1; - } else { - ch.buf = mv.new_buf; - ch.slot = mv.new_slot; - } - } - sched.push_back(std::move(step)); - } - } - - for (const A2AChunk& ch : chunks) { - if (!ch.done) { - std::fprintf(stderr, - "%s:%d: alltoall chunk (u=%d,v=%d) failed to reach its destination\n", - __FILE__, __LINE__, ch.u, ch.v); - std::abort(); - } - } - return sched; -} - -// ===================================================================================== -// Intra-step aliasing validation (shared, pure host code -- runs as part of build_schedule()'s -// final validation pass, so it applies identically to whichever executor -- simulator or -// NCCL/CUDA -- ends up running the assembled Schedule). -// ===================================================================================== -// -// Design/WHY (see also run_schedule()'s Notes, in both the NCCL and simulator branches below, -// for the two executors' actual ordering guarantees this assertion is checked against): the -// host-memory simulator's run_schedule() gives every Step's Xfers STRONGER semantics than the -// NCCL/CUDA executor actually provides -- it snapshots every Xfer's source data before writing -// ANY Xfer's destination (see that function's own "Design decision" comment), so a schedule with -// overlapping src/dst ranges within one Step would silently produce a correct result there even -// though it would NOT on real hardware. The NCCL executor instead relies only on: (1) all of a -// Step's Xfers issued together inside one ncclGroupStart/End, with no ordering guarantee among -// DIFFERENT Xfers of that group beyond NCCL's own send/recv rendezvous (which pairs a specific -// send with its specific matching recv -- it says nothing about two unrelated Xfers of the same -// Step racing each other), and (2) per-rank CUDA stream order, which guarantees only that ops -// enqueued LATER on one rank's stream see the effects of ops enqueued EARLIER on that SAME -// stream -- in particular, a Step's post LocalOps (always enqueued, on every rank's stream, -// strictly after that Step's Xfers -- see run_schedule()) are guaranteed to see every effect of -// that Step's own Xfers, but no other cross-operation ordering is guaranteed by construction. -// -// This function makes that gap an assertion instead of a latent, hard-to-reproduce bug: every -// schedule this file ever builds is validated here to never depend on any same-step read/write -// ordering stronger than "a post-op reads a range a same-step Xfer just wrote" -- the one -// relationship the NCCL executor's fixed enqueue order (a Step's Xfers, then that Step's -// post-ops) always provides "for free", with no host synchronization required. Any OTHER -// same-step read/write overlap on the same (rank, buffer) -- e.g. one Xfer's source overlapping -// another Xfer's destination, or a post-op's destination overlapping another post-op's source -- -// would only be safe under the simulator's stronger snapshot semantics, and must never occur. - -// One (rank, buffer) byte range read or written by one Xfer or LocalOp within a single Step, used -// only by check_step_aliasing_or_abort() below. -struct _ByteRange { - int rank; - int buf; - size_t begin, end; // half-open [begin, end), in bytes. - bool is_post; // true if this range comes from a LocalOp (post); false if from an Xfer. -}; - -// Returns whether two half-open byte ranges overlap. -inline bool _ranges_overlap(size_t a_begin, size_t a_end, size_t b_begin, size_t b_end) { - return a_begin < b_end && b_begin < a_end; -} - -// Validates one Step against the aliasing rule described above; aborts with a diagnostic naming -// the step index and both offending ranges on the first violation found. -// -// Parameters -// ---------- -// step_idx : size_t -- index of `step` within its Schedule (diagnostics only). -// step : const Step& -// -// Notes -// ----- -// The single allowed exception -- a post-op's SOURCE range overlapping an Xfer's DESTINATION -// range, both of the same Step -- is exactly the reduce_scatter reduce-phase pattern (an Xfer -// lands a value in BUF_TMP, and that same Step's post-op immediately adds it out of BUF_TMP); -// see run_schedule()'s Notes for why that ordering (post-ops always after that Step's Xfers) is -// safe on both executors. Every other overlap combination aborts. -inline void check_step_aliasing_or_abort(size_t step_idx, const Step& step) { - std::vector<_ByteRange> reads, writes; - for (const Xfer& x : step.xfers) { - reads.push_back({x.src, x.src_buf, x.src_off, x.src_off + x.bytes, false}); - writes.push_back({x.dst, x.dst_buf, x.dst_off, x.dst_off + x.bytes, false}); - } - for (const LocalOp& op : step.post) { - reads.push_back({op.rank, op.src_buf, op.src_off, op.src_off + op.bytes, true}); - writes.push_back({op.rank, op.dst_buf, op.dst_off, op.dst_off + op.bytes, true}); - } - for (const _ByteRange& r : reads) { - for (const _ByteRange& w : writes) { - if (r.rank != w.rank || r.buf != w.buf) continue; - if (!_ranges_overlap(r.begin, r.end, w.begin, w.end)) continue; - // The one guaranteed-safe relationship: a post-op reading exactly what a same-step - // Xfer just wrote (see this function's Notes above and run_schedule()'s Notes). - if (r.is_post && !w.is_post) continue; - std::fprintf( - stderr, - "%s:%d: INTRA-STEP ALIAS ASSERTION FAILED at step %zu: rank %d buf %s read " - "range [%zu,%zu) (from %s) overlaps write range [%zu,%zu) (from %s); this " - "schedule relies on same-step read/write ordering the NCCL executor does not " - "guarantee (see check_step_aliasing_or_abort()'s Notes)\n", - __FILE__, __LINE__, step_idx, r.rank, buf_name(r.buf), r.begin, r.end, - r.is_post ? "a post-op" : "an Xfer", w.begin, w.end, - w.is_post ? "a post-op" : "an Xfer"); - std::abort(); - } - } -} - -// Dispatches to the appropriate per-collective builder and then re-validates every Xfer in the -// assembled schedule against the torus-edge guarantee, and every Step against the intra-step -// aliasing rule above. -// -// Notes -// ----- -// Every builder above already calls check_edge_or_abort() at each Xfer's construction site, -// which fails fastest and with the most local context. The bulk re-scan here is a deliberate -// belt-and-suspenders duplication: it makes build_schedule() itself -- not just its helpers -- -// the literal authority for the scientific guarantee described in the file header, matching the -// spec's requirement that "build_schedule asserts is_torus_neighbor(...) for every Xfer". The -// aliasing pass (check_step_aliasing_or_abort()) has no earlier per-Xfer equivalent -- it is -// inherently a whole-Step check -- so build_schedule() is the only place it can run. -// -// Parameters -// ---------- -// c : Collective -// dims : const std::vector& -- torus extents. -// S : size_t -- total collective bytes for this sweep point. -// -// Returns -// ------- -// Schedule -- fully built, edge-validated, and aliasing-validated. -inline Schedule build_schedule(Collective c, const std::vector& dims, size_t S) { - Schedule sched; - switch (c) { - case Collective::SENDRECV: sched = build_sendrecv(dims, S); break; - case Collective::BROADCAST: sched = build_broadcast(dims, S); break; - case Collective::ALL_GATHER: sched = build_all_gather(dims, S); break; - case Collective::REDUCE_SCATTER: sched = build_reduce_scatter(dims, S); break; - case Collective::ALL_REDUCE: sched = build_all_reduce(dims, S); break; - case Collective::ALLTOALL: sched = build_alltoall(dims, S); break; - } - for (size_t i = 0; i < sched.size(); ++i) { - for (const Xfer& x : sched[i].xfers) check_edge_or_abort(x.src, x.dst, dims); - check_step_aliasing_or_abort(i, sched[i]); - } - return sched; -} - -// ===================================================================================== -// --check data pattern: shared, pure-host generation and verification logic -// ===================================================================================== -// -// The --check input convention is uniform across all six collectives: every rank's BUF_SEND -// buffer (whatever its collective-specific size happens to be) is filled with f(rank, i) at -// flat float index i = 0 .. (buffer_bytes/4 - 1). This single convention subsumes every -// per-collective fill rule described in the spec (e.g. reduce_scatter's "rank u's BUF_SEND slot -// v holds f(u, v*elems_per_slot + i)" and alltoall's analogous per-chunk rule are both just this -// same flat-buffer fill, since a slot/chunk is by construction a contiguous sub-range of the -// flat buffer and v*elems_per_slot+i (or v*celems+i) is exactly that sub-range's flat index). -// -// Because f() is a pure, deterministic function of (source rank, index), verification never -// needs to read another rank's actual buffer contents: for every element of a rank's BUF_RECV, -// we know analytically which (source rank, source flat index) it is supposed to equal (or, for -// the reducing collectives, which SET of them it is supposed to sum) and simply recompute f() -// directly. This is implemented once, in verify_collective() below, and reused unmodified by -// both executors. - -// Deterministic --check fill value. -// -// Parameters -// ---------- -// r : int -- source rank. -// i : long long -- flat float index within that rank's buffer; must be >= 0 in practice (the -// modulo-97 reduction below is defensive against negative input but this file never -// constructs a negative index). -// -// Returns -// ------- -// float -- r*100 + (i mod 97). Exact in fp32 (see verify_collective() notes on why summing up -// to N such values remains exactly representable for the N, dims this benchmark targets). -inline float f_pattern(int r, long long i) { - long long im = i % 97; - if (im < 0) im += 97; - return (float)(r * 100 + (int)im); -} - -// Generates the full --check fill pattern for one rank's BUF_SEND buffer. -// -// Parameters -// ---------- -// rank : int -// nbytes : size_t -- BUF_SEND size for this rank (from buffer_sizes()); must be a multiple of -// sizeof(float) (guaranteed by the S % (4*N*N) == 0 sweep-level precondition). -// -// Returns -// ------- -// std::vector -- nbytes/4 floats, out[i] == f_pattern(rank, i). -inline std::vector gen_pattern(int rank, size_t nbytes) { - size_t nf = nbytes / sizeof(float); - std::vector out(nf); - for (size_t i = 0; i < nf; ++i) out[i] = f_pattern(rank, (long long)i); - return out; -} - -// Diagnostic record for the first mismatch found by verify_collective(), used to format the -// "# CHECK ... FAIL (first mismatch: ...)" line. -struct MismatchInfo { - int rank = -1; - int buf = BUF_RECV; - long long idx = -1; - float want = 0.f; - float got = 0.f; -}; - -// Verifies one rank's BUF_RECV contents against the analytically-known-correct result for the -// given collective, per the check rules in the spec (all reducible, per the discussion above, -// to recomputing f_pattern() directly rather than reading other ranks' data). -// -// Parameters -// ---------- -// c : Collective -// dims : const std::vector& -// S : size_t -- total collective bytes for this sweep point. -// rank : int -- the rank whose BUF_RECV is being checked. -// recv : const std::vector& -- that rank's full BUF_RECV contents, already copied to -// host memory by the caller (trivial for the simulator; a cudaMemcpy D2H for the NCCL -// build). -// mm : MismatchInfo& -- filled in on the first mismatch found; unmodified if this returns true. -// -// Returns -// ------- -// bool -- true iff every checked element matched exactly. -// -// Notes -// ----- -// Comparisons use exact float equality. This is intentional and safe here, not a bug: every -// f_pattern() value is a small non-negative integer (r*100 + i%97, comfortably under 2^24 for -// the rank counts and indices this benchmark exercises), and summing up to N such values (N <= -// a few hundred in any realistic torus) never leaves the range of exactly-representable -// integers in fp32 at any intermediate step -- so no rounding ever occurs, in the executor's -// float accumulation OR in this function's own reference summation, and exact comparison is -// mathematically justified rather than merely convenient. -inline bool verify_collective(Collective c, const std::vector& dims, size_t S, int rank, - const std::vector& recv, MismatchInfo& mm) { - int N = num_ranks(dims); - int K = (int)dims.size(); - size_t m = S / (size_t)N; - - auto check_eq = [&](size_t idx, float want, float got) -> bool { - if (got != want) { - mm.rank = rank; - mm.buf = BUF_RECV; - mm.idx = (long long)idx; - mm.want = want; - mm.got = got; - return false; - } - return true; - }; - - switch (c) { - case Collective::SENDRECV: { - size_t nf = m / sizeof(float); - for (int d = 0; d < K; ++d) { - int src = neighbor(rank, d, -1, dims); - for (size_t i = 0; i < nf; ++i) { - size_t idx = (size_t)d * nf + i; - if (!check_eq(idx, f_pattern(src, (long long)i), recv[idx])) return false; - } - } - return true; - } - case Collective::BROADCAST: { - size_t nf = m / sizeof(float); - for (size_t i = 0; i < nf; ++i) - if (!check_eq(i, f_pattern(0, (long long)i), recv[i])) return false; - return true; - } - case Collective::ALL_GATHER: { - size_t nf = m / sizeof(float); - for (int u = 0; u < N; ++u) { - for (size_t i = 0; i < nf; ++i) { - size_t idx = (size_t)u * nf + i; - if (!check_eq(idx, f_pattern(u, (long long)i), recv[idx])) return false; - } - } - return true; - } - case Collective::REDUCE_SCATTER: { - size_t elems_per_slot = m / sizeof(float); - for (size_t i = 0; i < elems_per_slot; ++i) { - float want = 0.f; - for (int u = 0; u < N; ++u) - want += f_pattern(u, (long long)((size_t)rank * elems_per_slot + i)); - if (!check_eq(i, want, recv[i])) return false; - } - return true; - } - case Collective::ALL_REDUCE: { - size_t nf = m / sizeof(float); - for (size_t i = 0; i < nf; ++i) { - float want = 0.f; - for (int u = 0; u < N; ++u) want += f_pattern(u, (long long)i); - if (!check_eq(i, want, recv[i])) return false; - } - return true; - } - case Collective::ALLTOALL: { - size_t c_bytes = m / (size_t)N; - size_t celems = c_bytes / sizeof(float); - for (int u = 0; u < N; ++u) { - for (size_t i = 0; i < celems; ++i) { - size_t idx = (size_t)u * celems + i; - float want = f_pattern(u, (long long)((size_t)rank * celems + i)); - if (!check_eq(idx, want, recv[idx])) return false; - } - } - return true; - } - } - return true; -} - -// Prints the single required "# CHECK ..." line for one sweep size, per the exact output -// format in the spec. -inline void print_check_line(Collective c, size_t S, bool pass, const MismatchInfo& mm) { - if (pass) { - std::printf("# CHECK collective=%s S=%zu: PASS\n", collective_name(c), S); - } else { - std::printf( - "# CHECK collective=%s S=%zu: FAIL (first mismatch: rank=%d buf=%s idx=%lld " - "want=%g got=%g)\n", - collective_name(c), S, mm.rank, buf_name(mm.buf), mm.idx, (double)mm.want, - (double)mm.got); - } -} - -// ===================================================================================== -// CLI helpers -// ===================================================================================== - -// Parses a --dims argument of the form "2x2x2", "8", or "2x4" into per-dimension extents. -// -// Parameters -// ---------- -// s : const std::string& -- 'x'-separated positive integers. -// -// Returns -// ------- -// std::vector -- one entry per dimension, each > 0. Empty on any parse failure (missing -// token, non-digit character, or non-positive value), which callers treat as a CLI error. -inline std::vector parse_dims(const std::string& s) { - std::vector out; - size_t pos = 0; - while (pos <= s.size()) { - size_t next = s.find('x', pos); - std::string tok = (next == std::string::npos) ? s.substr(pos) : s.substr(pos, next - pos); - if (tok.empty()) return {}; - for (char ch : tok) - if (!std::isdigit((unsigned char)ch)) return {}; - int val = std::atoi(tok.c_str()); - if (val <= 0) return {}; - out.push_back(val); - if (next == std::string::npos) break; - pos = next + 1; - } - return out; -} - -// ===================================================================================== -// Executors: identical function signatures, divergent implementations. -// -// Everything above this point is pure host C++ with no CUDA dependency and is shared, -// unmodified, by both builds. Everything below is fenced per the spec: CUDA/NCCL code lives -// under `#ifndef TORUS_SIM`; the host-memory simulator (this environment's test vehicle, since -// no nvcc/GPU is available here) lives in the `#else` branch. Both branches implement the exact -// same set of function names and signatures -- alloc_buffers, free_buffers, sync_all_devices, -// fill_input_pattern, run_schedule, run_check, global_teardown -- so main() below is written -// once and never itself needs an #ifdef. -// ===================================================================================== - -#ifndef TORUS_SIM -// -------------------------------------------------------------------------------- -// NCCL/CUDA executor. Requires nvcc + a CUDA toolkit + NCCL; NOT buildable or runnable in this -// development environment (no nvcc/CUDA installed here -- see project report). Written to -// mirror every simulator-side function signature exactly, per the spec, so this branch never -// references anything sim-only. -// -------------------------------------------------------------------------------- -#include -#include - -// CUDA error-check macro: prints file:line and aborts on any non-success cudaError_t. -#define CUDA_CHECK(call) \ - do { \ - cudaError_t _e = (call); \ - if (_e != cudaSuccess) { \ - std::fprintf(stderr, "%s:%d: CUDA error: %s\n", __FILE__, __LINE__, \ - cudaGetErrorString(_e)); \ - std::abort(); \ - } \ - } while (0) - -// NCCL error-check macro: prints file:line and aborts on any non-success ncclResult_t. -#define NCCL_CHECK(call) \ - do { \ - ncclResult_t _r = (call); \ - if (_r != ncclSuccess) { \ - std::fprintf(stderr, "%s:%d: NCCL error: %s\n", __FILE__, __LINE__, \ - ncclGetErrorString(_r)); \ - std::abort(); \ - } \ - } while (0) - -// Elementwise in-place float accumulate: dst[i] += src[i] for i in [0, n). Used to implement -// LocalOp with add == true on the device (the host-side simulator does the equivalent with a -// plain loop; see the #else branch below). -__global__ void add_inplace(float* dst, const float* src, size_t n) { - size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) dst[i] += src[i]; -} - -static int g_N = 0; -static std::vector> g_dev_bufs; -static std::array g_buf_sizes{}; -static ncclComm_t* g_comms = nullptr; -static cudaStream_t* g_streams = nullptr; - -// Allocates per-(rank,buffer) device storage for the upcoming sweep size, and lazily -// initializes the (persistent, reused-across-sizes) NCCL communicators and per-device streams -// on the first call. -// -// Parameters -// ---------- -// N : int -- device/rank count (must match every subsequent call until the matching -// free_buffers()). -// sizes : const size_t[NUM_BUFS] -- from buffer_sizes(); a 0 entry allocates nothing (null ptr). -inline void alloc_buffers(int N, const size_t sizes[NUM_BUFS]) { - g_N = N; - for (int b = 0; b < NUM_BUFS; ++b) g_buf_sizes[b] = sizes[b]; - if (!g_comms) { - g_comms = new ncclComm_t[N]; - std::vector devs(N); - for (int i = 0; i < N; ++i) devs[i] = i; - NCCL_CHECK(ncclCommInitAll(g_comms, N, devs.data())); - g_streams = new cudaStream_t[N]; - for (int r = 0; r < N; ++r) { - CUDA_CHECK(cudaSetDevice(r)); - CUDA_CHECK(cudaStreamCreate(&g_streams[r])); - } - } - g_dev_bufs.assign(N, {nullptr, nullptr, nullptr, nullptr, nullptr}); - for (int r = 0; r < N; ++r) { - CUDA_CHECK(cudaSetDevice(r)); - for (int b = 0; b < NUM_BUFS; ++b) { - if (sizes[b] > 0) CUDA_CHECK(cudaMalloc(&g_dev_bufs[r][b], sizes[b])); - } - } -} - -// Frees the per-(rank,buffer) device storage allocated by the matching alloc_buffers() call. -// Communicators/streams are intentionally NOT destroyed here (they are reused across sweep -// sizes); see global_teardown() for final cleanup. -inline void free_buffers(int N) { - for (int r = 0; r < N; ++r) { - CUDA_CHECK(cudaSetDevice(r)); - for (int b = 0; b < NUM_BUFS; ++b) - if (g_dev_bufs[r][b]) CUDA_CHECK(cudaFree(g_dev_bufs[r][b])); - } - g_dev_bufs.clear(); -} - -// Blocks the host until every device has completed all work previously enqueued on its stream. -inline void sync_all_devices() { - for (int r = 0; r < g_N; ++r) { - CUDA_CHECK(cudaSetDevice(r)); - CUDA_CHECK(cudaDeviceSynchronize()); - } -} - -// Fills every rank's BUF_SEND with the deterministic --check pattern (see gen_pattern()), -// H2D-copying a host-generated array rather than launching a fill kernel (simplest correct -// option given the CUDA path cannot be tested locally; avoids a second untestable kernel). -inline void fill_input_pattern(int N) { - size_t send_bytes = g_buf_sizes[BUF_SEND]; - for (int r = 0; r < N; ++r) { - std::vector pat = gen_pattern(r, send_bytes); - CUDA_CHECK(cudaSetDevice(r)); - CUDA_CHECK(cudaMemcpy(g_dev_bufs[r][BUF_SEND], pat.data(), send_bytes, - cudaMemcpyHostToDevice)); - } -} - -// Executes one full pass of `sched` across all N devices. -// -// Notes -// ----- -// Correctness argument for the (deliberate) ABSENCE of any per-step host synchronization here -// (see the file header's SCHEDULE-AS-DATA section for the two-executor contract this satisfies): -// -// - RANK-LOCAL ordering is preserved by CUDA stream order alone, with no host round-trip -// needed. Every one of a given rank r's operations -- its ncclSend/ncclRecv calls (issued on -// g_streams[r] via g_comms[r]), its cudaMemcpyAsync D2D post-ops, and its add_inplace kernel -// launches -- are ALL enqueued on that same single stream g_streams[r], in the exact order -// this function enqueues them (every Step's Xfers, in loop order, followed by that Step's -// post LocalOps, in loop order, before moving to the next Step). CUDA guarantees operations -// enqueued on one stream execute in that enqueue order; a later-enqueued op on a stream is -// therefore guaranteed to see the effects of every earlier op on that SAME stream without -// any explicit sync between them. This is exactly what build_schedule()'s intra-step alias -// assertion (see that function) verifies is sufficient: it never lets a schedule reach this -// executor if it would require a per-rank read/write ordering stronger than "post-op reads -// what an Xfer of the same step already wrote" -- precisely the one relationship stream -// order already provides here, for free. -// - CROSS-RANK ordering (rank A's send must be matched by rank B's matching recv before either -// side's dependent work proceeds) is enforced by NCCL itself, not by any host synchronization -// this function performs: every Step's Xfers are issued inside one ncclGroupStart/End, and -// NCCL's own send/recv rendezvous protocol is what guarantees a recv only completes (on its -// own stream) once its matching send has actually transferred the data -- that handshake is -// GPU-side, asynchronous, and requires no host cudaStreamSynchronize call to be correct. -// -// Given both of the above, a host round-trip is not needed between every Step, nor between a -// Step's Xfers and its post LocalOps -- it is needed exactly ONCE per call to this function, to -// give the HOST a defined point at which every device's work for this entire schedule execution -// is known to have completed (callers that need to read results back, e.g. run_check() via -// cudaMemcpy, or that are timing this call, already provide that host sync themselves -- -// sync_all_devices() in main()'s check/warmup/timed-loop call sites -- but performing it once -// here too keeps this function's own postcondition self-contained rather than relying on every -// caller to remember to do it). This drops per-iteration host-sync round-trips from ~2x the -// number of Steps (the pre-fix per-step synchronize-after-Xfers + -// synchronize-after-post-LocalOps pattern) to exactly 1, with no change to the timed loop's -// measured semantics (main()'s warmup/timed loops already sync once before/after the whole w+n -// iteration count, not between individual run_schedule() calls). -inline void run_schedule(const Schedule& sched, int N) { - for (const Step& step : sched) { - ncclGroupStart(); - for (const Xfer& x : step.xfers) { - char* sptr = (char*)g_dev_bufs[x.src][x.src_buf] + x.src_off; - char* dptr = (char*)g_dev_bufs[x.dst][x.dst_buf] + x.dst_off; - NCCL_CHECK(ncclSend(sptr, x.bytes, ncclChar, x.dst, g_comms[x.src], g_streams[x.src])); - NCCL_CHECK(ncclRecv(dptr, x.bytes, ncclChar, x.src, g_comms[x.dst], g_streams[x.dst])); - } - ncclGroupEnd(); - for (const LocalOp& op : step.post) { - CUDA_CHECK(cudaSetDevice(op.rank)); - char* sptr = (char*)g_dev_bufs[op.rank][op.src_buf] + op.src_off; - char* dptr = (char*)g_dev_bufs[op.rank][op.dst_buf] + op.dst_off; - if (!op.add) { - CUDA_CHECK(cudaMemcpyAsync(dptr, sptr, op.bytes, cudaMemcpyDeviceToDevice, - g_streams[op.rank])); - } else { - size_t nf = op.bytes / sizeof(float); - int threads = 256; - int blocks = (int)((nf + (size_t)threads - 1) / (size_t)threads); - add_inplace<<>>((float*)dptr, - (const float*)sptr, nf); - } - } - } - // Host synchronization happens ONCE per schedule execution, here, after the last Step -- - // see the Notes above for why nothing between Steps (or between a Step's Xfers and its - // post-ops) needs it. - for (int r = 0; r < N; ++r) { - CUDA_CHECK(cudaSetDevice(r)); - CUDA_CHECK(cudaStreamSynchronize(g_streams[r])); - } -} - -// Runs the --check verification for every rank, D2H-copying each rank's BUF_RECV before -// delegating to the shared verify_collective(). Prints the single required "# CHECK ..." line. -// -// Returns -// ------- -// bool -- true iff every rank's BUF_RECV matched the expected result. -inline bool run_check(Collective c, const std::vector& dims, size_t S, int N) { - bool overall_pass = true; - MismatchInfo first_mm; - for (int r = 0; r < N; ++r) { - size_t nbytes = g_buf_sizes[BUF_RECV]; - size_t nf = nbytes / sizeof(float); - std::vector recv(nf); - CUDA_CHECK(cudaSetDevice(r)); - CUDA_CHECK(cudaMemcpy(recv.data(), g_dev_bufs[r][BUF_RECV], nbytes, cudaMemcpyDeviceToHost)); - MismatchInfo mm; - bool ok = verify_collective(c, dims, S, r, recv, mm); - if (!ok && overall_pass) { - overall_pass = false; - first_mm = mm; - } - } - print_check_line(c, S, overall_pass, first_mm); - return overall_pass; -} - -// Destroys the persistent NCCL communicators and CUDA streams created lazily by the first -// alloc_buffers() call. Safe to call even if alloc_buffers() was never called. -inline void global_teardown() { - if (g_comms) { - for (int r = 0; r < g_N; ++r) ncclCommDestroy(g_comms[r]); - delete[] g_comms; - g_comms = nullptr; - } - if (g_streams) { - for (int r = 0; r < g_N; ++r) { - cudaSetDevice(r); - cudaStreamDestroy(g_streams[r]); - } - delete[] g_streams; - g_streams = nullptr; - } -} - -#else -// -------------------------------------------------------------------------------- -// Host-memory simulator executor (TORUS_SIM). No CUDA/NCCL dependency whatsoever -- this is -// the local, GPU-free test vehicle exercised by the acceptance criteria in this environment. -// Timings produced here are, per the spec, not scientifically meaningful (there is no real -// interconnect being modeled); the schedule-building, edge-assertion, and per-collective -// correctness logic under test is identical to what the NCCL branch would execute. -// -------------------------------------------------------------------------------- - -static std::vector, NUM_BUFS>> g_host_bufs; - -// Allocates zero-initialized per-(rank,buffer) host storage for the upcoming sweep size. -// -// Parameters -// ---------- -// N : int -- rank count. -// sizes : const size_t[NUM_BUFS] -- from buffer_sizes(). -inline void alloc_buffers(int N, const size_t sizes[NUM_BUFS]) { - g_host_bufs.assign((size_t)N, std::array, NUM_BUFS>{}); - for (int r = 0; r < N; ++r) - for (int b = 0; b < NUM_BUFS; ++b) g_host_bufs[r][b].assign(sizes[b], 0); -} - -// Releases the host storage allocated by the matching alloc_buffers() call. -inline void free_buffers(int N) { - (void)N; - g_host_bufs.clear(); -} - -// No-op: the simulator is single-threaded host code, so every operation is already -// synchronous by construction. Present only so main()'s driver loop can call the same function -// name in both builds. -inline void sync_all_devices() {} - -// Fills every rank's BUF_SEND with the deterministic --check pattern (see gen_pattern()). -inline void fill_input_pattern(int N) { - for (int r = 0; r < N; ++r) { - std::vector pat = gen_pattern(r, g_host_bufs[r][BUF_SEND].size()); - std::memcpy(g_host_bufs[r][BUF_SEND].data(), pat.data(), pat.size() * sizeof(float)); - } -} - -// Executes one full pass of `sched` over host memory. -// -// Design decision: within a Step, ALL Xfer sources are first snapshotted into temporary -// buffers, and only then are all destinations written. This makes the "all xfers execute -// concurrently, reading pre-step state" semantics of Step literally true regardless of -// iteration order or any potential (believed absent, but not asserted) offset aliasing between -// a step's reads and writes -- a small, cheap robustness margin given how load-bearing exact -// schedule semantics are for this benchmark's scientific validity. -inline void run_schedule(const Schedule& sched, int N) { - (void)N; - for (const Step& step : sched) { - std::vector> staged(step.xfers.size()); - for (size_t i = 0; i < step.xfers.size(); ++i) { - const Xfer& x = step.xfers[i]; - const std::vector& src = g_host_bufs[x.src][x.src_buf]; - staged[i].assign(src.begin() + (long)x.src_off, src.begin() + (long)(x.src_off + x.bytes)); - } - for (size_t i = 0; i < step.xfers.size(); ++i) { - const Xfer& x = step.xfers[i]; - std::memcpy(g_host_bufs[x.dst][x.dst_buf].data() + x.dst_off, staged[i].data(), x.bytes); - } - for (const LocalOp& op : step.post) { - if (!op.add) { - std::memcpy(g_host_bufs[op.rank][op.dst_buf].data() + op.dst_off, - g_host_bufs[op.rank][op.src_buf].data() + op.src_off, op.bytes); - } else { - float* dst = reinterpret_cast(g_host_bufs[op.rank][op.dst_buf].data() + op.dst_off); - const float* src = reinterpret_cast(g_host_bufs[op.rank][op.src_buf].data() + op.src_off); - size_t nf = op.bytes / sizeof(float); - for (size_t i = 0; i < nf; ++i) dst[i] += src[i]; - } - } - } -} - -// Runs the --check verification for every rank directly against host memory (no device copy -// needed) and prints the single required "# CHECK ..." line. -// -// Returns -// ------- -// bool -- true iff every rank's BUF_RECV matched the expected result. -inline bool run_check(Collective c, const std::vector& dims, size_t S, int N) { - bool overall_pass = true; - MismatchInfo first_mm; - for (int r = 0; r < N; ++r) { - const std::vector& raw = g_host_bufs[r][BUF_RECV]; - size_t nf = raw.size() / sizeof(float); - std::vector recv(nf); - std::memcpy(recv.data(), raw.data(), raw.size()); - MismatchInfo mm; - bool ok = verify_collective(c, dims, S, r, recv, mm); - if (!ok && overall_pass) { - overall_pass = false; - first_mm = mm; - } - } - print_check_line(c, S, overall_pass, first_mm); - return overall_pass; -} - -// No-op in the simulator: there is no persistent device/communicator state to tear down. -inline void global_teardown() {} - -#endif // TORUS_SIM - -// ===================================================================================== -// main(): CLI parsing, sweep loop, and output. Shared verbatim by both builds -- everything it -// calls (alloc_buffers, free_buffers, fill_input_pattern, run_schedule, sync_all_devices, -// run_check, global_teardown, build_schedule, buffer_sizes) has an identical signature in both -// the CUDA and simulator branches above, so no #ifdef is needed here at all. -// ===================================================================================== - -int main(int argc, char** argv) { - std::string collective_str; - std::string dims_str; - long long b = -1, e = -1; - long long f = 2; - long long w = 5; - long long n = 20; - bool check = false; - - for (int i = 1; i < argc; ++i) { - std::string a = argv[i]; - auto need_val = [&](const char* flag) -> std::string { - if (i + 1 >= argc) { - std::fprintf(stderr, "torus_bench: missing value for %s\n", flag); - std::exit(2); - } - return std::string(argv[++i]); - }; - if (a == "--collective") collective_str = need_val("--collective"); - else if (a == "--dims") dims_str = need_val("--dims"); - else if (a == "-b") b = std::atoll(need_val("-b").c_str()); - else if (a == "-e") e = std::atoll(need_val("-e").c_str()); - else if (a == "-f") f = std::atoll(need_val("-f").c_str()); - else if (a == "-w") w = std::atoll(need_val("-w").c_str()); - else if (a == "-n") n = std::atoll(need_val("-n").c_str()); - else if (a == "--check") check = true; - else { - std::fprintf(stderr, "torus_bench: unknown argument '%s'\n", a.c_str()); - return 2; - } - } - - if (collective_str.empty() || dims_str.empty() || b <= 0 || e <= 0) { - std::fprintf(stderr, - "usage: torus_bench --collective {all_reduce,all_gather,reduce_scatter," - "alltoall,broadcast,sendrecv} --dims -b " - "-e [-f 2] [-w 5] [-n 20] [--check]\n"); - return 2; - } - if (f <= 1) { - std::fprintf(stderr, "torus_bench: -f must be >= 2 (got %lld)\n", f); - return 2; - } - - Collective c; - if (!parse_collective(collective_str, c)) { - std::fprintf(stderr, "torus_bench: unknown --collective '%s'\n", collective_str.c_str()); - return 2; - } - std::vector dims = parse_dims(dims_str); - if (dims.empty()) { - std::fprintf(stderr, "torus_bench: invalid --dims '%s'\n", dims_str.c_str()); - return 2; - } - int N = num_ranks(dims); - -#ifdef TORUS_SIM - std::printf("# torus_bench [SIMULATOR build -- host memory only, no GPU/NCCL]\n"); -#else - std::printf("# torus_bench [NCCL/CUDA build]\n"); -#endif - std::printf( - "# collective=%s dims=%s N=%d range=[%lld,%lld] factor=%lld warmup=%lld iters=%lld " - "check=%d\n", - collective_name(c), dims_str.c_str(), N, b, e, f, w, n, check ? 1 : 0); - - bool any_check_failure = false; - for (long long S = b; S <= e; S *= f) { - size_t divisor = 4ull * (size_t)N * (size_t)N; - if ((size_t)S % divisor != 0) { - std::printf( - "# SKIP S=%lld not divisible by 4*N*N=%zu (N=%d): shard/chunk boundaries " - "would not align to whole floats\n", - S, divisor, N); - continue; - } - - Schedule sched = build_schedule(c, dims, (size_t)S); - std::array sizes = buffer_sizes(c, dims, (size_t)S); - alloc_buffers(N, sizes.data()); - - char check_char = '-'; - if (check) { - fill_input_pattern(N); - run_schedule(sched, N); - sync_all_devices(); - bool pass = run_check(c, dims, (size_t)S, N); - if (!pass) any_check_failure = true; - check_char = pass ? '1' : '0'; - } - - for (long long it = 0; it < w; ++it) run_schedule(sched, N); - sync_all_devices(); - auto t0 = std::chrono::steady_clock::now(); - for (long long it = 0; it < n; ++it) run_schedule(sched, N); - sync_all_devices(); - auto t1 = std::chrono::steady_clock::now(); - double total_us = std::chrono::duration(t1 - t0).count(); - double avg_us = (n > 0) ? (total_us / (double)n) : 0.0; - - free_buffers(N); - - std::printf("TORUSBENCH,%s,%s,%lld,%.2f,%c\n", collective_name(c), dims_str.c_str(), S, - avg_us, check_char); - std::fflush(stdout); - } - - global_teardown(); - return any_check_failure ? 1 : 0; -} -