From a7cd6970a49da280d665522147187b6b346a2ca8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 26 Aug 2026 15:21:29 +0300 Subject: [PATCH 01/52] feat: support API key for CryptoCompare market source (#14746) --- .../explorer/market/source/crypto_compare.ex | 7 ++--- .../market/source/crypto_compare_test.exs | 30 +++++++++++++++++++ bin/install_chrome_headless.sh | 2 +- config/runtime.exs | 1 + docker-compose/envs/common-blockscout.env | 1 + 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/apps/explorer/lib/explorer/market/source/crypto_compare.ex b/apps/explorer/lib/explorer/market/source/crypto_compare.ex index 0b7cc0e9801..572af5dc176 100644 --- a/apps/explorer/lib/explorer/market/source/crypto_compare.ex +++ b/apps/explorer/lib/explorer/market/source/crypto_compare.ex @@ -89,10 +89,9 @@ defmodule Explorer.Market.Source.CryptoCompare do end defp headers do - if config(:api_key) do - [{"Authorization", "Apikey #{config(:api_key)}"}] - else - [] + case config(:api_key) do + empty when empty in [nil, ""] -> [] + api_key -> [{"Authorization", "Apikey #{api_key}"}] end end diff --git a/apps/explorer/test/explorer/market/source/crypto_compare_test.exs b/apps/explorer/test/explorer/market/source/crypto_compare_test.exs index edd6226fbca..d248df33927 100644 --- a/apps/explorer/test/explorer/market/source/crypto_compare_test.exs +++ b/apps/explorer/test/explorer/market/source/crypto_compare_test.exs @@ -125,6 +125,36 @@ defmodule Explorer.Market.Source.CryptoCompareTest do } ]} == CryptoCompare.fetch_native_coin_price_history(3) end + + test "sets Authorization header if api key is configured", %{bypass: bypass} do + config = Application.get_env(:explorer, CryptoCompare) + Application.put_env(:explorer, CryptoCompare, Keyword.merge(config, api_key: "test_api_key")) + + on_exit(fn -> Application.put_env(:explorer, CryptoCompare, config) end) + + Bypass.expect_once(bypass, "GET", "data/v2/histoday", fn conn -> + assert Conn.get_req_header(conn, "authorization") == ["Apikey test_api_key"] + + Conn.resp(conn, 200, json_data_v2_histoday()) + end) + + assert {:ok, _} = CryptoCompare.fetch_native_coin_price_history(3) + end + + test "doesn't set Authorization header if api key is empty", %{bypass: bypass} do + config = Application.get_env(:explorer, CryptoCompare) + Application.put_env(:explorer, CryptoCompare, Keyword.merge(config, api_key: "")) + + on_exit(fn -> Application.put_env(:explorer, CryptoCompare, config) end) + + Bypass.expect_once(bypass, "GET", "data/v2/histoday", fn conn -> + assert Conn.get_req_header(conn, "authorization") == [] + + Conn.resp(conn, 200, json_data_v2_histoday()) + end) + + assert {:ok, _} = CryptoCompare.fetch_native_coin_price_history(3) + end end describe "secondary_coin_price_history_fetching_enabled?" do diff --git a/bin/install_chrome_headless.sh b/bin/install_chrome_headless.sh index dd6219ce36a..5944e2f2abf 100755 --- a/bin/install_chrome_headless.sh +++ b/bin/install_chrome_headless.sh @@ -2,7 +2,7 @@ export DISPLAY=:99.0 sh -e /etc/init.d/xvfb start #export CHROMEDRIVER_VERSION=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json" | jq -r '.channels' | jq -r '.Stable' | jq -r '.version') -export CHROMEDRIVER_VERSION=150.0.7871.125 +export CHROMEDRIVER_VERSION=152.0.7977.65 curl -L -O "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${CHROMEDRIVER_VERSION}/linux64/chromedriver-linux64.zip" unzip -j chromedriver-linux64.zip sudo chmod +x chromedriver diff --git a/config/runtime.exs b/config/runtime.exs index c7db8f4a3ce..a71ba22407d 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -529,6 +529,7 @@ config :explorer, Explorer.Market.Source.CoinMarketCap, config :explorer, Explorer.Market.Source.CryptoCompare, base_url: ConfigHelper.parse_url_env_var("MARKET_CRYPTOCOMPARE_BASE_URL", "https://min-api.cryptocompare.com"), + api_key: System.get_env("MARKET_CRYPTOCOMPARE_API_KEY"), coin_symbol: System.get_env("MARKET_CRYPTOCOMPARE_COIN_SYMBOL", coin), secondary_coin_symbol: System.get_env("MARKET_CRYPTOCOMPARE_SECONDARY_COIN_SYMBOL") || diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index b0b69cf8b90..ed568d357ec 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -70,6 +70,7 @@ DISABLE_MARKET=true # MARKET_COINMARKETCAP_COIN_ID= # MARKET_COINMARKETCAP_SECONDARY_COIN_ID= # MARKET_CRYPTOCOMPARE_BASE_URL= +# MARKET_CRYPTOCOMPARE_API_KEY= # MARKET_CRYPTOCOMPARE_COIN_SYMBOL= # MARKET_CRYPTOCOMPARE_SECONDARY_COIN_SYMBOL= # MARKET_CRYPTORANK_SECONDARY_COIN_ID= From 99f2c8474b1ff88874205d13b38a82873a058081 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:48:29 +0300 Subject: [PATCH 02/52] chore(deps): bump redix from 1.6.0 to 1.8.0 (#14738) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index cf265a01644..3d2f6b93688 100644 --- a/mix.lock +++ b/mix.lock @@ -156,7 +156,7 @@ "que": {:hex, :que, "0.12.0", "4629c156036b8a3bf9ffdd79aa4069ab414822144d3ef031b5cc61fc805942d3", [:mix], [{:ex_utils, "~> 0.1", [hex: :ex_utils, repo: "hexpm", optional: false]}, {:memento, "~> 0.6", [hex: :memento, repo: "hexpm", optional: false]}], "hexpm", "906651d60b1036840f1afacc0036541583eda9ba4b0b03f322530a1c84100507"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"}, - "redix": {:hex, :redix, "1.6.0", "694179c7a3c71bffac8848fcbfe16f6f6e2a1e020f00a2ad01bc6484815470d1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b2eccb05e02f21c0c3ca57513e6bacb4dd48e6406dadbd7ff9fbe07bd6745999"}, + "redix": {:hex, :redix, "1.8.0", "f0a2d1dc047a4a1b3004525d27e5df7a4e5b42768246bf161e44bb5e744f2a34", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "24288583e0b08cde42aa781053cd14d992a02976fe06986fb664b8ebfcf0f853"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"}, From 509e8f612450649f48cdf4af9d80bebf4ce25201 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:35:59 +0300 Subject: [PATCH 03/52] chore(deps-dev): bump phoenix_live_reload from 1.6.2 to 1.7.0 (#14654) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 3d2f6b93688..180a649e6af 100644 --- a/mix.lock +++ b/mix.lock @@ -135,9 +135,9 @@ "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"}, "phoenix_html": {:hex, :phoenix_html, "4.2.1", "35279e2a39140068fc03f8874408d58eef734e488fc142153f055c5454fd1c08", [:mix], [], "hexpm", "cff108100ae2715dd959ae8f2a8cef8e20b593f8dfd031c9cba92702cf23e053"}, "phoenix_html_helpers": {:hex, :phoenix_html_helpers, "1.0.1", "7eed85c52eff80a179391036931791ee5d2f713d76a81d0d2c6ebafe1e11e5ec", [:mix], [{:phoenix_html, "~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cffd2385d1fa4f78b04432df69ab8da63dc5cf63e07b713a4dcf36a3740e3090"}, - "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.6.2", "b18b0773a1ba77f28c52decbb0f10fd1ac4d3ae5b8632399bbf6986e3b665f62", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "d1f89c18114c50d394721365ffb428cce24f1c13de0467ffa773e2ff4a30d5b9"}, + "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.7.0", "fb1e429f6d8778ce3a6962debdc5e555428a05a6e7b058d6dbad13d281a2c31f", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "dc9f44271aa6fc4ab7797f2aa374ba096ef2c87520586280eb095626b7387a68"}, "phoenix_live_view": {:hex, :phoenix_live_view, "1.2.10", "eb4958045f71d4962373e9ed5967b592375a9dafc73f52997f3996759998a39d", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bcf9d64846b770bc64b1a58dc40af406d80eb61b6e516f7aea4cc8cc15e0a1d9"}, - "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.2.0", "ff3a5616e1bed6804de7773b92cbccfc0b0f473faf1f63d7daf1206c7aeaaa6f", [:mix], [], "hexpm", "adc313a5bf7136039f63cfd9668fde73bba0765e0614cba80c06ac9460ff3e96"}, + "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.3.0", "03916bfbc31a5121945b3cfffe5aec647a5c97fe1dc172a319b94428562359c9", [:mix], [], "hexpm", "eec7be6e9cf02e2551d389b558402d6c637cd3973796326e7ba4bb03c6b2e91d"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, "plug": {:hex, :plug, "1.20.3", "56c480c633ec2ce10140e236e15233bf576e1d323887d7c96711bd02ab5160db", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "be266aee1b8536ef6409d58cf39a3121319f0ec47cfa1b24024485aa0e76ad76"}, From b6e0e0adde5112580b1f9beb9e0452c48734303c Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:34:32 +0300 Subject: [PATCH 04/52] perf: Remove unconditional contract_address preload for tokens (#14750) --- .../controllers/api/v2/token_controller.ex | 23 +++++++++++++++++-- .../transaction_interpretation.ex | 23 ++++++++++++++++--- apps/explorer/lib/explorer/chain.ex | 1 - 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index e140b16d8fa..20b23aad0f4 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -1,7 +1,11 @@ # SPDX-License-Identifier: LicenseRef-Blockscout defmodule BlockScoutWeb.API.V2.TokenController do use BlockScoutWeb, :controller - use Utils.CompileTimeEnvHelper, bridged_tokens_enabled: [:explorer, [Explorer.Chain.BridgedToken, :enabled]] + + use Utils.CompileTimeEnvHelper, + bridged_tokens_enabled: [:explorer, [Explorer.Chain.BridgedToken, :enabled]], + chain_type: [:explorer, :chain_type] + use OpenApiSpex.ControllerSpecs alias BlockScoutWeb.{AccessHelper, AuthenticationHelper} @@ -48,7 +52,22 @@ defmodule BlockScoutWeb.API.V2.TokenController do @api_true [api?: true] - @token_options [api?: true, necessity_by_association: %{reputation_association() => :optional}] + case @chain_type do + :filecoin -> + @chain_type_token_necessity_by_association %{contract_address: :optional} + + :zilliqa -> + @chain_type_token_necessity_by_association %{contract_address: :optional} + + _ -> + @chain_type_token_necessity_by_association %{} + end + + @token_options [ + api?: true, + necessity_by_association: + Map.merge(%{reputation_association() => :optional}, @chain_type_token_necessity_by_association) + ] operation :token, summary: "Retrieve detailed information about a specific token", diff --git a/apps/block_scout_web/lib/block_scout_web/microservice_interfaces/transaction_interpretation.ex b/apps/block_scout_web/lib/block_scout_web/microservice_interfaces/transaction_interpretation.ex index 5b53275d17e..4d588047a9b 100644 --- a/apps/block_scout_web/lib/block_scout_web/microservice_interfaces/transaction_interpretation.ex +++ b/apps/block_scout_web/lib/block_scout_web/microservice_interfaces/transaction_interpretation.ex @@ -27,12 +27,29 @@ defmodule BlockScoutWeb.MicroserviceInterfaces.TransactionInterpretation do @request_error_msg "Error while sending request to Transaction Interpretation Service" @api_true api?: true @items_limit 50 + + case @chain_type do + :filecoin -> + @chain_type_token_necessity_by_association %{contract_address: :optional} + + :zilliqa -> + @chain_type_token_necessity_by_association %{contract_address: :optional} + + _ -> + @chain_type_token_necessity_by_association %{} + end + @token_options [ api?: true, - necessity_by_association: %{ - reputation_association() => :optional - } + necessity_by_association: + Map.merge( + %{ + reputation_association() => :optional + }, + @chain_type_token_necessity_by_association + ) ] + @internal_transaction_address_preloads [ address_preloads: [ created_contract_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index a127a18c07f..c995fc4bfda 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -2475,7 +2475,6 @@ defmodule Explorer.Chain do query |> join_associations(necessity_by_association) - |> preload(:contract_address) |> select_repo(options).one() |> case do nil -> From 8f4799722d41c6ebf009a84b68007ec1bd605947 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:52:51 +0300 Subject: [PATCH 05/52] perf: Remove redundant preloads in api/v2/addresses/* (#14751) --- .../address_coin_balance_controller.ex | 4 +- .../controllers/api/v2/address_controller.ex | 59 +++++++++---------- .../controllers/api/v2/fallback_controller.ex | 9 +++ .../explorer/chain/address/coin_balance.ex | 29 ++++----- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_coin_balance_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_coin_balance_controller.ex index b5a2997df70..fe61bf158c5 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_coin_balance_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_coin_balance_controller.ex @@ -18,11 +18,11 @@ defmodule BlockScoutWeb.AddressCoinBalanceController do def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), - {:ok, address} <- Chain.hash_to_address(address_hash, []), + {:ok, _address} <- Chain.hash_to_address(address_hash, []), {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do full_options = paging_options(params) - coin_balances_plus_one = CoinBalance.address_to_coin_balances(address, full_options) + coin_balances_plus_one = CoinBalance.address_hash_to_coin_balances(address_hash, full_options) {coin_balances, next_page} = split_list_by_page(coin_balances_plus_one) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex index 5f9118b36fe..2e526c60d5d 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex @@ -250,8 +250,7 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec counters(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def counters(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - # TODO: check if @address_options is needed here - case Chain.hash_to_address(address_hash, @address_options) do + case Chain.hash_to_address(address_hash, Keyword.merge(@api_true, necessity_by_association: %{})) do {:ok, address} -> {validation_count} = Counters.address_counters(address, @api_true) @@ -314,8 +313,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do ip = AccessHelper.conn_to_ip_string(conn) with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> token_balances = address_hash |> Chain.fetch_last_token_balances( @@ -395,8 +394,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec transactions(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def transactions(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> options = [necessity_by_association: address_transactions_necessity_by_association()] |> Keyword.merge(@api_true) @@ -519,8 +518,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params), {:ok, token_address_hash} <- validate_optional_address_hash(params[:token], params), token_address_exists <- (token_address_hash && Token.check_token_exists(token_address_hash)) || :ok do - case {Chain.hash_to_address(address_hash, @address_options), token_address_exists} do - {{:ok, _address}, :ok} -> + case {Address.check_address_exists(address_hash, @api_true), token_address_exists} do + {:ok, :ok} -> paging_options = paging_options(params) options = @@ -607,8 +606,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec internal_transactions(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def internal_transactions(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> full_options = [ address_preloads: [ @@ -682,8 +681,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do def logs(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params), {:ok, topic} <- validate_optional_topic(params[:topic]) do - case Chain.hash_to_address(address_hash, @api_true) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> options = params |> paging_options() @@ -755,8 +754,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec blocks_validated(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def blocks_validated(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> full_options = [ necessity_by_association: %{ @@ -822,11 +821,11 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec coin_balance_history(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def coin_balance_history(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> full_options = params |> paging_options() |> Keyword.merge(@api_true) - results_plus_one = CoinBalance.address_to_coin_balances(address, full_options) + results_plus_one = CoinBalance.address_hash_to_coin_balances(address_hash, full_options) {coin_balances, next_page} = split_list_by_page(results_plus_one) @@ -884,8 +883,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def coin_balance_history_by_day(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> balances_by_day = address_hash |> Chain.address_to_balances_by_day(@api_true) @@ -945,8 +944,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do ip = AccessHelper.conn_to_ip_string(conn) with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> results_plus_one = address_hash |> Chain.fetch_paginated_last_token_balances( @@ -1017,8 +1016,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec withdrawals(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def withdrawals(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> options = @api_true |> Keyword.merge(paging_options(params)) withdrawals_plus_one = address_hash |> Chain.address_hash_to_withdrawals(options) {withdrawals, next_page} = split_list_by_page(withdrawals_plus_one) @@ -1150,8 +1149,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do beacon_deposits: :beacon_deposits_count } - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> counters_json = address_hash |> Counters.address_limited_counters(@api_true) @@ -1227,8 +1226,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec nft_list(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def nft_list(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> results_plus_one = Instance.nft_list( address_hash, @@ -1303,8 +1302,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec nft_collections(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def nft_collections(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do - case Chain.hash_to_address(address_hash, @address_options) do - {:ok, _address} -> + case Address.check_address_exists(address_hash, @api_true) do + :ok -> results_plus_one = Instance.nft_collections( address_hash, @@ -1369,7 +1368,7 @@ defmodule BlockScoutWeb.API.V2.AddressController do @spec celo_election_rewards(Plug.Conn.t(), map()) :: {:format, :error} | {:restricted_access, true} | Plug.Conn.t() def celo_election_rewards(conn, %{address_hash_param: address_hash_string} = params) do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params), - {:ok, _address} <- Chain.hash_to_address(address_hash, api?: true) do + :ok <- Address.check_address_exists(address_hash, api?: true) do full_options = @celo_election_rewards_options |> Keyword.put( diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex index 8e50705e493..a768392e55f 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex @@ -128,6 +128,15 @@ defmodule BlockScoutWeb.API.V2.FallbackController do |> call({:not_found, nil}) end + def call(conn, :not_found) do + Logger.debug(fn -> + [":not_found"] + end) + + conn + |> call({:not_found, nil}) + end + def call(conn, {:error, %Changeset{} = changeset}) do conn |> put_status(:unprocessable_entity) diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance.ex b/apps/explorer/lib/explorer/chain/address/coin_balance.ex index a961bf15cc9..6c4538c77e8 100644 --- a/apps/explorer/lib/explorer/chain/address/coin_balance.ex +++ b/apps/explorer/lib/explorer/chain/address/coin_balance.ex @@ -167,28 +167,23 @@ defmodule Explorer.Chain.Address.CoinBalance do end @doc """ - Retrieves paginated coin balance records for a given address with timestamp interpolation. + Retrieves paginated coin balance records for a given address hash with timestamp interpolation. - This function fetches coin balance history for an address, applying pagination - and performing timestamp calculations for blocks. It includes an optimization - that returns an empty list immediately when the paging key is `{0}`, avoiding - unnecessary database queries. For other cases, it processes balances by - filtering records with values, calculating block timestamp ranges, and - interpolating timestamps for intermediate blocks when multiple blocks are - present. + Fetches coin balance history for an address, applying pagination and performing + timestamp calculations for blocks. Returns an empty list immediately when the + paging key is `{0}`, avoiding unnecessary database queries. ## Parameters - - `address`: Address.t() - The address record to fetch coin balances for - - `options`: [Chain.paging_options() | Chain.api?()] - Query options including - paging configuration and API mode selection + - `address_hash`: `Hash.Address.t()` - The address hash to fetch coin balances for. + - `options`: keyword list with paging and API mode options. ## Returns - `[t()]` - List of coin balance records sorted by block number in descending order, with interpolated timestamps, or empty list if paging key is `{0}` or - no balances exist + no balances exist. """ - @spec address_to_coin_balances(Address.t(), [Chain.paging_options() | Chain.api?()]) :: [t()] - def address_to_coin_balances(address, options) do + @spec address_hash_to_coin_balances(Hash.Address.t(), [Chain.paging_options() | Chain.api?()]) :: [t()] + def address_hash_to_coin_balances(address_hash, options) do paging_options = Keyword.get(options, :paging_options, PagingOptions.default_paging_options()) case paging_options do @@ -196,13 +191,13 @@ defmodule Explorer.Chain.Address.CoinBalance do [] _ -> - address_to_coin_balances_internal(address, options, paging_options) + address_hash_to_coin_balances_internal(address_hash, options, paging_options) end end - defp address_to_coin_balances_internal(address, options, paging_options) do + defp address_hash_to_coin_balances_internal(address_hash, options, paging_options) do balances_raw = - address.hash + address_hash |> fetch_coin_balances(paging_options) |> page_coin_balances(paging_options) |> Chain.select_repo(options).all() From 3b4147abdbb973e4c5a7d96d568765a4a61b1537 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Sat, 29 Aug 2026 08:47:28 +0400 Subject: [PATCH 06/52] perf: Reduce addresses preload queries count (#14755) --- .../controllers/api/v2/address_controller.ex | 71 ++++++++++++------- apps/explorer/lib/explorer/chain.ex | 69 ++++++++++++++++-- 2 files changed, 107 insertions(+), 33 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex index 2e526c60d5d..2578e1a5950 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex @@ -73,8 +73,6 @@ defmodule BlockScoutWeb.API.V2.AddressController do @token_transfer_necessity_by_association [ necessity_by_association: %{ - [to_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()]] => :optional, - [from_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()]] => :optional, :block => :optional, :transaction => :optional, [token: reputation_association()] => :optional @@ -82,6 +80,33 @@ defmodule BlockScoutWeb.API.V2.AddressController do api?: true ] + # Address-info associations shared by every participant role of a list item. + # Loaded once for the whole page by `Chain.preload_address_participants/4` + # rather than per role through `necessity_by_association`. + @transaction_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + proxy_implementations_association() => :optional + } + + @token_transfer_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + :smart_contract => :optional, + proxy_implementations_association() => :optional + } + + @transaction_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + + @token_transfer_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] + case @chain_identity do {:optimism, :celo} -> @chain_type_address_necessity_by_association %{ @@ -419,7 +444,14 @@ defmodule BlockScoutWeb.API.V2.AddressController do |> put_status(200) |> put_view(TransactionView) |> render(:transactions, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: + transactions + |> Chain.preload_address_participants( + @transaction_address_fields, + @transaction_participant_necessity_by_association, + @api_true + ) + |> maybe_preload_ens_and_metadata(:transactions), next_page_params: next_page_params }) @@ -435,31 +467,11 @@ defmodule BlockScoutWeb.API.V2.AddressController do end end + # Address participants are loaded separately by + # `Chain.preload_address_participants/4`, which shares one query pass between + # `from`/`to`/`created_contract` instead of repeating it per role. defp address_transactions_necessity_by_association do - %{ - [ - created_contract_address: [ - :scam_badge, - :names, - proxy_implementations_association() - ] - ] => :optional, - [ - from_address: [ - :scam_badge, - :names, - proxy_implementations_association() - ] - ] => :optional, - [ - to_address: [ - :scam_badge, - :names, - proxy_implementations_association() - ] - ] => :optional, - :block => :optional - } + %{:block => :optional} |> Map.merge(@chain_type_transaction_necessity_by_association) end @@ -548,6 +560,11 @@ defmodule BlockScoutWeb.API.V2.AddressController do |> render(:token_transfers, %{ token_transfers: token_transfers + |> Chain.preload_address_participants( + @token_transfer_address_fields, + @token_transfer_participant_necessity_by_association, + @api_true + ) |> Instance.preload_nft(@api_true) |> maybe_preload_ens_and_metadata(:token_transfers), next_page_params: next_page_params diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index c995fc4bfda..0ec23f22dd6 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -1174,12 +1174,7 @@ defmodule Explorer.Chain do |> Enum.reject(&is_nil/1) |> Enum.uniq() - addresses = - Address - |> where([address], address.hash in ^participant_hashes) - |> join_associations(address_necessity_by_association) - |> select_repo(options).all() - |> Map.new(&{&1.hash, &1}) + addresses = addresses_by_hash(participant_hashes, address_necessity_by_association, options) to_address = addresses @@ -1202,6 +1197,68 @@ defmodule Explorer.Chain do } end + @doc """ + Loads address-info associations for every address referenced by a list of items in a single query pass. + + Addresses shared between items, and between roles of the same item, are + deduplicated. Preloading the same associations per role instead — as + `necessity_by_association` does — repeats both the `addresses` query and + every nested association query once per role. This issues one `addresses` + query and one query per entry in `address_necessity_by_association`, + regardless of how many roles are populated. + + Must run before `Explorer.Chain.Address.MetadataPreloader`, which writes ENS + and metadata into the very address structs assigned here. + + ## Parameters + - `items`: The list of structs whose address associations are populated. + - `address_fields`: The `{hash_field, association_field}` pairs to populate, for + example `[{:from_address_hash, :from_address}, {:to_address_hash, :to_address}]`. + - `address_necessity_by_association`: A map of address associations to load, + shared by every role. + - `options`: An optional keyword list of options, such as selecting a specific repository. + + ## Returns + - The list of items with every association named in `address_fields` set to the + matching `t:Explorer.Chain.Address.t/0`, or to `nil` when the hash field is + `nil` or no address row exists. + """ + @spec preload_address_participants([struct()], [{atom(), atom()}], %{any() => :optional | :required}, [api?]) :: + [struct()] + def preload_address_participants(items, address_fields, address_necessity_by_association, options) + + def preload_address_participants([], _address_fields, _address_necessity_by_association, _options), do: [] + + def preload_address_participants(items, address_fields, address_necessity_by_association, options) do + addresses = + items + |> Enum.flat_map(fn item -> + Enum.map(address_fields, fn {hash_field, _association_field} -> Map.fetch!(item, hash_field) end) + end) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> addresses_by_hash(address_necessity_by_association, options) + + Enum.map(items, fn item -> + Enum.reduce(address_fields, item, fn {hash_field, association_field}, item_acc -> + Map.replace!(item_acc, association_field, Map.get(addresses, Map.fetch!(item_acc, hash_field))) + end) + end) + end + + @spec addresses_by_hash([Hash.Address.t()], %{any() => :optional | :required}, [api?]) :: %{ + Hash.Address.t() => Address.t() + } + defp addresses_by_hash([], _address_necessity_by_association, _options), do: %{} + + defp addresses_by_hash(hashes, address_necessity_by_association, options) do + Address + |> where([address], address.hash in ^hashes) + |> join_associations(address_necessity_by_association) + |> select_repo(options).all() + |> Map.new(&{&1.hash, &1}) + end + defp preload_full_smart_contract(%Address{contract_code: contract_code} = address, options) when not is_nil(contract_code) do select_repo(options).preload(address, :smart_contract, force: true) From 0995dc3296c1a8d53454ca54c79af0753ef6e33e Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:40:38 +0400 Subject: [PATCH 07/52] perf: Deduplicate addresses preloads (#14758) --- .../controllers/api/v2/address_controller.ex | 38 ++++++++++++++----- .../controllers/api/v2/token_controller.ex | 17 +++++++++ .../api/v2/transaction_controller.ex | 21 ++++++++-- .../controllers/tokens/transfer_controller.ex | 21 ++++++++++ .../lib/explorer/chain/token_transfer.ex | 4 +- 5 files changed, 85 insertions(+), 16 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex index 2578e1a5950..0a5f6c3d041 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex @@ -107,6 +107,19 @@ defmodule BlockScoutWeb.API.V2.AddressController do {:to_address_hash, :to_address} ] + @internal_transaction_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + :smart_contract => :optional, + proxy_implementations_association() => :optional + } + + @internal_transaction_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + case @chain_identity do {:optimism, :celo} -> @chain_type_address_necessity_by_association %{ @@ -625,15 +638,15 @@ defmodule BlockScoutWeb.API.V2.AddressController do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do case Address.check_address_exists(address_hash, @api_true) do :ok -> + # Nested address-info preloads are intentionally absent from + # `:address_preloads`: `address_to_internal_transactions/2` resolves the + # bare addresses and aligns the `*_address_hash` fields for both the + # hash-based and the address-id-based schema, after which + # `Chain.preload_address_participants/4` loads the info for all three + # roles in a single pass instead of one per role. full_options = - [ - address_preloads: [ - created_contract_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], - from_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], - to_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()] - ] - ] - |> Keyword.merge(paging_options(params)) + params + |> paging_options() |> Keyword.merge(current_filter(params)) |> Keyword.merge(@api_true) @@ -647,7 +660,14 @@ defmodule BlockScoutWeb.API.V2.AddressController do |> put_status(200) |> put_view(TransactionView) |> render(:internal_transactions, %{ - internal_transactions: internal_transactions |> maybe_preload_ens_and_metadata(), + internal_transactions: + internal_transactions + |> Chain.preload_address_participants( + @internal_transaction_address_fields, + @internal_transaction_participant_necessity_by_association, + @api_true + ) + |> maybe_preload_ens_and_metadata(), next_page_params: next_page_params }) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index 20b23aad0f4..42d4716f8ea 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -52,6 +52,18 @@ defmodule BlockScoutWeb.API.V2.TokenController do @api_true [api?: true] + @token_transfer_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + :smart_contract => :optional, + proxy_implementations_association() => :optional + } + + @token_transfer_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] + case @chain_type do :filecoin -> @chain_type_token_necessity_by_association %{contract_address: :optional} @@ -203,6 +215,11 @@ defmodule BlockScoutWeb.API.V2.TokenController do |> render(:token_transfers, %{ token_transfers: token_transfers + |> Chain.preload_address_participants( + @token_transfer_address_fields, + @token_transfer_participant_necessity_by_association, + @api_true + ) |> Instance.preload_nft(@api_true) |> maybe_preload_ens_and_metadata(:token_transfers), next_page_params: next_page_params diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index 81f89a4847a..d25fb44e548 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -105,13 +105,21 @@ defmodule BlockScoutWeb.API.V2.TransactionController do } @token_transfers_necessity_by_association %{ - [from_address: [:scam_badge, :names, SmartContract.association_without_abi(), proxy_implementations_association()]] => - :optional, - [to_address: [:scam_badge, :names, SmartContract.association_without_abi(), proxy_implementations_association()]] => - :optional, [token: reputation_association()] => :optional } + @token_transfer_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + SmartContract.association_without_abi() => :optional, + proxy_implementations_association() => :optional + } + + @token_transfer_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] + # Transfer from/to address preloads are handled by # `Chain.preload_transaction_participants/3`, see # `@transaction_participants_necessity_by_association`. @@ -707,6 +715,11 @@ defmodule BlockScoutWeb.API.V2.TransactionController do |> render(:token_transfers, %{ token_transfers: token_transfers + |> Chain.preload_address_participants( + @token_transfer_address_fields, + @token_transfer_participant_necessity_by_association, + @api_true + ) |> Instance.preload_nft(@api_true) |> maybe_preload_ens_and_metadata(:token_transfers), next_page_params: next_page_params diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex index a7aa73ec6e3..941835a8624 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex @@ -14,10 +14,23 @@ defmodule BlockScoutWeb.Tokens.TransferController do import BlockScoutWeb.Chain, only: [split_list_by_page: 1, paging_options: 1, next_page_params: 3] import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + import Explorer.Chain.SmartContract.Proxy.Models.Implementation, only: [proxy_implementations_association: 0] {:ok, burn_address_hash} = Chain.string_to_address_hash(burn_address_hash_string()) @burn_address_hash burn_address_hash + @token_transfer_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + :smart_contract => :optional, + proxy_implementations_association() => :optional + } + + @token_transfer_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] + def index(conn, %{"token_id" => address_hash_string, "type" => "JSON"} = params) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash), @@ -25,6 +38,14 @@ defmodule BlockScoutWeb.Tokens.TransferController do {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do {token_transfers_paginated, next_page} = split_list_by_page(token_transfers) + token_transfers_paginated = + Chain.preload_address_participants( + token_transfers_paginated, + @token_transfer_address_fields, + @token_transfer_participant_necessity_by_association, + [] + ) + next_page_path = case next_page_params(next_page, token_transfers_paginated, params) do nil -> diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 2ac8c87b7ed..5ffe1820f6f 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -279,9 +279,7 @@ defmodule Explorer.Chain.TokenTransfer do preloads = DenormalizationHelper.extend_transaction_preload([ :transaction, - [token: reputation_association()], - [from_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]], - [to_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]] + [token: reputation_association()] ]) only_consensus_transfers_query() From 160bba231a483b138acd63f85848c474fe238114 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:50:28 +0300 Subject: [PATCH 08/52] chore(deps): bump typed_ecto_schema from 0.4.3 to 0.5.0 (#14761) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- apps/explorer/mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index 616f7957fa1..f29eebbf3e9 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -121,7 +121,7 @@ defmodule Explorer.Mixfile do {:cloak_ecto, "~> 1.3.0"}, {:redix, "~> 1.1"}, {:hammer_backend_redis, "~> 7.0"}, - {:typed_ecto_schema, "~> 0.4.1"}, + {:typed_ecto_schema, "~> 0.5.0"}, {:ueberauth, "~> 0.7"}, {:recon, "~> 2.5"}, {:varint, "~> 1.4"}, diff --git a/mix.lock b/mix.lock index 180a649e6af..e2308acb2e3 100644 --- a/mix.lock +++ b/mix.lock @@ -174,7 +174,7 @@ "tesla": {:hex, :tesla, "1.21.2", "71b3a8a000802fa82f0b87c22acbca1e8f6a03f23d727e01b66687111681debc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.21 or >= 4.0.2 and < 5.0.0-0", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.5.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "296bbafd1a328533c57e6b0e93078983cf40a827546e58a20464ce75485d5254"}, "timex": {:hex, :timex, "3.7.13", "0688ce11950f5b65e154e42b47bf67b15d3bc0e0c3def62199991b8a8079a1e2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.26", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "09588e0522669328e973b8b4fd8741246321b3f0d32735b589f78b136e6d4c54"}, "ton": {:hex, :ton, "0.5.1", "79745434a93e5f7de3572fdcf04feb048620f0edab9794fc412a73528672927d", [:mix], [{:cafezinho, "~> 0.4.4", [hex: :cafezinho, repo: "hexpm", optional: false]}, {:evil_crc32c, "~> 0.2.9", [hex: :evil_crc32c, repo: "hexpm", optional: false]}, {:ex_pbkdf2, "~> 0.8.4", [hex: :ex_pbkdf2, repo: "hexpm", optional: false]}, {:mnemoniac, "~> 0.1.4", [hex: :mnemoniac, repo: "hexpm", optional: false]}], "hexpm", "916f656c870902a61690347da9500c5ce27f04c02e02441363bac7b128030f07"}, - "typed_ecto_schema": {:hex, :typed_ecto_schema, "0.4.3", "1e5f3b6c763f9b5725975d3ab7f1554525f1f1399b966f2425acf04f9d8dd4fe", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "dcbd9b35b9fda5fa9258e0ae629a99cf4473bd7adfb85785d3f71dfe7a9b2bc0"}, + "typed_ecto_schema": {:hex, :typed_ecto_schema, "0.5.0", "cf0e387727dd898b6a67c80c8528d54bdb96e2318ee07dfbd65848fec2cde91f", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "061c500aa5ce03ed4defd85a2fa8ba9a53d036fc096459d570e7a4de05171ed1"}, "tzdata": {:hex, :tzdata, "1.1.3", "b1cef7bb6de1de90d4ddc25d33892b32830f907e7fc2fccd1e7e22778ab7dfbc", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "d4ca85575a064d29d4e94253ee95912edfb165938743dbf002acdf0dcecb0c28"}, "ueberauth": {:hex, :ueberauth, "0.10.8", "ba78fbcbb27d811a6cd06ad851793aaf7d27c3b30c9e95349c2c362b344cd8f0", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f2d3172e52821375bccb8460e5fa5cb91cfd60b19b636b6e57e9759b6f8c10c1"}, "ueberauth_auth0": {:hex, :ueberauth_auth0, "2.1.0", "0632d5844049fa2f26823f15e1120aa32f27df6f27ce515a4b04641736594bf4", [:mix], [{:oauth2, "~> 2.0", [hex: :oauth2, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.7", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm", "8d3b30fa27c95c9e82c30c4afb016251405706d2e9627e603c3c9787fd1314fc"}, From 45fb7a23d75df0502fa107b6bea6e6684659a741 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:21:40 +0300 Subject: [PATCH 09/52] chore(deps): bump phoenix_live_view from 1.2.10 to 1.2.11 (#14764) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index e2308acb2e3..31b9a9459f2 100644 --- a/mix.lock +++ b/mix.lock @@ -136,7 +136,7 @@ "phoenix_html": {:hex, :phoenix_html, "4.2.1", "35279e2a39140068fc03f8874408d58eef734e488fc142153f055c5454fd1c08", [:mix], [], "hexpm", "cff108100ae2715dd959ae8f2a8cef8e20b593f8dfd031c9cba92702cf23e053"}, "phoenix_html_helpers": {:hex, :phoenix_html_helpers, "1.0.1", "7eed85c52eff80a179391036931791ee5d2f713d76a81d0d2c6ebafe1e11e5ec", [:mix], [{:phoenix_html, "~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cffd2385d1fa4f78b04432df69ab8da63dc5cf63e07b713a4dcf36a3740e3090"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.7.0", "fb1e429f6d8778ce3a6962debdc5e555428a05a6e7b058d6dbad13d281a2c31f", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "dc9f44271aa6fc4ab7797f2aa374ba096ef2c87520586280eb095626b7387a68"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "1.2.10", "eb4958045f71d4962373e9ed5967b592375a9dafc73f52997f3996759998a39d", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bcf9d64846b770bc64b1a58dc40af406d80eb61b6e516f7aea4cc8cc15e0a1d9"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "1.2.11", "1c83fdfd007e8d554076baee00ee6f55bb9c87ad9439f33ee520e47ae4a983bb", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c4087267cbd4dc4ed7ac36512a97830af8e9c857ca7cb22557f8640fc5f9d306"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.3.0", "03916bfbc31a5121945b3cfffe5aec647a5c97fe1dc172a319b94428562359c9", [:mix], [], "hexpm", "eec7be6e9cf02e2551d389b558402d6c637cd3973796326e7ba4bb03c6b2e91d"}, "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, From 99b2ade85d4baeaf96bf3e66a15321b3b3334cac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:41:53 +0300 Subject: [PATCH 10/52] chore(deps): bump oban from 2.23.1 to 2.24.0 (#14763) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 31b9a9459f2..4ade0fedc17 100644 --- a/mix.lock +++ b/mix.lock @@ -126,7 +126,7 @@ "numbers": {:hex, :numbers, "5.2.4", "f123d5bb7f6acc366f8f445e10a32bd403c8469bdbce8ce049e1f0972b607080", [:mix], [{:coerce, "~> 1.0", [hex: :coerce, repo: "hexpm", optional: false]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "eeccf5c61d5f4922198395bf87a465b6f980b8b862dd22d28198c5e6fab38582"}, "nx": {:hex, :nx, "0.12.1", "6e9fee43a77646d04faad2ba4e449b9e270c6b23e413f203cb4d81e71c0a617f", [:mix], [{:complex, "~> 0.7", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee80f6ae898f68bbfe7f30216b06aab10096231ec99ded1208a827256b23d0fb"}, "oauth2": {:hex, :oauth2, "2.1.1", "3bec9bf49e88e1b0c0ddf9f44c393d71fd0f88e905766f2e2cc5d57e6bcc5352", [:mix], [{:tesla, "~> 1.18", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "1d5997cb1ff1643dac17076b6c00c91e4381d8389f3c49a8c390984606dad439"}, - "oban": {:hex, :oban, "2.23.1", "0b9495e28a236ca0478c80d666c58a8b2b55182731c3603c6e766e298a08342a", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a9855b9f5d87e31de3e2f46731b163f372e329613892a56c5b2aacceb50ed508"}, + "oban": {:hex, :oban, "2.24.0", "cd877a089ead66658c40864fc616e90c75ddba95559fffae3fccaf05da58a4e5", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab497b2be51191a2d9d7a1b267714463891be711b20487b7b65b5c2269e4b312"}, "open_api_spex": {:hex, :open_api_spex, "3.22.3", "0e383bf23cc3a060bffaebbcd09fc06bfc908d948c00e518aed36bbf8a2fe473", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "5f74f1878fdc38f8e961b0b943ac7af88dcf3a82a0c0ef6680ddfd3d161aecbd"}, "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, From 8b70593f5907b3c3ec299cb7d7adff31ac90c72a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 3 Sep 2026 12:31:41 +0300 Subject: [PATCH 11/52] perf: Remove unused GasUsageSum cache with heavy DB query (#14771) --- .../css/components/_custom_tooltips.scss | 12 --- .../controllers/api/v2/stats_controller.ex | 3 +- .../controllers/chain_controller.ex | 4 +- .../schemas/api/v2/stats/response.ex | 1 - .../templates/chain/show.html.eex | 11 --- .../api/v2/stats_controller_test.exs | 1 - apps/explorer/lib/explorer/application.ex | 2 - .../chain/cache/counters/gas_usage_sum.ex | 97 ------------------- config/runtime.exs | 4 - docker-compose/envs/common-blockscout.env | 1 - 10 files changed, 2 insertions(+), 134 deletions(-) delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/gas_usage_sum.ex diff --git a/apps/block_scout_web/assets/css/components/_custom_tooltips.scss b/apps/block_scout_web/assets/css/components/_custom_tooltips.scss index 9328e761b3c..56d9411c025 100644 --- a/apps/block_scout_web/assets/css/components/_custom_tooltips.scss +++ b/apps/block_scout_web/assets/css/components/_custom_tooltips.scss @@ -84,18 +84,6 @@ $tooltip-background-color: $btn-line-color !default; } } -.tooltip-gas-usage { - .tooltip-inner { - @media (min-width: 576px) { - max-width: 150px !important; - } - @media (min-width: 768px) { - max-width: 150px !important; - } - width: 150px !important; - } -} - .tooltip-gas-tracker { .tooltip-inner { @media (min-width: 576px) { diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex index 96ab4cf6b35..073258705fc 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/stats_controller.ex @@ -24,7 +24,7 @@ defmodule BlockScoutWeb.API.V2.StatsController do alias BlockScoutWeb.API.V2.Helper alias BlockScoutWeb.Chain.MarketHistoryChartController alias Explorer.{Chain, Market} - alias Explorer.Chain.Cache.Counters.{AddressesCount, AverageBlockTime, BlocksCount, GasUsageSum, TransactionsCount} + alias Explorer.Chain.Cache.Counters.{AddressesCount, AverageBlockTime, BlocksCount, TransactionsCount} alias Explorer.Chain.Cache.GasPriceOracle alias Explorer.Chain.Supply.RSK alias Explorer.Chain.Transaction.History.TransactionStats @@ -105,7 +105,6 @@ defmodule BlockScoutWeb.API.V2.StatsController do "coin_price" => exchange_rate.fiat_value, "coin_price_change_percentage" => coin_price_change, "secondary_coin_price" => secondary_coin_exchange_rate.fiat_value, - "total_gas_used" => GasUsageSum.total() |> to_string(), "transactions_today" => Enum.at(transaction_stats, 0).number_of_transactions |> to_string(), "gas_used_today" => Enum.at(transaction_stats, 0).gas_used, "gas_prices" => gas_prices, diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex index 5b2579ce8d6..d215c96f446 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/chain_controller.ex @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.ChainController do alias BlockScoutWeb.{ChainView, Controller} alias Explorer.{Chain, PagingOptions, Repo} alias Explorer.Chain.{Address, Block, Hash, Transaction} - alias Explorer.Chain.Cache.Counters.{AddressesCount, AverageBlockTime, BlocksCount, GasUsageSum, TransactionsCount} + alias Explorer.Chain.Cache.Counters.{AddressesCount, AverageBlockTime, BlocksCount, TransactionsCount} alias Explorer.Chain.Search alias Explorer.Chain.Supply.RSK alias Explorer.Market @@ -16,7 +16,6 @@ defmodule BlockScoutWeb.ChainController do def show(conn, _params) do transactions_count = TransactionsCount.get() - total_gas_usage = GasUsageSum.total() block_count = BlocksCount.get() address_count = AddressesCount.fetch() @@ -51,7 +50,6 @@ defmodule BlockScoutWeb.ChainController do chart_data_paths: chart_data_paths, market_cap_calculation: market_cap_calculation, transaction_estimated_count: transactions_count, - total_gas_usage: total_gas_usage, transactions_path: recent_transactions_path(conn, :index), transaction_stats: transaction_stats, block_count: block_count, diff --git a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/stats/response.ex b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/stats/response.ex index be2bd076a4a..eee389ef387 100644 --- a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/stats/response.ex +++ b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/stats/response.ex @@ -109,7 +109,6 @@ defmodule BlockScoutWeb.Schemas.API.V2.Stats.Response do static_gas_price: General.IntegerStringNullable, total_addresses: General.IntegerString, total_blocks: General.IntegerString, - total_gas_used: General.IntegerString, total_transactions: General.IntegerString, transactions_today: General.IntegerString, tvl: General.IntegerStringNullable diff --git a/apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex b/apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex index 6f065e60338..5d9442d9dc7 100644 --- a/apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex +++ b/apps/block_scout_web/lib/block_scout_web/templates/chain/show.html.eex @@ -117,17 +117,6 @@ <%= BlockScoutWeb.Cldr.Number.to_string!(@transaction_estimated_count, format: "#,###") %> - <%= if @total_gas_usage > 0 do %> - " - class="custom-tooltip-total-transactions"> - - - <% end %>
diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/stats_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/stats_controller_test.exs index e8925384dd0..d28f0eb923e 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/stats_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/stats_controller_test.exs @@ -28,7 +28,6 @@ defmodule BlockScoutWeb.API.V2.StatsControllerTest do assert Map.has_key?(response, "total_transactions") assert Map.has_key?(response, "average_block_time") assert Map.has_key?(response, "coin_price") - assert Map.has_key?(response, "total_gas_used") assert Map.has_key?(response, "transactions_today") assert Map.has_key?(response, "gas_used_today") assert Map.has_key?(response, "gas_prices") diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index c2ad1da0359..4be5c481d20 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -26,7 +26,6 @@ defmodule Explorer.Application do AddressesCoinBalanceSumMinusBurnt, AddressTabsElementsCount, BlocksCount, - GasUsageSum, PendingBlockOperationCount, PendingTransactionOperationCount, TransactionsCount @@ -92,7 +91,6 @@ defmodule Explorer.Application do Blocks, ChainId, GasPriceOracle, - GasUsageSum, PendingBlockOperationCount, PendingTransactionOperationCount, TransactionsCount, diff --git a/apps/explorer/lib/explorer/chain/cache/counters/gas_usage_sum.ex b/apps/explorer/lib/explorer/chain/cache/counters/gas_usage_sum.ex deleted file mode 100644 index 3c5d4e194d4..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/gas_usage_sum.ex +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.GasUsageSum do - @moduledoc """ - Cache for total gas usage. - """ - require Logger - - import Ecto.Query, - only: [ - from: 2 - ] - - use Explorer.Chain.MapCache, - name: :gas_usage, - key: :sum, - key: :async_task, - global_ttl: :infinity, - ttl_check_interval: :timer.seconds(1), - callback: &async_task_on_deletion(&1) - - alias Explorer.Chain.Cache.Counters.Helper, as: CacheCountersHelper - alias Explorer.Chain.Cache.Counters.LastFetchedCounter - alias Explorer.Chain.Transaction - alias Explorer.Repo - - @cache_key "gas_usage_sum" - - @spec total() :: non_neg_integer() - def total do - cached_value_from_ets = __MODULE__.get_sum() - - CacheCountersHelper.evaluate_count(@cache_key, cached_value_from_ets) - end - - defp handle_fallback(:sum) do - # This will get the task PID if one exists, check if it's running and launch - # a new task if task doesn't exist or it's not running. - # See next `handle_fallback` definition - safe_get_async_task() - - {:return, nil} - end - - defp handle_fallback(:async_task) do - if Application.get_env(:explorer, __MODULE__)[:enabled] do - # If this gets called it means an async task was requested, but none exists - # so a new one needs to be launched - {:ok, task} = - Task.start_link(fn -> - try do - result = fetch_sum_gas_used() - - params = %{ - counter_type: @cache_key, - value: result - } - - LastFetchedCounter.upsert(params) - - set_sum(%ConCache.Item{ - ttl: CacheCountersHelper.ttl(__MODULE__, "CACHE_TOTAL_GAS_USAGE_PERIOD"), - value: result - }) - rescue - e -> - Logger.debug([ - "Couldn't update total gas used: ", - Exception.format(:error, e, __STACKTRACE__) - ]) - end - - set_async_task(nil) - end) - - {:update, task} - else - {:update, nil} - end - end - - # By setting this as a `callback` an async task will be started each time the - # `sum` expires (unless there is one already running) - defp async_task_on_deletion({:delete, _, :sum}), do: safe_get_async_task() - - defp async_task_on_deletion(_data), do: nil - - @spec fetch_sum_gas_used() :: non_neg_integer - defp fetch_sum_gas_used do - query = - from( - t0 in Transaction, - select: fragment("SUM(t0.gas_used)") - ) - - Repo.one!(query, timeout: :infinity) || 0 - end -end diff --git a/config/runtime.exs b/config/runtime.exs index a71ba22407d..d239dc0251e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -397,10 +397,6 @@ config :explorer, Explorer.Chain.Cache.Counters.AddressesCoinBalanceSum, global_ config :explorer, Explorer.Chain.Cache.Counters.AddressesCoinBalanceSumMinusBurnt, global_ttl: address_sum_global_ttl -config :explorer, Explorer.Chain.Cache.Counters.GasUsageSum, - global_ttl: ConfigHelper.parse_time_env_var("CACHE_TOTAL_GAS_USAGE_PERIOD", "2h"), - enabled: ConfigHelper.parse_bool_env_var("CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED") - config :explorer, Explorer.Chain.Cache.Counters.BlocksCount, global_ttl: ConfigHelper.parse_time_env_var("CACHE_BLOCK_COUNT_PERIOD", "2h") diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index ed568d357ec..f148615eeb1 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -121,7 +121,6 @@ RELEASE_LINK= # CACHE_BLOCK_COUNT_PERIOD=7200 # CACHE_TXS_COUNT_PERIOD=7200 # CACHE_ADDRESS_SUM_PERIOD=3600 -# CACHE_TOTAL_GAS_USAGE_PERIOD=7200 # CACHE_ADDRESS_TRANSACTIONS_GAS_USAGE_COUNTER_PERIOD=1800 # CACHE_TOKEN_HOLDERS_COUNTER_PERIOD=3600 # CACHE_TOKEN_TRANSFERS_COUNTER_PERIOD=3600 From 5b10b4396e8ddac6e93d7660b6dcf31d6e6ed1a9 Mon Sep 17 00:00:00 2001 From: nikitosing <32202610+nikitosing@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:45:45 +0300 Subject: [PATCH 12/52] fix: Forward [api?: true] where missed (#14740) --- .../graphql/resolvers/internal_transaction.ex | 8 +- apps/explorer/lib/explorer/chain.ex | 2 +- apps/explorer/lib/explorer/chain/address.ex | 32 +++-- .../lib/explorer/chain/address/counters.ex | 8 +- .../lib/explorer/chain/advanced_filter.ex | 123 +++++++++++++----- .../lib/explorer/chain/bridged_token.ex | 2 +- .../explorer/chain/internal_transaction.ex | 19 +-- .../lib/explorer/chain/transaction.ex | 18 +-- apps/explorer/lib/explorer/etherscan.ex | 4 +- .../utility/address_id_to_address_hash.ex | 36 ++--- .../fetcher/on_demand/internal_transaction.ex | 94 ++++++------- .../on_demand/contract_creator_test.exs | 24 ++-- 12 files changed, 220 insertions(+), 150 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/graphql/resolvers/internal_transaction.ex b/apps/block_scout_web/lib/block_scout_web/graphql/resolvers/internal_transaction.ex index be0e4ab25bc..2363321700b 100644 --- a/apps/block_scout_web/lib/block_scout_web/graphql/resolvers/internal_transaction.ex +++ b/apps/block_scout_web/lib/block_scout_web/graphql/resolvers/internal_transaction.ex @@ -2,7 +2,7 @@ defmodule BlockScoutWeb.GraphQL.Resolvers.InternalTransaction do @moduledoc false - import Explorer.Chain, only: [hash_to_transaction: 1] + import Explorer.Chain, only: [hash_to_transaction: 2] alias Absinthe.Relay.Connection alias BlockScoutWeb.Chain @@ -11,12 +11,12 @@ defmodule BlockScoutWeb.GraphQL.Resolvers.InternalTransaction do alias Indexer.Fetcher.OnDemand.InternalTransaction, as: InternalTransactionOnDemand def get_by(%{transaction_hash: transaction_hash, index: index} = args, _) do - case hash_to_transaction(transaction_hash) do + case hash_to_transaction(transaction_hash, api?: true) do {:ok, transaction} -> if InternalTransaction.present_in_db?(transaction.block_number) do GraphQL.get_internal_transaction(args) else - options = [paging_options: %PagingOptions{page_size: index + 1}] + options = [paging_options: %PagingOptions{page_size: index + 1}, api?: true] transaction |> InternalTransactionOnDemand.fetch_by_transaction(options) @@ -41,7 +41,7 @@ defmodule BlockScoutWeb.GraphQL.Resolvers.InternalTransaction do defp options(%{before: _}), do: [] - defp options(%{count: count}), do: [paging_options: %PagingOptions{page_size: count}] + defp options(%{count: count}), do: [paging_options: %PagingOptions{page_size: count}, api?: true] defp options(_), do: [] end diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index 0ec23f22dd6..f52edb9e5f6 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -882,7 +882,7 @@ defmodule Explorer.Chain do |> select_repo(options).one() |> then(fn address -> if Keyword.get(options, :preload_contract_creation_internal_transaction, false) do - Address.maybe_preload_contract_creation_internal_transaction(address, select_repo(options)) + Address.maybe_preload_contract_creation_internal_transaction(address, options) else address end diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index 57bb153ec3a..cbfc7bfa74e 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -473,11 +473,9 @@ defmodule Explorer.Chain.Address do do: address def maybe_preload_smart_contract_associations(%__MODULE__{contract_code: _} = address, associations, options) do - repo = Chain.select_repo(options) - address - |> repo.preload(associations) - |> maybe_preload_contract_creation_internal_transaction(repo) + |> Chain.select_repo(options).preload(associations) + |> maybe_preload_contract_creation_internal_transaction(options) end @doc """ @@ -849,7 +847,7 @@ defmodule Explorer.Chain.Address do ## Parameters - `addresses`: An `Explorer.Chain.Address.t/0`, a list of addresses, `[]`, or `nil` - - `repo`: The repo module used to execute the query. Defaults to `Explorer.Repo` + - `options`: A keyword list of options used to select the repo (for example, `api?: true`) ## Returns @@ -858,14 +856,14 @@ defmodule Explorer.Chain.Address do - A single address with `:contract_creation_internal_transaction` populated when the input is a single struct """ - @spec maybe_preload_contract_creation_internal_transaction([__MODULE__.t()] | __MODULE__.t() | nil, module()) :: + @spec maybe_preload_contract_creation_internal_transaction([__MODULE__.t()] | __MODULE__.t() | nil, [Chain.api?()]) :: [__MODULE__.t()] | __MODULE__.t() | nil - def maybe_preload_contract_creation_internal_transaction(addresses, repo \\ Repo) + def maybe_preload_contract_creation_internal_transaction(addresses, options) - def maybe_preload_contract_creation_internal_transaction([], _repo), do: [] - def maybe_preload_contract_creation_internal_transaction(nil, _repo), do: nil + def maybe_preload_contract_creation_internal_transaction([], _options), do: [] + def maybe_preload_contract_creation_internal_transaction(nil, _options), do: nil - def maybe_preload_contract_creation_internal_transaction(addresses, repo) when is_list(addresses) do + def maybe_preload_contract_creation_internal_transaction(addresses, options) when is_list(addresses) do if Application.get_env(:explorer, :api_disable_contract_creation_internal_transaction_association, false) do addresses else @@ -873,21 +871,21 @@ defmodule Explorer.Chain.Address do internal_transactions_map = contract_creation_internal_transaction_preload_query() - |> InternalTransaction.where_address_match(:created_contract_address, address_hashes) - |> repo.all() - |> InternalTransaction.preload_addresses([], repo) + |> InternalTransaction.where_address_match_by_hash(:created_contract_address, address_hashes, options) + |> Chain.select_repo(options).all() + |> InternalTransaction.preload_addresses([], Chain.select_repo(options)) |> Map.new(&{&1.created_contract_address_hash, &1}) Enum.map(addresses, &%{&1 | contract_creation_internal_transaction: internal_transactions_map[&1.hash]}) end end - def maybe_preload_contract_creation_internal_transaction(address, repo) do + def maybe_preload_contract_creation_internal_transaction(address, options) do if Application.get_env(:explorer, :api_disable_contract_creation_internal_transaction_association, false) do address else [address] - |> maybe_preload_contract_creation_internal_transaction(repo) + |> maybe_preload_contract_creation_internal_transaction(options) |> List.first() end end @@ -1022,7 +1020,7 @@ defmodule Explorer.Chain.Address do def creation_internal_transaction_query(address_hash) do InternalTransaction |> InternalTransaction.join_transaction_query() - |> InternalTransaction.where_address_match(:created_contract_address, address_hash) + |> InternalTransaction.where_address_match_by_hash(:created_contract_address, address_hash, []) |> where(as(:transaction).status == ^:ok) |> order_by([it], desc: it.block_number, desc: it.transaction_index, desc: it.index) |> limit(1) @@ -1058,7 +1056,7 @@ defmodule Explorer.Chain.Address do |> Chain.select_repo(options).one() |> then(fn address -> if Keyword.get(options, :preload_contract_creation_internal_transaction, false) do - Address.maybe_preload_contract_creation_internal_transaction(address) + Address.maybe_preload_contract_creation_internal_transaction(address, options) else address end diff --git a/apps/explorer/lib/explorer/chain/address/counters.ex b/apps/explorer/lib/explorer/chain/address/counters.ex index 4624da788cf..4fd8adefdad 100644 --- a/apps/explorer/lib/explorer/chain/address/counters.ex +++ b/apps/explorer/lib/explorer/chain/address/counters.ex @@ -258,11 +258,11 @@ defmodule Explorer.Chain.Address.Counters do end end - defp address_hash_to_internal_transactions_limited_count_query(address_hash) do + defp address_hash_to_internal_transactions_limited_count_query(address_hash, options) do query_to_address_hash_wrapped = InternalTransaction |> InternalTransaction.where_nonpending_operation() - |> InternalTransaction.where_address_fields_match(address_hash, :to) + |> InternalTransaction.where_address_fields_match(address_hash, :to, options) |> InternalTransaction.where_is_different_from_parent_transaction() |> limit(@counters_limit) |> wrapped_union_subquery() @@ -270,7 +270,7 @@ defmodule Explorer.Chain.Address.Counters do query_from_address_hash_wrapped = InternalTransaction |> InternalTransaction.where_nonpending_operation() - |> InternalTransaction.where_address_fields_match(address_hash, :from_address_hash) + |> InternalTransaction.where_address_fields_match(address_hash, :from_address_hash, options) |> InternalTransaction.where_is_different_from_parent_transaction() |> limit(@counters_limit) |> wrapped_union_subquery() @@ -475,7 +475,7 @@ defmodule Explorer.Chain.Address.Counters do configure_task( :internal_transactions, cached_counters, - address_hash_to_internal_transactions_limited_count_query(address_hash), + address_hash_to_internal_transactions_limited_count_query(address_hash, options), address_hash, options ) diff --git a/apps/explorer/lib/explorer/chain/advanced_filter.ex b/apps/explorer/lib/explorer/chain/advanced_filter.ex index bb65a51ce46..bab9581f309 100644 --- a/apps/explorer/lib/explorer/chain/advanced_filter.ex +++ b/apps/explorer/lib/explorer/chain/advanced_filter.ex @@ -836,7 +836,8 @@ defmodule Explorer.Chain.AdvancedFilter do options[:from_address_hashes], options[:to_address_hashes], options[:address_relation], - order_by + order_by, + options ) end @@ -1221,14 +1222,14 @@ defmodule Explorer.Chain.AdvancedFilter do end end - defp filter_internal_transactions_by_addresses(query, from_addresses, to_addresses, relation, order_by) do + defp filter_internal_transactions_by_addresses(query, from_addresses, to_addresses, relation, order_by, options) do order_by = fn query -> query |> exclude(:order_by) |> order_by.() end case {process_address_inclusion(from_addresses), process_address_inclusion(to_addresses)} do {nil, nil} -> query - {from, nil} -> do_filter_internal_transactions_by_address(query, from, :from_address, order_by) - {nil, to} -> do_filter_internal_transactions_by_address(query, to, :to_address, order_by) - {from, to} -> do_filter_internal_transactions_by_both_addresses(query, from, to, relation, order_by) + {from, nil} -> do_filter_internal_transactions_by_address(query, from, :from_address, order_by, options) + {nil, to} -> do_filter_internal_transactions_by_address(query, to, :to_address, order_by, options) + {from, to} -> do_filter_internal_transactions_by_both_addresses(query, from, to, relation, order_by, options) end end @@ -1253,12 +1254,12 @@ defmodule Explorer.Chain.AdvancedFilter do |> order_by.() end - defp do_filter_internal_transactions_by_address(query, {:include, addresses}, binding, order_by) do + defp do_filter_internal_transactions_by_address(query, {:include, addresses}, binding, order_by, options) do queries = addresses |> Enum.map(fn address -> query - |> InternalTransaction.where_address_match(binding, address) + |> InternalTransaction.where_address_match_by_hash(binding, address, options) |> order_by.() end) |> map_first(&subquery/1) @@ -1267,8 +1268,8 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(queries))) end - defp do_filter_internal_transactions_by_address(query, {:exclude, addresses}, binding, order_by) do - address_ids = AddressIdToAddressHash.hashes_to_ids(addresses) + defp do_filter_internal_transactions_by_address(query, {:exclude, addresses}, binding, order_by, options) do + address_ids = AddressIdToAddressHash.hashes_to_ids(addresses, options) address_id_field = String.to_existing_atom("#{binding}_id") address_hash_field = String.to_existing_atom("#{binding}_hash") @@ -1403,19 +1404,33 @@ defmodule Explorer.Chain.AdvancedFilter do |> order_by.() end - defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:include, to}, :and, order_by) do + defp do_filter_internal_transactions_by_both_addresses( + query, + {:include, from}, + {:include, to}, + :and, + order_by, + options + ) do query - |> InternalTransaction.where_address_match(:from_address, from) - |> InternalTransaction.where_address_match(:to_address, to) + |> InternalTransaction.where_address_match_by_hash(:from_address, from, options) + |> InternalTransaction.where_address_match_by_hash(:to_address, to, options) |> order_by.() end - defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:include, to}, _relation, order_by) do + defp do_filter_internal_transactions_by_both_addresses( + query, + {:include, from}, + {:include, to}, + _relation, + order_by, + options + ) do from_queries = from |> Enum.map(fn from_address -> query - |> InternalTransaction.where_address_match(:from_address, from_address) + |> InternalTransaction.where_address_match_by_hash(:from_address, from_address, options) |> order_by.() end) @@ -1423,7 +1438,7 @@ defmodule Explorer.Chain.AdvancedFilter do to |> Enum.map(fn to_address -> query - |> InternalTransaction.where_address_match(:to_address, to_address) + |> InternalTransaction.where_address_match_by_hash(:to_address, to_address, options) |> order_by.() end) @@ -1436,14 +1451,21 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(union_query))) end - defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:exclude, to}, :and, order_by) do - to_address_ids = AddressIdToAddressHash.hashes_to_ids(to) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:include, from}, + {:exclude, to}, + :and, + order_by, + options + ) do + to_address_ids = AddressIdToAddressHash.hashes_to_ids(to, options) from_queries = from |> Enum.map(fn from_address -> query - |> InternalTransaction.where_address_match(:from_address, from_address) + |> InternalTransaction.where_address_match_by_hash(:from_address, from_address, options) |> where([it], it.to_address_id not in ^to_address_ids and it.to_address_hash not in ^to) |> order_by.() end) @@ -1453,14 +1475,21 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(from_queries))) end - defp do_filter_internal_transactions_by_both_addresses(query, {:include, from}, {:exclude, to}, _relation, order_by) do - to_address_ids = AddressIdToAddressHash.hashes_to_ids(to) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:include, from}, + {:exclude, to}, + _relation, + order_by, + options + ) do + to_address_ids = AddressIdToAddressHash.hashes_to_ids(to, options) from_queries = from |> Enum.map(fn from_address -> query - |> InternalTransaction.where_address_match(:from_address, from_address) + |> InternalTransaction.where_address_match_by_hash(:from_address, from_address, options) |> or_where([it], it.to_address_id not in ^to_address_ids and it.to_address_hash not in ^to) |> order_by.() end) @@ -1470,14 +1499,21 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(from_queries))) end - defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:include, to}, :and, order_by) do - from_address_ids = AddressIdToAddressHash.hashes_to_ids(from) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:exclude, from}, + {:include, to}, + :and, + order_by, + options + ) do + from_address_ids = AddressIdToAddressHash.hashes_to_ids(from, options) to_queries = to |> Enum.map(fn to_address -> query - |> InternalTransaction.where_address_match(:to_address, to_address) + |> InternalTransaction.where_address_match_by_hash(:to_address, to_address, options) |> where([it], it.from_address_id not in ^from_address_ids and it.from_address_hash not in ^from) |> order_by.() end) @@ -1487,14 +1523,21 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(to_queries))) end - defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:include, to}, _relation, order_by) do - from_address_ids = AddressIdToAddressHash.hashes_to_ids(from) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:exclude, from}, + {:include, to}, + _relation, + order_by, + options + ) do + from_address_ids = AddressIdToAddressHash.hashes_to_ids(from, options) to_queries = to |> Enum.map(fn to_address -> query - |> InternalTransaction.where_address_match(:to_address, to_address) + |> InternalTransaction.where_address_match_by_hash(:to_address, to_address, options) |> or_where([it], it.from_address_id not in ^from_address_ids and it.from_address_hash not in ^from) |> order_by.() end) @@ -1504,9 +1547,16 @@ defmodule Explorer.Chain.AdvancedFilter do order_by.(from(internal_transaction in subquery(to_queries))) end - defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:exclude, to}, :and, order_by) do - from_address_ids = AddressIdToAddressHash.hashes_to_ids(from) - to_address_ids = AddressIdToAddressHash.hashes_to_ids(to) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:exclude, from}, + {:exclude, to}, + :and, + order_by, + options + ) do + from_address_ids = AddressIdToAddressHash.hashes_to_ids(from, options) + to_address_ids = AddressIdToAddressHash.hashes_to_ids(to, options) query |> where([it], it.from_address_id not in ^from_address_ids and it.from_address_hash not in ^from) @@ -1514,9 +1564,16 @@ defmodule Explorer.Chain.AdvancedFilter do |> order_by.() end - defp do_filter_internal_transactions_by_both_addresses(query, {:exclude, from}, {:exclude, to}, _relation, order_by) do - from_address_ids = AddressIdToAddressHash.hashes_to_ids(from) - to_address_ids = AddressIdToAddressHash.hashes_to_ids(to) + defp do_filter_internal_transactions_by_both_addresses( + query, + {:exclude, from}, + {:exclude, to}, + _relation, + order_by, + options + ) do + from_address_ids = AddressIdToAddressHash.hashes_to_ids(from, options) + to_address_ids = AddressIdToAddressHash.hashes_to_ids(to, options) query |> where(as(:from_address).hash not in ^from or as(:to_address).hash not in ^to) diff --git a/apps/explorer/lib/explorer/chain/bridged_token.ex b/apps/explorer/lib/explorer/chain/bridged_token.ex index e6cea0c8857..c34b42d14cd 100644 --- a/apps/explorer/lib/explorer/chain/bridged_token.ex +++ b/apps/explorer/lib/explorer/chain/bridged_token.ex @@ -230,7 +230,7 @@ defmodule Explorer.Chain.BridgedToken do created_from_internal_transaction_query = InternalTransaction - |> InternalTransaction.where_address_match(:created_contract_address, token_address_hash) + |> InternalTransaction.where_address_match_by_hash(:created_contract_address, token_address_hash, []) created_from_internal_transaction = created_from_internal_transaction_query diff --git a/apps/explorer/lib/explorer/chain/internal_transaction.ex b/apps/explorer/lib/explorer/chain/internal_transaction.ex index d02e35d929a..0e3496a5ad8 100644 --- a/apps/explorer/lib/explorer/chain/internal_transaction.ex +++ b/apps/explorer/lib/explorer/chain/internal_transaction.ex @@ -508,8 +508,8 @@ defmodule Explorer.Chain.InternalTransaction do - returns a query considering that the given address_hash can be: to_address_hash, from_address_hash, created_contract_address_hash from internal_transactions' table. """ - def where_address_fields_match(query, address_hash, direction) do - address_id = AddressIdToAddressHash.hash_to_id(address_hash) + def where_address_fields_match(query, address_hash, direction, options) do + address_id = AddressIdToAddressHash.hash_to_id(address_hash, options) case direction do :to -> @@ -576,14 +576,15 @@ defmodule Explorer.Chain.InternalTransaction do An `Ecto.Query.t/0` with the address filter applied. """ - @spec where_address_match( + @spec where_address_match_by_hash( Ecto.Query.t() | module(), :from_address | :to_address | :created_contract_address, - Hash.Address.t() | [Hash.Address.t()] + Hash.Address.t() | [Hash.Address.t()], + [Chain.api?()] ) :: Ecto.Query.t() - def where_address_match(query, address_field, address_hash_or_hashes) do + def where_address_match_by_hash(query, address_field, address_hash_or_hashes, options) do address_hashes = List.wrap(address_hash_or_hashes) - address_ids = AddressIdToAddressHash.hashes_to_ids(address_hashes) + address_ids = AddressIdToAddressHash.hashes_to_ids(address_hashes, options) where_address_match(query, address_field, address_hashes, address_ids) end @@ -892,7 +893,7 @@ defmodule Explorer.Chain.InternalTransaction do query_to_address_hash_wrapped = __MODULE__ |> where_nonpending_operation() - |> where_address_fields_match(hash, :to) + |> where_address_fields_match(hash, :to, options) |> BlockReaderGeneral.where_block_number_in_period(from_block, to_block) |> where_is_different_from_parent_transaction() |> common_where_limit_order(paging_options) @@ -901,7 +902,7 @@ defmodule Explorer.Chain.InternalTransaction do query_from_address_hash_wrapped = __MODULE__ |> where_nonpending_operation() - |> where_address_fields_match(hash, :from_address_hash) + |> where_address_fields_match(hash, :from_address_hash, options) |> BlockReaderGeneral.where_block_number_in_period(from_block, to_block) |> where_is_different_from_parent_transaction() |> common_where_limit_order(paging_options) @@ -921,7 +922,7 @@ defmodule Explorer.Chain.InternalTransaction do else __MODULE__ |> where_nonpending_operation() - |> where_address_fields_match(hash, direction) + |> where_address_fields_match(hash, direction, options) |> BlockReaderGeneral.where_block_number_in_period(from_block, to_block) |> where_is_different_from_parent_transaction() |> common_where_limit_order(paging_options) diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index 8ee5a01ffa4..012fca533bb 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -2404,7 +2404,7 @@ defmodule Explorer.Chain.Transaction do """ @spec decode_transactions([__MODULE__.t()], boolean(), Keyword.t()) :: [nil | {:ok, String.t(), String.t(), map()}] def decode_transactions(transactions, skip_sig_provider?, opts) do - smart_contract_full_abi_map = combine_smart_contract_full_abi_map(transactions) + smart_contract_full_abi_map = combine_smart_contract_full_abi_map(transactions, opts) # first we assemble an empty methods map, so that decoded_input_data will skip ContractMethod.t() lookup and decoding empty_methods_map = @@ -2485,7 +2485,7 @@ defmodule Explorer.Chain.Transaction do defp decode_remaining_transaction({decoded, _}, _, _, _, _), do: decoded - defp combine_smart_contract_full_abi_map(transactions) do + defp combine_smart_contract_full_abi_map(transactions, opts) do # parse unique address hashes of smart-contracts from to_address and created_contract_address properties of the transactions list unique_to_address_hashes = transactions @@ -2498,14 +2498,16 @@ defmodule Explorer.Chain.Transaction do # query from the DB proxy implementation objects for those address hashes multiple_proxy_implementations = - Implementation.get_proxy_implementations_for_multiple_proxies(unique_to_address_hashes) + Implementation.get_proxy_implementations_for_multiple_proxies(unique_to_address_hashes, opts) # query from the DB address objects with smart_contract preload for all found above proxy and implementation addresses addresses_with_smart_contracts = multiple_proxy_implementations |> Enum.flat_map(fn proxy_implementations -> proxy_implementations.address_hashes end) |> Enum.concat(unique_to_address_hashes) - |> Chain.hashes_to_addresses(necessity_by_association: %{smart_contract: :optional}) + |> Chain.hashes_to_addresses( + Keyword.merge(Keyword.take(opts, [:api?]), necessity_by_association: %{smart_contract: :optional}) + ) |> Enum.into(%{}, &{&1.hash, &1}) # combine map %{proxy_address_hash => implementation address hashes} @@ -3092,19 +3094,19 @@ defmodule Explorer.Chain.Transaction do @doc """ Finds all transactions of a certain block number """ - def get_transactions_of_block_number(block_number) do + def get_transactions_of_block_number(block_number, options \\ []) do block_number |> transactions_with_block_number() - |> Repo.all() + |> Chain.select_repo(options).all() end @doc """ Finds all transactions of a certain block numbers """ - def get_transactions_of_block_numbers(block_numbers) do + def get_transactions_of_block_numbers(block_numbers, options \\ []) do block_numbers |> transactions_for_block_numbers() - |> Repo.all() + |> Chain.select_repo(options).all() end @doc """ diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index 95c12d4dcba..ea38e4cec84 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -26,6 +26,8 @@ defmodule Explorer.Etherscan do end_timestamp: nil } + @api_true [api?: true] + @doc """ Returns the maximum allowed page size number. @@ -204,7 +206,7 @@ defmodule Explorer.Etherscan do |> Enum.map(fn direction -> options |> consensus_internal_transactions_with_transactions_and_blocks_query() - |> InternalTransaction.where_address_fields_match(address_hash, direction) + |> InternalTransaction.where_address_fields_match(address_hash, direction, @api_true) |> InternalTransaction.where_is_different_from_parent_transaction() |> InternalTransaction.include_zero_value(options.include_zero_value) |> where_start_block_match_internal_transaction(options) diff --git a/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex b/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex index 499027b48b6..056ae6fc926 100644 --- a/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex +++ b/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex @@ -6,6 +6,7 @@ defmodule Explorer.Utility.AddressIdToAddressHash do use Explorer.Schema + alias Explorer.Chain alias Explorer.Chain.{Address, Hash} alias Explorer.Repo @@ -100,12 +101,13 @@ defmodule Explorer.Utility.AddressIdToAddressHash do ## Returns - The address_id if found, nil otherwise """ - @spec hash_to_id(Hash.Address.t()) :: integer() | nil - def hash_to_id(nil), do: nil + @spec hash_to_id(Hash.Address.t(), [Chain.api?()]) :: integer() | nil + def hash_to_id(hash, options \\ []) + def hash_to_id(nil, _options), do: nil - def hash_to_id(hash) do + def hash_to_id(hash, options) do [hash] - |> hashes_to_ids() + |> hashes_to_ids(options) |> List.first() end @@ -118,12 +120,12 @@ defmodule Explorer.Utility.AddressIdToAddressHash do ## Returns - A list of address ids for the matching mappings """ - @spec hashes_to_ids([Hash.Address.t()]) :: [integer()] - def hashes_to_ids(hashes) do + @spec hashes_to_ids([Hash.Address.t()], [Chain.api?()]) :: [integer()] + def hashes_to_ids(hashes, options \\ []) do __MODULE__ |> where([a], a.address_hash in ^hashes) |> select([a], a.address_id) - |> Repo.all() + |> Chain.select_repo(options).all() end @doc """ @@ -139,12 +141,14 @@ defmodule Explorer.Utility.AddressIdToAddressHash do - The address hash if found - `nil` if the id is `nil` or no mapping exists """ - @spec id_to_hash(integer() | nil) :: Hash.Address.t() | nil - def id_to_hash(nil), do: nil + @spec id_to_hash(integer() | nil, [Chain.api?()]) :: Hash.Address.t() | nil + def id_to_hash(id, options \\ []) + + def id_to_hash(nil, _options), do: nil - def id_to_hash(id) do + def id_to_hash(id, options) do [id] - |> ids_to_hashes() + |> ids_to_hashes(options) |> List.first() end @@ -157,13 +161,15 @@ defmodule Explorer.Utility.AddressIdToAddressHash do ## Returns - A list of address hashes for the matching mappings """ - @spec ids_to_hashes([integer()]) :: [Hash.Address.t()] - def ids_to_hashes([]), do: [] + @spec ids_to_hashes([integer()], [Chain.api?()]) :: [Hash.Address.t()] + def ids_to_hashes(ids, options \\ []) - def ids_to_hashes(ids) do + def ids_to_hashes([], _options), do: [] + + def ids_to_hashes(ids, options) do __MODULE__ |> where([a], a.address_id in ^ids) |> select([a], a.address_hash) - |> Repo.all() + |> Chain.select_repo(options).all() end end diff --git a/apps/indexer/lib/indexer/fetcher/on_demand/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/on_demand/internal_transaction.ex index 045c2a0b448..0c5b761da1b 100644 --- a/apps/indexer/lib/indexer/fetcher/on_demand/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/on_demand/internal_transaction.ex @@ -12,7 +12,6 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do alias Explorer.{Chain, Etherscan, PagingOptions} alias Explorer.Chain.{Block, BlockNumberHelper, Hash, InternalTransaction, Transaction} alias Explorer.Chain.Cache.BlockNumber - alias Explorer.Repo alias Explorer.Utility.{AddressIdToAddressHash, InternalTransactionsAddressPlaceholder} alias Indexer.Fetcher.InternalTransaction, as: InternalTransactionFetcher @@ -141,26 +140,26 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do |> Enum.map(&serialize/1) |> different_from_parent_transaction() |> Enum.sort_by(& &1.index) - |> join_associations(necessity_by_association) + |> join_associations(necessity_by_association, options) |> page_internal_transaction(paging_options) |> Enum.take(paging_options.page_size) - |> Repo.preload(:block) - |> InternalTransaction.preload_error() + |> Chain.select_repo(options).preload(:block) + |> InternalTransaction.preload_error(options) |> InternalTransaction.preload_transaction() |> InternalTransaction.preload_addresses(options) :ignore -> [transaction.block_number] - |> fetch_block_internal_transactions() + |> fetch_block_internal_transactions(options) |> Enum.map(&serialize/1) |> Enum.filter(&(&1.block_number == transaction.block_number and &1.transaction_index == transaction.index)) |> different_from_parent_transaction() |> Enum.sort_by(& &1.index) - |> join_associations(necessity_by_association) + |> join_associations(necessity_by_association, options) |> page_internal_transaction(paging_options) |> Enum.take(paging_options.page_size) - |> Repo.preload(:block) - |> InternalTransaction.preload_error() + |> Chain.select_repo(options).preload(:block) + |> InternalTransaction.preload_error(options) |> InternalTransaction.preload_transaction() |> InternalTransaction.preload_addresses(options) @@ -209,7 +208,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do unlimited? = Keyword.get(options, :unlimited) [block.number] - |> fetch_block_internal_transactions() + |> fetch_block_internal_transactions(options) |> Enum.map(&serialize/1) |> different_from_parent_transaction() |> filter_by_type(type_filter, call_type_filter) @@ -217,8 +216,8 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do |> page_block_internal_transaction(paging_options) |> Enum.sort_by(&{&1.transaction_index, &1.index}) |> then(&if unlimited?, do: &1, else: Enum.take(&1, paging_options.page_size)) - |> join_associations(necessity_by_association) - |> InternalTransaction.preload_error() + |> join_associations(necessity_by_association, options) + |> InternalTransaction.preload_error(options) |> InternalTransaction.preload_transaction() end end @@ -257,7 +256,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do paging_options = Keyword.get(options, :paging_options, @default_paging_options) - address_id = AddressIdToAddressHash.hash_to_id(address_hash) + address_id = AddressIdToAddressHash.hash_to_id(address_hash, options) block_number_from_paging_options = case paging_options do @@ -291,7 +290,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do index_internal_transaction_desc_order = Keyword.get(options, :index_internal_transaction_desc_order, true) address_id - |> do_fetch_for_address(max_block_number, from_block, paging_options.page_size, sum_mode, sort_direction) + |> do_fetch_for_address(max_block_number, from_block, paging_options.page_size, sum_mode, sort_direction, options) |> Enum.map(&serialize/1) |> filter_by_address(address_hash, direction) |> different_from_parent_transaction() @@ -300,23 +299,24 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do }) |> Enum.sort_by(&{&1.block_number, &1.transaction_index, &1.index}, sort_func) |> Enum.take(paging_options.page_size) - |> join_associations(necessity_by_association) - |> Repo.preload(:block) - |> InternalTransaction.preload_error() + |> join_associations(necessity_by_association, options) + |> Chain.select_repo(options).preload(:block) + |> InternalTransaction.preload_error(options) |> InternalTransaction.preload_transaction() end end - defp do_fetch_for_address(address_id, to_block, from_block, limit, sum_mode, sort_direction, acc \\ []) + defp do_fetch_for_address(address_id, to_block, from_block, limit, sum_mode, sort_direction, options, acc \\ []) - defp do_fetch_for_address(_, to_block, from_block, _, _, _, acc) - when is_integer(from_block) and is_integer(to_block) and from_block >= to_block, do: acc + defp do_fetch_for_address(_, to_block, from_block, _, _, _, _options, acc) + when is_integer(from_block) and is_integer(to_block) and from_block >= to_block, + do: acc - defp do_fetch_for_address(address_id, to_block, from_block, limit, sum_mode, sort_direction, acc) do + defp do_fetch_for_address(address_id, to_block, from_block, limit, sum_mode, sort_direction, options, acc) do internal_transactions = address_id - |> get_block_numbers_for_address(to_block, from_block, limit, sum_mode, sort_direction) - |> fetch_block_internal_transactions() + |> get_block_numbers_for_address(to_block, from_block, limit, sum_mode, sort_direction, options) + |> fetch_block_internal_transactions(options) result = Enum.concat(internal_transactions, acc) @@ -335,6 +335,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do limit - count, sum_mode, sort_direction, + options, result ) @@ -346,6 +347,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do limit - count, sum_mode, sort_direction, + options, result ) end @@ -369,7 +371,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do options = Map.merge(Etherscan.default_options(), raw_options) transaction - |> fetch_by_transaction(paging_options: %PagingOptions{page_size: options.page_size}) + |> fetch_by_transaction(paging_options: %PagingOptions{page_size: options.page_size}, api?: true) |> Enum.map(ðerscan_serialize/1) end @@ -400,12 +402,13 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do from_block: Map.get(options, :startblock), to_block: Map.get(options, :endblock), sort_direction: Map.get(options, :order_by_direction), - index_internal_transaction_desc_order: Map.get(options, :order_by_direction) != :asc + index_internal_transaction_desc_order: Map.get(options, :order_by_direction) != :asc, + api?: true ] address_hash |> fetch_by_address(prepared_options) - |> Repo.preload(:block) + |> Chain.select_repo(prepared_options).preload(:block) |> Enum.map(ðerscan_serialize/1) end @@ -427,13 +430,14 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do from_block: Map.get(options, :startblock), to_block: Map.get(options, :endblock), sort_direction: Map.get(options, :order_by_direction), - index_internal_transaction_desc_order: Map.get(options, :order_by_direction) != :asc + index_internal_transaction_desc_order: Map.get(options, :order_by_direction) != :asc, + api?: true ] prepared_options |> fetch_latest() |> different_from_parent_transaction() - |> Repo.preload(:block) + |> Chain.select_repo(prepared_options).preload(:block) |> Enum.map(ðerscan_serialize/1) end @@ -461,9 +465,9 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do defp fetch_enough(_start_number, _end_number, _count, _options, acc), do: acc - defp get_block_numbers_for_address(nil, _end_block, _start_block, _limit, _sum_mode, _order), do: [] + defp get_block_numbers_for_address(nil, _end_block, _start_block, _limit, _sum_mode, _order, _options), do: [] - defp get_block_numbers_for_address(address_id, end_block, start_block, limit, sum_mode, order) do + defp get_block_numbers_for_address(address_id, end_block, start_block, limit, sum_mode, order, options) do ranked_query = InternalTransactionsAddressPlaceholder |> where([q], q.address_id == ^address_id) @@ -547,7 +551,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do select: c.block_number ) - Repo.all(final_query) + Chain.select_repo(options).all(final_query) end defp filter_by_address(internal_transactions, address_hash, direction) do @@ -567,9 +571,9 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do end) end - defp fetch_block_internal_transactions([]), do: [] + defp fetch_block_internal_transactions([], _options), do: [] - defp fetch_block_internal_transactions(block_numbers) do + defp fetch_block_internal_transactions(block_numbers, options) do if internal_transactions_fetching_disabled?() do [] else @@ -580,7 +584,7 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do fetch_blocks_internal_transactions(block_numbers, json_rpc_named_arguments) else block_numbers - |> transactions_to_trace() + |> transactions_to_trace(options) |> fetch_transactions_internal_transactions(json_rpc_named_arguments) end end @@ -617,10 +621,10 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do # Collects traceable transactions of all the blocks with a single DB query, # preserving the order of `block_numbers` (and transaction index order within # a block). - defp transactions_to_trace(block_numbers) do + defp transactions_to_trace(block_numbers, options) do transactions_by_block_number = block_numbers - |> Transaction.get_transactions_of_block_numbers() + |> Transaction.get_transactions_of_block_numbers(options) |> Transaction.filter_non_traceable_transactions() |> Enum.group_by(& &1.block_number) @@ -760,22 +764,22 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do Stream.filter(internal_transactions, &(&1.call_type in call_types)) end - defp join_associations(records, necessity_by_association) + defp join_associations(records, necessity_by_association, options) when is_list(records) and is_map(necessity_by_association) do Enum.reduce(necessity_by_association, records, fn {association, necessity}, acc -> - join_association(acc, association, necessity) + join_association(acc, association, necessity, options) end) end - defp join_association(records, [{association, nested_preload}], :optional) + defp join_association(records, [{association, nested_preload}], :optional, options) when is_atom(association) do - Repo.preload(records, [{association, nested_preload}]) + Chain.select_repo(options).preload(records, [{association, nested_preload}]) end - defp join_association(records, [{association, nested_preload}], :required) + defp join_association(records, [{association, nested_preload}], :required, options) when is_atom(association) do records - |> Repo.preload([{association, nested_preload}]) + |> Chain.select_repo(options).preload([{association, nested_preload}]) |> Enum.filter(fn struct -> case Map.fetch(struct, association) do {:ok, value} -> not is_nil(value) @@ -784,13 +788,13 @@ defmodule Indexer.Fetcher.OnDemand.InternalTransaction do end) end - defp join_association(records, association, :optional) do - Repo.preload(records, association) + defp join_association(records, association, :optional, options) do + Chain.select_repo(options).preload(records, association) end - defp join_association(records, association, :required) do + defp join_association(records, association, :required, options) do records - |> Repo.preload(association) + |> Chain.select_repo(options).preload(association) |> Enum.filter(fn struct -> case Map.fetch(struct, association) do {:ok, value} -> diff --git a/apps/indexer/test/indexer/fetcher/on_demand/contract_creator_test.exs b/apps/indexer/test/indexer/fetcher/on_demand/contract_creator_test.exs index 2b0f35e17e4..9724725f2ac 100644 --- a/apps/indexer/test/indexer/fetcher/on_demand/contract_creator_test.exs +++ b/apps/indexer/test/indexer/fetcher/on_demand/contract_creator_test.exs @@ -67,7 +67,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do assert :ignore = ContractCreatorOnDemand.trigger_fetch( contract_address - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) end @@ -81,7 +81,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) end @@ -95,7 +95,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) end @@ -114,7 +114,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) end @@ -128,7 +128,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) end @@ -154,7 +154,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(300) @@ -197,7 +197,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(300) @@ -232,7 +232,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(300) @@ -269,7 +269,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(1200) @@ -301,7 +301,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(5400) @@ -337,7 +337,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(300) @@ -382,7 +382,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreatorTest do ContractCreatorOnDemand.trigger_fetch( contract_address |> Repo.preload([:contract_creation_transaction]) - |> Address.maybe_preload_contract_creation_internal_transaction() + |> Address.maybe_preload_contract_creation_internal_transaction([]) ) :timer.sleep(300) From 17976c6779338b2f4b5a1a0e0b5f71a331338bcd Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 4 Sep 2026 11:23:07 +0300 Subject: [PATCH 13/52] perf: Batch uncataloged token transfers scan via Migrator framework (#14774) --- apps/explorer/config/config.exs | 1 + apps/explorer/config/runtime/test.exs | 1 + apps/explorer/lib/explorer/application.ex | 1 + .../lib/explorer/chain/token_transfer.ex | 12 +- ...blocks_with_uncataloged_token_transfers.ex | 59 ++++++++++ .../explorer/chain/token_transfer_test.exs | 32 ++++- ..._with_uncataloged_token_transfers_test.exs | 98 ++++++++++++++++ apps/indexer/lib/indexer/supervisor.ex | 6 +- .../temporary/uncataloged_token_transfers.ex | 105 ----------------- .../uncataloged_token_transfers_test.exs | 110 ------------------ config/runtime.exs | 7 ++ docker-compose/envs/common-blockscout.env | 3 + 12 files changed, 206 insertions(+), 229 deletions(-) create mode 100644 apps/explorer/lib/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers.ex create mode 100644 apps/explorer/test/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers_test.exs delete mode 100644 apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex delete mode 100644 apps/indexer/test/indexer/temporary/uncataloged_token_transfers_test.exs diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index eedd6b54845..211ff4e69f9 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -144,6 +144,7 @@ for migrator <- [ Explorer.Migrator.UnescapeQuotesInTokens, Explorer.Migrator.UnescapeAmpersandsInTokens, Explorer.Migrator.SanitizeDuplicateSmartContractAdditionalSources, + Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, Explorer.Migrator.EmptyInternalTransactionsData, Explorer.Migrator.FillInternalTransactionsAddressIds ] do diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index d42b6c5cc0d..10e5825742d 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -77,6 +77,7 @@ for migrator <- [ Explorer.Migrator.UnescapeQuotesInTokens, Explorer.Migrator.UnescapeAmpersandsInTokens, Explorer.Migrator.SanitizeDuplicateSmartContractAdditionalSources, + Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, Explorer.Migrator.DeleteZeroValueInternalTransactions, Explorer.Migrator.EmptyInternalTransactionsData, Explorer.Migrator.FillInternalTransactionsAddressIds, diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 4be5c481d20..5458e303811 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -228,6 +228,7 @@ defmodule Explorer.Application do configure_mode_dependent_process(Explorer.Migrator.UnescapeQuotesInTokens, :indexer), configure_mode_dependent_process(Explorer.Migrator.UnescapeAmpersandsInTokens, :indexer), configure_mode_dependent_process(Explorer.Migrator.ReindexBlocksWithMissingTransactions, :indexer), + configure_mode_dependent_process(Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, :indexer), configure_mode_dependent_process(Explorer.Migrator.SanitizeDuplicateSmartContractAdditionalSources, :indexer), configure_mode_dependent_process(Explorer.Migrator.DeleteZeroValueInternalTransactions, :indexer), configure_mode_dependent_process(Explorer.Migrator.EmptyInternalTransactionsData, :indexer), diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 5ffe1820f6f..d29f6f394f8 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -686,21 +686,23 @@ defmodule Explorer.Chain.TokenTransfer do end @doc """ - Returns a list of block numbers token transfer `t:Log.t/0`s that don't have an - associated `t:TokenTransfer.t/0` record. + Returns a list of block numbers within the given range that contain token + transfer `t:Log.t/0`s without an associated `t:TokenTransfer.t/0` record. """ - @spec uncataloged_token_transfer_block_numbers :: {:ok, [non_neg_integer()]} - def uncataloged_token_transfer_block_numbers do + @spec uncataloged_token_transfer_block_numbers(non_neg_integer(), non_neg_integer()) :: [non_neg_integer()] + def uncataloged_token_transfer_block_numbers(from_block_number, to_block_number) do query = from(l in Log, as: :log, + where: l.block_number >= ^from_block_number, + where: l.block_number <= ^to_block_number, where: ^token_transfer_log_filter_dynamic(), where: not exists(token_transfer_exists_query()), select: l.block_number, distinct: l.block_number ) - Repo.stream_reduce(query, [], &[&1 | &2]) + Repo.all(query, timeout: :infinity) end # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity diff --git a/apps/explorer/lib/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers.ex b/apps/explorer/lib/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers.ex new file mode 100644 index 00000000000..711d87d8f73 --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers.ex @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers do + @moduledoc """ + Searches for blocks containing token transfer logs that don't have an + associated token transfer record and adds them to missing block ranges so + they are re-fetched and the missed token transfers get cataloged. + + Missed token transfers happen due to formats that weren't supported at the + time they were parsed during main indexing. The migration processes blocks + in batches from the maximum block number down to 0 and persists its + progress, so it runs to completion only once. To re-run it after a parser + update, delete the "reindex_blocks_with_uncataloged_token_transfers" record + from the `migration_status` table. + """ + + use Explorer.Migrator.FillingMigration + + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.TokenTransfer + alias Explorer.Migrator.FillingMigration + alias Explorer.Utility.MissingBlockRange + + @migration_name "reindex_blocks_with_uncataloged_token_transfers" + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers(%{"max_block_number" => -1} = state), do: {[], state} + + def last_unprocessed_identifiers(%{"max_block_number" => from_block_number} = state) do + limit = batch_size() * concurrency() + to_block_number = max(from_block_number - limit + 1, 0) + + {Enum.to_list(from_block_number..to_block_number//-1), %{state | "max_block_number" => to_block_number - 1}} + end + + def last_unprocessed_identifiers(state) do + state + |> Map.put("max_block_number", BlockNumber.get_max()) + |> last_unprocessed_identifiers() + end + + @impl FillingMigration + def unprocessed_data_query, do: nil + + @impl FillingMigration + def update_batch(block_numbers) do + {min_block_number, max_block_number} = Enum.min_max(block_numbers) + + case TokenTransfer.uncataloged_token_transfer_block_numbers(min_block_number, max_block_number) do + [] -> :ok + uncataloged_block_numbers -> MissingBlockRange.add_ranges_by_block_numbers(uncataloged_block_numbers) + end + end + + @impl FillingMigration + def update_cache, do: :ok +end diff --git a/apps/explorer/test/explorer/chain/token_transfer_test.exs b/apps/explorer/test/explorer/chain/token_transfer_test.exs index bd4b67a2c63..ca78f06979c 100644 --- a/apps/explorer/test/explorer/chain/token_transfer_test.exs +++ b/apps/explorer/test/explorer/chain/token_transfer_test.exs @@ -330,7 +330,7 @@ defmodule Explorer.Chain.TokenTransferTest do end end - describe "uncataloged_token_transfer_block_numbers/0" do + describe "uncataloged_token_transfer_block_numbers/2" do test "returns a list of block numbers" do block = insert(:block) address = insert(:address) @@ -351,7 +351,31 @@ defmodule Explorer.Chain.TokenTransferTest do ) block_number = log.block_number - assert {:ok, [^block_number]} = TokenTransfer.uncataloged_token_transfer_block_numbers() + assert [^block_number] = TokenTransfer.uncataloged_token_transfer_block_numbers(block_number, block_number) + end + + test "does not return block numbers outside the given range" do + block = insert(:block) + address = insert(:address) + + log = + insert(:token_transfer_log, + transaction: + insert(:transaction, + block_number: block.number, + block_hash: block.hash, + cumulative_gas_used: 0, + gas_used: 0, + index: 0 + ), + block: block, + address_hash: address.hash, + address: address + ) + + block_number = log.block_number + assert [] = TokenTransfer.uncataloged_token_transfer_block_numbers(block_number + 1, block_number + 100) + assert [] = TokenTransfer.uncataloged_token_transfer_block_numbers(max(block_number - 100, 0), block_number - 1) end end @@ -364,7 +388,7 @@ defmodule Explorer.Chain.TokenTransferTest do ) block_number = log.block_number - assert {:ok, [^block_number]} = TokenTransfer.uncataloged_token_transfer_block_numbers() + assert [^block_number] = TokenTransfer.uncataloged_token_transfer_block_numbers(block_number, block_number) end test "does not return block numbers when matching token transfer exists for Celo epoch blocks" do @@ -398,7 +422,7 @@ defmodule Explorer.Chain.TokenTransferTest do to_address: to_address ) - assert {:ok, []} = TokenTransfer.uncataloged_token_transfer_block_numbers() + assert [] = TokenTransfer.uncataloged_token_transfer_block_numbers(log.block_number, log.block_number) end end diff --git a/apps/explorer/test/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers_test.exs b/apps/explorer/test/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers_test.exs new file mode 100644 index 00000000000..62906d12eec --- /dev/null +++ b/apps/explorer/test/explorer/migrator/reindex_blocks_with_uncataloged_token_transfers_test.exs @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfersTest do + use Explorer.DataCase, async: false + + alias Explorer.Migrator.{MigrationStatus, ReindexBlocksWithUncatalogedTokenTransfers} + alias Explorer.Repo + alias Explorer.Utility.MissingBlockRange + + setup do + Repo.delete_all( + from(ms in MigrationStatus, where: ms.migration_name == ^"reindex_blocks_with_uncataloged_token_transfers") + ) + + configuration = Application.get_env(:explorer, ReindexBlocksWithUncatalogedTokenTransfers) + + Application.put_env( + :explorer, + ReindexBlocksWithUncatalogedTokenTransfers, + Keyword.merge(configuration || [], batch_size: 100, concurrency: 1) + ) + + on_exit(fn -> + Application.put_env(:explorer, ReindexBlocksWithUncatalogedTokenTransfers, configuration) + end) + end + + test "adds blocks with uncataloged token transfers to missing block ranges" do + uncataloged_block = insert(:block) + address = insert(:address) + + insert(:token_transfer_log, + transaction: + insert(:transaction, + block_number: uncataloged_block.number, + block_hash: uncataloged_block.hash, + cumulative_gas_used: 0, + gas_used: 0, + index: 0 + ), + block: uncataloged_block, + block_number: uncataloged_block.number, + address_hash: address.hash, + address: address + ) + + cataloged_block = insert(:block) + + cataloged_transaction = + insert(:transaction, + block_number: cataloged_block.number, + block_hash: cataloged_block.hash, + cumulative_gas_used: 0, + gas_used: 0, + index: 0 + ) + + cataloged_log = + insert(:token_transfer_log, + transaction: cataloged_transaction, + block: cataloged_block, + block_number: cataloged_block.number, + address_hash: address.hash, + address: address + ) + + insert(:token_transfer, + transaction: cataloged_transaction, + block: cataloged_block, + block_number: cataloged_block.number, + log_index: cataloged_log.index + ) + + assert MigrationStatus.get_status("reindex_blocks_with_uncataloged_token_transfers") == nil + + ReindexBlocksWithUncatalogedTokenTransfers.start_link([]) + + wait_for_results( + fn -> + Repo.one!( + from(ms in MigrationStatus, + where: ms.migration_name == ^"reindex_blocks_with_uncataloged_token_transfers" and ms.status == "completed" + ) + ) + end, + 60 + ) + + ranges = Repo.all(MissingBlockRange) + + assert Enum.any?(ranges, fn range -> + range.from_number >= uncataloged_block.number and range.to_number <= uncataloged_block.number + end) + + refute Enum.any?(ranges, fn range -> + range.from_number >= cataloged_block.number and range.to_number <= cataloged_block.number + end) + end +end diff --git a/apps/indexer/lib/indexer/supervisor.ex b/apps/indexer/lib/indexer/supervisor.ex index 642e15e5998..ce4abe299da 100644 --- a/apps/indexer/lib/indexer/supervisor.ex +++ b/apps/indexer/lib/indexer/supervisor.ex @@ -72,10 +72,7 @@ defmodule Indexer.Supervisor do alias Indexer.Migrator.RecoveryWETHTokenTransfers - alias Indexer.Temporary.{ - UncatalogedTokenTransfers, - UnclesWithoutIndex - } + alias Indexer.Temporary.UnclesWithoutIndex alias Indexer.Utils.EventNotificationsCleaner @@ -279,7 +276,6 @@ defmodule Indexer.Supervisor do configure(EventNotificationsCleaner, [[]]), # Temporary workers - {UncatalogedTokenTransfers.Supervisor, [[]]}, {UnclesWithoutIndex.Supervisor, [[json_rpc_named_arguments: json_rpc_named_arguments, memory_monitor: memory_monitor]]}, {PendingOpsCleaner, [[], []]}, diff --git a/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex b/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex deleted file mode 100644 index a6cca831def..00000000000 --- a/apps/indexer/lib/indexer/temporary/uncataloged_token_transfers.ex +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Indexer.Temporary.UncatalogedTokenTransfers do - @moduledoc """ - Catalogs token transfer logs missing an accompanying token transfer record. - - Missed token transfers happen due to formats that aren't supported at the time - they were parsed during main indexing. Updated the parser and rebooting will allow - this process to properly catalog those missed token transfers. - """ - - use GenServer - use Indexer.Fetcher - - require Logger - - alias Explorer.Chain.TokenTransfer - alias Explorer.Utility.MissingBlockRange - alias Indexer.Temporary.UncatalogedTokenTransfers - - def child_spec([init_arguments]) do - child_spec([init_arguments, []]) - end - - def child_spec([_init_arguments, _gen_server_options] = start_link_arguments) do - spec = %{ - id: __MODULE__, - start: {__MODULE__, :start_link, start_link_arguments}, - restart: :transient, - type: :worker - } - - Supervisor.child_spec(spec, []) - end - - def start_link(init_arguments, gen_server_options \\ []) do - GenServer.start_link(__MODULE__, init_arguments, gen_server_options) - end - - def init(opts) do - sup_pid = Keyword.fetch!(opts, :supervisor) - retry_interval = Keyword.get(opts, :retry_interval, 10_000) - - send(self(), :scan) - - state = %{ - block_numbers: [], - retry_interval: retry_interval, - sup_pid: sup_pid, - task_ref: nil - } - - {:ok, state} - end - - def handle_info(:scan, state) do - {:ok, block_numbers} = TokenTransfer.uncataloged_token_transfer_block_numbers() - - case block_numbers do - [] -> - Supervisor.stop(state.sup_pid, :normal) - {:noreply, state} - - block_numbers -> - Process.send_after(self(), :push_front_blocks, state.retry_interval) - {:noreply, %{state | block_numbers: block_numbers}} - end - end - - def handle_info(:push_front_blocks, %{block_numbers: block_numbers} = state) do - %Task{ref: ref} = async_push_front(block_numbers) - {:noreply, %{state | task_ref: ref}} - end - - def handle_info({ref, :ok}, %{task_ref: ref, sup_pid: sup_pid}) do - Process.demonitor(ref, [:flush]) - Supervisor.stop(sup_pid, :normal) - {:stop, :shutdown} - end - - def handle_info({ref, {:error, reason}}, %{task_ref: ref, retry_interval: millis} = state) do - case reason do - :queue_unavailable -> :ok - _ -> Logger.error(fn -> inspect(reason) end) - end - - Process.demonitor(ref, [:flush]) - Process.send_after(self(), :push_front_blocks, millis) - - {:noreply, %{state | task_ref: nil}} - end - - def handle_info({:DOWN, ref, :process, _, _}, %{task_ref: ref, retry_interval: millis} = state) do - Process.send_after(self(), :push_front_blocks, millis) - {:noreply, %{state | task_ref: nil}} - end - - defp async_push_front(block_numbers) do - Task.Supervisor.async_nolink( - UncatalogedTokenTransfers.TaskSupervisor, - MissingBlockRange, - :add_ranges_by_block_numbers, - [block_numbers] - ) - end -end diff --git a/apps/indexer/test/indexer/temporary/uncataloged_token_transfers_test.exs b/apps/indexer/test/indexer/temporary/uncataloged_token_transfers_test.exs deleted file mode 100644 index 830196857a6..00000000000 --- a/apps/indexer/test/indexer/temporary/uncataloged_token_transfers_test.exs +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Indexer.Temporary.UncatalogedTokenTransfersTest do - use Explorer.DataCase - - alias Indexer.Temporary.UncatalogedTokenTransfers - - @moduletag :capture_log - - describe "start_link/1" do - test "starts the worker" do - assert {:ok, _pid} = UncatalogedTokenTransfers.start_link(supervisor: self()) - end - end - - describe "init/1" do - test "sends message to self" do - pid = self() - - assert {:ok, %{task_ref: nil, block_numbers: [], sup_pid: ^pid}} = - UncatalogedTokenTransfers.init(supervisor: self()) - - assert_received :scan - end - end - - describe "handle_info with :scan" do - test "sends shutdown to supervisor" do - state = %{task_ref: nil, block_numbers: [], sup_pid: self()} - Task.async(fn -> UncatalogedTokenTransfers.handle_info(:scan, state) end) - assert_receive {_, _, {:terminate, :normal}}, 200 - end - - test "sends message to self when uncataloged token transfers are found" do - block = insert(:block) - address = insert(:address) - - transaction = - insert(:transaction, - block_number: block.number, - block_hash: block.hash, - cumulative_gas_used: 0, - gas_used: 0, - index: 0 - ) - - log = - insert(:token_transfer_log, - transaction: transaction, - address_hash: address.hash, - address: address, - block: block - ) - - block_number = log.block_number - - expected_state = %{task_ref: nil, block_numbers: [block_number], retry_interval: 1} - state = %{task_ref: nil, block_numbers: [], retry_interval: 1} - - assert {:noreply, ^expected_state} = UncatalogedTokenTransfers.handle_info(:scan, state) - assert_receive :push_front_blocks - end - end - - describe "handle_info with :push_front_blocks" do - test "starts a task" do - task_sup_pid = start_supervised!({Task.Supervisor, name: UncatalogedTokenTransfers.TaskSupervisor}) - - state = %{task_ref: nil, block_numbers: [1]} - assert {:noreply, %{task_ref: task_ref}} = UncatalogedTokenTransfers.handle_info(:push_front_blocks, state) - assert is_reference(task_ref) - - refute_receive {^task_ref, {:error, :queue_unavailable}} - assert_receive {^task_ref, :ok} - - stop_supervised(task_sup_pid) - end - end - - describe "handle_info with task ref tuple" do - test "sends shutdown to supervisor on success" do - ref = Process.monitor(self()) - state = %{task_ref: ref, block_numbers: [], sup_pid: self()} - Task.async(fn -> assert UncatalogedTokenTransfers.handle_info({ref, :ok}, state) end) - assert_receive {_, _, {:terminate, :normal}} - end - - test "sends message to self to try again on failure" do - ref = Process.monitor(self()) - state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1} - expected_state = %{state | task_ref: nil} - - assert {:noreply, ^expected_state} = - UncatalogedTokenTransfers.handle_info({ref, {:error, :queue_unavailable}}, state) - - assert_receive :push_front_blocks - end - end - - describe "handle_info with failed task" do - test "sends message to self to try again" do - ref = Process.monitor(self()) - state = %{task_ref: ref, block_numbers: [1], sup_pid: self(), retry_interval: 1} - - assert {:noreply, %{task_ref: nil}} = - UncatalogedTokenTransfers.handle_info({:DOWN, ref, :process, self(), :EXIT}, state) - - assert_receive :push_front_blocks - end - end -end diff --git a/config/runtime.exs b/config/runtime.exs index d239dc0251e..2181b9bc333 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -851,6 +851,13 @@ config :explorer, Explorer.Migrator.ReindexBlocksWithMissingTransactions, timeout: ConfigHelper.parse_time_env_var("MIGRATION_REINDEX_BLOCKS_WITH_MISSING_TRANSACTIONS_TIMEOUT", "0s"), enabled: ConfigHelper.parse_bool_env_var("MIGRATION_REINDEX_BLOCKS_WITH_MISSING_TRANSACTIONS_ENABLED", "false") +config :explorer, Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, + batch_size: + ConfigHelper.parse_integer_env_var("MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_BATCH_SIZE", 1000), + concurrency: + ConfigHelper.parse_integer_env_var("MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_CONCURRENCY", 1), + timeout: ConfigHelper.parse_time_env_var("MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_TIMEOUT", "0s") + config :explorer, Explorer.Migrator.RestoreOmittedWETHTransfers, concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_CONCURRENCY", 5), batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_RESTORE_OMITTED_WETH_TOKEN_TRANSFERS_BATCH_SIZE", 50), diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index f148615eeb1..3284431f5d9 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -580,6 +580,9 @@ ACCOUNT_REDIS_URL=redis://redis-db:6379 # MIGRATION_REINDEX_BLOCKS_WITH_MISSING_TRANSACTIONS_CONCURRENCY= # MIGRATION_REINDEX_BLOCKS_WITH_MISSING_TRANSACTIONS_TIMEOUT= # MIGRATION_REINDEX_BLOCKS_WITH_MISSING_TRANSACTIONS_ENABLED= +# MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_BATCH_SIZE= +# MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_CONCURRENCY= +# MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_TIMEOUT= # MIGRATION_SHRINK_INTERNAL_TRANSACTIONS_BATCH_SIZE= # MIGRATION_SHRINK_INTERNAL_TRANSACTIONS_CONCURRENCY= # MIGRATION_ARBITRUM_DA_RECORDS_NORMALIZATION_BATCH_SIZE= From 8c191ddd8a6ca795849ab5da918877c09a36d57b Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 4 Sep 2026 14:43:50 +0300 Subject: [PATCH 14/52] perf: Switch address counters to incremental consolidation (#14759) --- .../controllers/address_controller.ex | 18 +- .../controllers/api/v2/address_controller.ex | 17 +- .../api/v2/address_controller_test.exs | 7 +- apps/explorer/config/config.exs | 17 +- apps/explorer/config/runtime/test.exs | 4 + apps/explorer/lib/explorer/application.ex | 6 +- apps/explorer/lib/explorer/chain/address.ex | 3 +- .../lib/explorer/chain/address/counters.ex | 222 ++---- .../chain/cache/background_migrations.ex | 9 + .../chain/cache/counters/address_counters.ex | 525 +++++++++++++ .../counters/address_counters_consolidator.ex | 721 ++++++++++++++++++ .../counters/address_token_transfers_count.ex | 86 --- .../counters/address_transactions_count.ex | 86 --- .../address_transactions_gas_usage_sum.ex | 88 --- .../explorer/chain/cache/counters/helper.ex | 16 +- apps/explorer/lib/explorer/chain/import.ex | 55 ++ .../explorer/chain/import/runner/blocks.ex | 157 +++- .../migrator/backfill_address_counters.ex | 107 +++ .../utility/address_counters_refetch_block.ex | 52 ++ ...0_add_counters_updated_at_to_addresses.exs | 10 + ...create_address_counters_refetch_blocks.exs | 12 + .../address_counters_consolidator_test.exs | 290 +++++++ .../cache/counters/address_counters_test.exs | 244 ++++++ .../chain/import/runner/blocks_test.exs | 94 ++- .../backfill_address_counters_test.exs | 105 +++ config/runtime.exs | 25 +- docker-compose/envs/common-blockscout.env | 15 +- 27 files changed, 2522 insertions(+), 469 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/counters/address_counters.ex create mode 100644 apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/address_token_transfers_count.ex delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/address_transactions_count.ex delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/address_transactions_gas_usage_sum.ex create mode 100644 apps/explorer/lib/explorer/migrator/backfill_address_counters.ex create mode 100644 apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex create mode 100644 apps/explorer/priv/repo/migrations/20260831000000_add_counters_updated_at_to_addresses.exs create mode 100644 apps/explorer/priv/repo/migrations/20260831000001_create_address_counters_refetch_blocks.exs create mode 100644 apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs create mode 100644 apps/explorer/test/explorer/chain/cache/counters/address_counters_test.exs create mode 100644 apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex index 3d936fffece..f68551b900a 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/address_controller.ex @@ -19,7 +19,7 @@ defmodule BlockScoutWeb.AddressController do alias Explorer.{Chain, Market} alias Explorer.Chain.{Address, Wei} alias Explorer.Chain.Address.Counters - alias Explorer.Chain.Cache.Counters.AddressesCount + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressesCount} alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand alias Indexer.Fetcher.OnDemand.ContractCode, as: ContractCodeOnDemand alias Phoenix.View @@ -176,16 +176,18 @@ defmodule BlockScoutWeb.AddressController do def address_counters(conn, %{"id" => address_hash_string}) do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, address} <- Chain.hash_to_address(address_hash) do - {validation_count} = Counters.address_counters(address) + validation_count = Counters.address_to_validation_count(address.hash, []) - transactions_from_db = address.transactions_count || 0 - token_transfers_from_db = address.token_transfers_count || 0 - address_gas_usage_from_db = address.gas_used || 0 + %{ + transactions_count: transactions_count, + token_transfers_count: token_transfers_count, + gas_used: gas_used + } = AddressCounters.fetch(address) json(conn, %{ - transaction_count: transactions_from_db, - token_transfer_count: token_transfers_from_db, - gas_usage_count: address_gas_usage_from_db, + transaction_count: transactions_count, + token_transfer_count: token_transfers_count, + gas_usage_count: gas_used, validation_count: validation_count }) else diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex index 0a5f6c3d041..f72534d4257 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/address_controller.ex @@ -49,6 +49,7 @@ defmodule BlockScoutWeb.API.V2.AddressController do alias Explorer.{Chain, Market, PagingOptions} alias Explorer.Chain.{Address, Beacon.Deposit, Block, Hash, Token, Transaction} alias Explorer.Chain.Address.{CoinBalance, Counters} + alias Explorer.Chain.Cache.Counters.AddressCounters alias Explorer.Chain.Token.FiatValue alias Explorer.Chain.Token.Instance @@ -290,16 +291,18 @@ defmodule BlockScoutWeb.API.V2.AddressController do with {:ok, address_hash} <- validate_address_hash(address_hash_string, params) do case Chain.hash_to_address(address_hash, Keyword.merge(@api_true, necessity_by_association: %{})) do {:ok, address} -> - {validation_count} = Counters.address_counters(address, @api_true) + validation_count = Counters.address_to_validation_count(address.hash, @api_true) - transactions_from_db = address.transactions_count || 0 - token_transfers_from_db = address.token_transfers_count || 0 - address_gas_usage_from_db = address.gas_used || 0 + %{ + transactions_count: transactions_count, + token_transfers_count: token_transfers_count, + gas_used: gas_used + } = AddressCounters.fetch(address) json(conn, %{ - transactions_count: to_string(transactions_from_db), - token_transfers_count: to_string(token_transfers_from_db), - gas_usage_count: to_string(address_gas_usage_from_db), + transactions_count: to_string(transactions_count), + token_transfers_count: to_string(token_transfers_count), + gas_usage_count: to_string(gas_used), validations_count: to_string(validation_count) }) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs index 38f6f211ea7..5e0f0f99dfb 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/address_controller_test.exs @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.API.V2.AddressControllerTest do alias ABI.{TypeDecoder, TypeEncoder} alias Explorer.{Chain, Repo, TestHelper} - alias Explorer.Chain.Address.Counters + alias Explorer.Chain.Cache.Counters.AddressCountersConsolidator alias Explorer.Chain.{ Address, @@ -608,9 +608,8 @@ defmodule BlockScoutWeb.API.V2.AddressControllerTest do insert(:block, miner: address) - Counters.transactions_count(address) - Counters.token_transfers_count(address) - Counters.gas_usage_count(address) + safe_block = Repo.aggregate(Block, :max, :number) + AddressCountersConsolidator.consolidate_addresses([address.hash], safe_block) request = get(conn, "/api/v2/addresses/#{address.hash}/counters") json_response = json_response(request, 200) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 211ff4e69f9..11dfd949dc5 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -37,9 +37,9 @@ config :explorer, Explorer.Chain.Cache.Counters.AddressesCount, enabled: true, enable_consolidation: true -config :explorer, Explorer.Chain.Cache.Counters.AddressTransactionsGasUsageSum, - enabled: true, - enable_consolidation: true +config :explorer, Explorer.Chain.Cache.Counters.AddressCounters, enabled: true + +config :explorer, Explorer.Chain.Cache.Counters.AddressCountersConsolidator, enabled: true config :explorer, Explorer.Chain.Cache.Counters.AddressTokensUsdSum, enabled: true, @@ -92,14 +92,6 @@ config :explorer, Explorer.Chain.Cache.Counters.TokenTransfersCount, enabled: true, enable_consolidation: true -config :explorer, Explorer.Chain.Cache.Counters.AddressTransactionsCount, - enabled: true, - enable_consolidation: true - -config :explorer, Explorer.Chain.Cache.Counters.AddressTokenTransfersCount, - enabled: true, - enable_consolidation: true - config :explorer, Explorer.Chain.Cache.Counters.BlockBurntFeeCount, enabled: true, enable_consolidation: true @@ -146,7 +138,8 @@ for migrator <- [ Explorer.Migrator.SanitizeDuplicateSmartContractAdditionalSources, Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, Explorer.Migrator.EmptyInternalTransactionsData, - Explorer.Migrator.FillInternalTransactionsAddressIds + Explorer.Migrator.FillInternalTransactionsAddressIds, + Explorer.Migrator.BackfillAddressCounters ] do config :explorer, migrator, enabled: true end diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 10e5825742d..84e068f714e 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -15,6 +15,9 @@ config :explorer, Explorer.Chain.Cache.Counters.AverageBlockTime, enabled: false # This causes a ConnectionOwnership error config :explorer, Explorer.Chain.Transaction.History.Historian, enabled: false +config :explorer, Explorer.Chain.Cache.Counters.AddressCounters, enabled: false +config :explorer, Explorer.Chain.Cache.Counters.AddressCountersConsolidator, enabled: false + for counter <- [ Explorer.Chain.Cache.Counters.AddressesCount, Explorer.Chain.Cache.Counters.Optimism.LastOutputRootSizeCount, @@ -81,6 +84,7 @@ for migrator <- [ Explorer.Migrator.DeleteZeroValueInternalTransactions, Explorer.Migrator.EmptyInternalTransactionsData, Explorer.Migrator.FillInternalTransactionsAddressIds, + Explorer.Migrator.BackfillAddressCounters, # Heavy DB index operations Explorer.Migrator.HeavyDbIndexOperation.CreateLogsBlockHashIndex, diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 5458e303811..ee94f58bba9 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -152,9 +152,8 @@ defmodule Explorer.Application do configure_mode_dependent_process(Explorer.Chain.Transaction.History.Historian, :indexer), configure(Explorer.Chain.Events.Listener), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressesCount, :api), - configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressTransactionsCount, :api), - configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressTokenTransfersCount, :api), - configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressTransactionsGasUsageSum, :api), + configure(Explorer.Chain.Cache.Counters.AddressCounters), + configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressCountersConsolidator, :indexer), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressTokensUsdSum, :api), configure(Explorer.Chain.Cache.Counters.TokenHoldersCount), configure(Explorer.Chain.Cache.Counters.TokenTransfersCount), @@ -190,6 +189,7 @@ defmodule Explorer.Application do configure_mode_dependent_process(Explorer.Migrator.TokenTransferBlockConsensus, :indexer), configure_mode_dependent_process(Explorer.Migrator.RestoreOmittedWETHTransfers, :indexer), configure_mode_dependent_process(Explorer.Migrator.FilecoinPendingAddressOperations, :indexer), + configure_mode_dependent_process(Explorer.Migrator.BackfillAddressCounters, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloL2Epochs, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloAccounts, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloAggregatedElectionRewards, :indexer), diff --git a/apps/explorer/lib/explorer/chain/address.ex b/apps/explorer/lib/explorer/chain/address.ex index cbfc7bfa74e..fc115c80cac 100644 --- a/apps/explorer/lib/explorer/chain/address.ex +++ b/apps/explorer/lib/explorer/chain/address.ex @@ -136,6 +136,7 @@ defmodule Explorer.Chain.Address.Schema do field(:transactions_count, :integer) field(:token_transfers_count, :integer) field(:gas_used, :integer) + field(:counters_updated_at, :integer) :: Block.block_number() | nil field(:ens_domain_name, :string, virtual: true) field(:metadata, :any, virtual: true) field(:contract_creation_internal_transaction, :map, virtual: true) @@ -191,7 +192,7 @@ defmodule Explorer.Chain.Address do import Explorer.Chain.SmartContract.Proxy.Models.Implementation, only: [proxy_implementations_association: 0] - @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce verified gas_used transactions_count token_transfers_count)a + @optional_attrs ~w(contract_code fetched_coin_balance fetched_coin_balance_block_number nonce verified gas_used transactions_count token_transfers_count counters_updated_at)a @chain_type_optional_attrs (case @chain_type do :filecoin -> ~w(filecoin_id filecoin_robust filecoin_actor_type)a diff --git a/apps/explorer/lib/explorer/chain/address/counters.ex b/apps/explorer/lib/explorer/chain/address/counters.ex index 4fd8adefdad..82734d4b087 100644 --- a/apps/explorer/lib/explorer/chain/address/counters.ex +++ b/apps/explorer/lib/explorer/chain/address/counters.ex @@ -11,14 +11,9 @@ defmodule Explorer.Chain.Address.Counters do import Explorer.Chain, only: [select_repo: 1, wrapped_union_subquery: 1] - alias Explorer.{Chain, PagingOptions, Repo} + alias Explorer.{Chain, PagingOptions} - alias Explorer.Chain.Cache.Counters.{ - AddressTabsElementsCount, - AddressTokenTransfersCount, - AddressTransactionsCount, - AddressTransactionsGasUsageSum - } + alias Explorer.Chain.Cache.Counters.AddressTabsElementsCount alias Explorer.Chain.{ Address, @@ -117,23 +112,22 @@ defmodule Explorer.Chain.Address.Counters do select_repo(options).one(query) end - def address_hash_to_transaction_count_query(address_hash) do + @doc """ + Builds a query for collated transactions sent from or received by the given + address, optionally bounded by a `(from_block_number, to_block_number]` + block range (either bound may be `nil` to leave that side open). + """ + @spec address_hash_to_transaction_count_query( + Hash.Address.t(), + Block.block_number() | nil, + Block.block_number() | nil + ) :: Ecto.Query.t() + def address_hash_to_transaction_count_query(address_hash, from_block_number \\ nil, to_block_number \\ nil) do dynamic = Transaction.where_transactions_to_from(address_hash) Transaction |> where([transaction], ^dynamic) - end - - @spec address_hash_to_transaction_count(Hash.Address.t()) :: non_neg_integer() - def address_hash_to_transaction_count(address_hash) do - query = address_hash_to_transaction_count_query(address_hash) - - Repo.aggregate(query, :count, :hash, timeout: :infinity) - end - - @spec address_to_transaction_count(Address.t()) :: non_neg_integer() - def address_to_transaction_count(address) do - address_hash_to_transaction_count(address.hash) + |> where_block_number_in_range(from_block_number, to_block_number) end @doc """ @@ -147,67 +141,68 @@ defmodule Explorer.Chain.Address.Counters do end @doc """ - Calculates the total gas used by incoming transactions to a given address. + Builds a query for the transactions contributing to the gas usage sum of the + given address, optionally bounded by a `(from_block_number, to_block_number]` + block range. - This function queries the database for all transactions where the - `to_address_hash` matches the provided `address_hash`, and sums up the - `gas_used` for these transactions. - - ## Parameters - - `address_hash`: The address hash to query for incoming transactions. - - ## Returns - - The total gas used by incoming transactions, or `nil` if no transactions - are found or if the sum is null. + `direction_field` selects which side of the transaction is attributed to the + address (see `gas_usage_direction_field/1`). """ - @spec address_to_incoming_transaction_gas_usage(Hash.Address.t()) :: Decimal.t() | nil - def address_to_incoming_transaction_gas_usage(address_hash) do - to_address_query = - from( - transaction in Transaction, - where: transaction.to_address_hash == ^address_hash - ) - - Repo.aggregate(to_address_query, :sum, :gas_used, timeout: :infinity) + @spec address_to_gas_usage_sum_query( + Hash.Address.t(), + :to_address_hash | :from_address_hash, + Block.block_number() | nil, + Block.block_number() | nil + ) :: Ecto.Query.t() + def address_to_gas_usage_sum_query(address_hash, direction_field, from_block_number \\ nil, to_block_number \\ nil) do + Transaction + |> where([transaction], field(transaction, ^direction_field) == ^address_hash) + |> where_block_number_in_range(from_block_number, to_block_number) end @doc """ - Calculates the total gas used by outgoing transactions from a given address. - - This function queries the database for all transactions where the - `from_address_hash` matches the provided `address_hash`, and sums up the - `gas_used` for these transactions. - - ## Parameters - - `address_hash`: the address to query. + Returns the transaction side whose `gas_used` is accumulated into the + address gas usage counter: incoming transactions for smart contracts, + outgoing transactions for EOAs (including EOAs with delegated code). - ## Returns - - The total gas used, or `nil` if no transactions are found or if the sum is null. + The address must have `contract_code` loaded. """ - @spec address_to_outcoming_transaction_gas_usage(Hash.Address.t()) :: Decimal.t() | nil - def address_to_outcoming_transaction_gas_usage(address_hash) do - to_address_query = - from( - transaction in Transaction, - where: transaction.from_address_hash == ^address_hash - ) - - Repo.aggregate(to_address_query, :sum, :gas_used, timeout: :infinity) + @spec gas_usage_direction_field(Address.t()) :: :to_address_hash | :from_address_hash + def gas_usage_direction_field(address) do + if Address.smart_contract?(address) && !Address.eoa_with_code?(address) do + :to_address_hash + else + :from_address_hash + end end - def address_to_token_transfer_count_query(address_hash) do - from( - token_transfer in TokenTransfer, - where: token_transfer.to_address_hash == ^address_hash, - or_where: token_transfer.from_address_hash == ^address_hash + @doc """ + Builds a query for token transfers sent from or received by the given + address, optionally bounded by a `(from_block_number, to_block_number]` + block range. + """ + @spec address_to_token_transfer_count_query( + Hash.Address.t(), + Block.block_number() | nil, + Block.block_number() | nil + ) :: Ecto.Query.t() + def address_to_token_transfer_count_query(address_hash, from_block_number \\ nil, to_block_number \\ nil) do + TokenTransfer + |> where( + [token_transfer], + token_transfer.to_address_hash == ^address_hash or token_transfer.from_address_hash == ^address_hash ) + |> where_block_number_in_range(from_block_number, to_block_number) end - @spec address_to_token_transfer_count(Address.t()) :: non_neg_integer() - def address_to_token_transfer_count(address) do - query = address_to_token_transfer_count_query(address.hash) - - Repo.aggregate(query, :count, timeout: :infinity) + defp where_block_number_in_range(query, from_block_number, to_block_number) do + query + |> then(fn q -> + if is_nil(from_block_number), do: q, else: where(q, [t], t.block_number > ^from_block_number) + end) + |> then(fn q -> + if is_nil(to_block_number), do: q, else: where(q, [t], t.block_number <= ^to_block_number) + end) end def address_hash_to_token_balances_query(address_hash) do @@ -218,46 +213,6 @@ defmodule Explorer.Chain.Address.Counters do ) end - @doc """ - Calculates the total gas usage for a given address. - - This function determines the appropriate gas usage calculation based on the - address type: - - - For smart contracts (excluding EOAs with code), it first checks the gas - usage of incoming transactions. If there are no incoming transactions or - their gas usage is zero, it falls back to the gas usage of outgoing - transactions. - - For regular addresses and EOAs with code, it calculates the gas usage of - outgoing transactions. - - ## Parameters - - `address`: The address to calculate gas usage for. - - ## Returns - - The total gas usage for the address. - - `nil` if no relevant transactions are found or if the sum is null. - """ - @spec address_to_gas_usage_count(Address.t()) :: Decimal.t() | nil - def address_to_gas_usage_count(address) do - if Address.smart_contract?(address) and not Address.eoa_with_code?(address) do - incoming_transaction_gas_usage = address_to_incoming_transaction_gas_usage(address.hash) - - cond do - !incoming_transaction_gas_usage -> - address_to_outcoming_transaction_gas_usage(address.hash) - - Decimal.compare(incoming_transaction_gas_usage, 0) == :eq -> - address_to_outcoming_transaction_gas_usage(address.hash) - - true -> - incoming_transaction_gas_usage - end - else - address_to_outcoming_transaction_gas_usage(address.hash) - end - end - defp address_hash_to_internal_transactions_limited_count_query(address_hash, options) do query_to_address_hash_wrapped = InternalTransaction @@ -287,55 +242,6 @@ defmodule Explorer.Chain.Address.Counters do ) end - def address_counters(address, options \\ []) do - validation_count_task = - Task.async(fn -> - address_to_validation_count(address.hash, options) - end) - - Task.start_link(fn -> - transactions_count(address) - end) - - Task.start_link(fn -> - token_transfers_count(address) - end) - - Task.start_link(fn -> - gas_usage_count(address) - end) - - [ - validation_count_task - ] - |> Task.yield_many(:infinity) - |> Enum.map(fn {_task, res} -> - case res do - {:ok, result} -> - result - - {:exit, reason} -> - raise "Query fetching address counters terminated: #{inspect(reason)}" - - nil -> - raise "Query fetching address counters timed out." - end - end) - |> List.to_tuple() - end - - def transactions_count(address) do - AddressTransactionsCount.fetch(address) - end - - def token_transfers_count(address) do - AddressTokenTransfersCount.fetch(address) - end - - def gas_usage_count(address) do - AddressTransactionsGasUsageSum.fetch(address) - end - @spec address_limited_counters(Hash.t(), Keyword.t()) :: %{atom() => counter} def address_limited_counters(address_hash, options) do cached_counters = diff --git a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex index bc4ac7ae245..8b3339ff0fe 100644 --- a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex +++ b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex @@ -70,6 +70,7 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do key: :heavy_indexes_create_addresses_hash_contract_code_not_null_index_finished, key: :heavy_indexes_create_address_ids_internal_transactions_indexes_finished, key: :fill_internal_transactions_address_ids_finished, + key: :backfill_address_counters_finished, key: :heavy_indexes_create_logs_address_hash_first_topic_second_topic_block_number_index_finished, key: :heavy_indexes_create_address_current_token_balances_address_hash_block_number_index_finished @@ -79,6 +80,7 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do AddressCurrentTokenBalanceTokenType, AddressTokenBalanceTokenType, ArbitrumDaRecordsNormalization, + BackfillAddressCounters, BackfillMultichainSearchDB, EmptyInternalTransactionsData, FillInternalTransactionsAddressIds, @@ -426,6 +428,13 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do ) end + defp handle_fallback(:backfill_address_counters_finished) do + set_and_return_migration_status( + BackfillAddressCounters, + &set_backfill_address_counters_finished/1 + ) + end + defp handle_fallback(:heavy_indexes_create_logs_address_hash_first_topic_second_topic_block_number_index_finished) do set_and_return_migration_status( CreateLogsAddressHashFirstTopicSecondTopicBlockNumberIndex, diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_counters.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_counters.ex new file mode 100644 index 00000000000..847da796d1b --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/counters/address_counters.ex @@ -0,0 +1,525 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.AddressCounters do + @moduledoc """ + Shared ETS cache for the per-address counters: transactions count, token + transfers count and gas usage sum. + + The durable values live in the `addresses` table columns + (`transactions_count`, `token_transfers_count`, `gas_used`) and are advanced + incrementally by `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` + up to the block number stored in `addresses.counters_updated_at`. This + module keeps two ETS tables on top of them: + + * `:address_counters` — the display cache consumed by the API. Entries are + seeded from the DB columns on read (`fetch/1`), incremented in place with + the deltas of newly imported transactions and token transfers + (`handle_new_data/3`), re-based from the DB by the consolidator + (`refresh_from_consolidation/4`) and evicted a configurable TTL after + their last authoritative base (seed or consolidator re-base), so any + accumulated live-bump drift is bounded by the TTL even for addresses that + are read constantly. Live bumps never create entries, so every cached + value always sits on top of a real DB-derived base. + + * `:address_counters_dirty` — the consolidator work list: address hash bytes + mapped to the maximum block number seen for the address since its last + consolidation. Markers are written for every import (realtime and catchup) + on nodes where the consolidator runs (`:indexer`/`:all` modes) and are + capped at a configurable size — a dropped marker only delays consolidation + until the address's next activity, it never corrupts the counters. + + On pure API nodes of split deployments the display cache is kept warm by the + realtime `:transactions`/`:token_transfers` chain events (dirty markers are + not written there — no local consolidator would ever drain them); on indexer + and all-mode nodes `Explorer.Chain.Import` calls `handle_new_data/3` + directly, so chain events are ignored there to avoid double counting. + """ + + use GenServer + + alias Explorer.Chain.Address.Counters + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.Helper + alias Explorer.Chain.Events.Subscriber + alias Explorer.Chain.Hash + + @cache_name :address_counters + @dirty_cache_name :address_counters_dirty + + @typedoc """ + Per-address contribution of a set of imported (or deleted) transactions and + token transfers. `gas_in`/`gas_out` are tracked separately because which one + is accumulated into the gas usage counter depends on the address type (see + `Explorer.Chain.Address.Counters.gas_usage_direction_field/1`). + """ + @type delta :: %{ + transactions_count: non_neg_integer(), + token_transfers_count: non_neg_integer(), + gas_in: non_neg_integer(), + gas_out: non_neg_integer(), + max_block_number: non_neg_integer() + } + + @type counters :: %{ + transactions_count: non_neg_integer(), + token_transfers_count: non_neg_integer(), + gas_used: non_neg_integer() + } + + @empty_delta %{transactions_count: 0, token_transfers_count: 0, gas_in: 0, gas_out: 0, max_block_number: 0} + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + Helper.create_cache_table(@cache_name, write_concurrency: true) + Helper.create_cache_table(@dirty_cache_name, write_concurrency: true) + + if Explorer.mode() == :api do + Subscriber.to(:transactions, :realtime) + Subscriber.to(:token_transfers, :realtime) + end + + schedule_eviction() + + {:ok, %{}} + end + + @doc """ + Returns the counters for the given address: the cached values when present, + the `addresses` table columns of the given struct otherwise (seeding the + cache along the way). + + An address whose counters were never calculated (all three columns are + `NULL`) is reported as all-zero and marked dirty so that the consolidator + computes its values ahead of the background backfill (on nodes with a local + consolidator; on pure API nodes such addresses stay at zero until the + backfill reaches them). + """ + @spec fetch(Explorer.Chain.Address.t()) :: counters() + def fetch(address) do + if cache_table_exists?() do + bytes = hash_bytes(address.hash) + + case snapshot(bytes) do + nil -> seed_from_db(address, bytes) + counters -> counters + end + else + db_counters(address) + end + end + + @doc """ + Registers newly imported `transactions` and `token_transfers` in the cache: + marks the involved addresses dirty for the consolidator and, when `live?` is + `true` (realtime imports), increments the already-cached display values. + """ + @spec handle_new_data([map()], [map()], boolean()) :: :ok + def handle_new_data(transactions, token_transfers, live?) do + if cache_table_exists?() do + transactions + |> compute_deltas(token_transfers) + |> apply_new_deltas(live?) + end + + :ok + end + + defp apply_new_deltas(deltas, _live?) when map_size(deltas) == 0, do: :ok + + defp apply_new_deltas(deltas, live?) do + if live? do + Enum.each(deltas, fn {bytes, delta} -> bump_live(bytes, delta) end) + end + + deltas + |> Enum.map(fn {bytes, delta} -> {bytes, delta.max_block_number} end) + |> mark_dirty() + end + + @doc """ + Computes per-address deltas for the given transactions and token transfers. + + Rows without a block number (pending transactions) are skipped. A + transaction (or transfer) between an address and itself contributes once to + the address counts, matching the `from = address OR to = address` semantics + of the full aggregates. The optional `include?` predicate receives the + address hash bytes and the row block number and can exclude single + contributions (used to restrict deltas to blocks at or below an address + consolidation watermark). + """ + @spec compute_deltas([map()], [map()], (binary(), non_neg_integer() -> boolean())) :: %{binary() => delta()} + def compute_deltas(transactions, token_transfers, include? \\ fn _hash_bytes, _block_number -> true end) do + deltas = Enum.reduce(transactions, %{}, &add_transaction_delta(&2, &1, include?)) + + Enum.reduce(token_transfers, deltas, &add_token_transfer_delta(&2, &1, include?)) + end + + defp add_transaction_delta(acc, %{block_number: nil}, _include?), do: acc + + defp add_transaction_delta(acc, transaction, include?) do + block_number = transaction.block_number + from_bytes = hash_bytes(transaction.from_address_hash) + to_bytes = hash_bytes(transaction.to_address_hash) + gas_used = Helper.gas_to_integer(transaction.gas_used) + + [from_bytes, to_bytes] + |> participants(block_number, include?) + |> Enum.reduce(acc, fn bytes, inner_acc -> + add_delta(inner_acc, bytes, block_number, %{ + transactions_count: 1, + token_transfers_count: 0, + gas_in: if(bytes == to_bytes, do: gas_used, else: 0), + gas_out: if(bytes == from_bytes, do: gas_used, else: 0) + }) + end) + end + + defp add_token_transfer_delta(acc, %{block_number: nil}, _include?), do: acc + + defp add_token_transfer_delta(acc, token_transfer, include?) do + block_number = token_transfer.block_number + + [hash_bytes(token_transfer.from_address_hash), hash_bytes(token_transfer.to_address_hash)] + |> participants(block_number, include?) + |> Enum.reduce(acc, fn bytes, inner_acc -> + add_delta(inner_acc, bytes, block_number, %{ + transactions_count: 0, + token_transfers_count: 1, + gas_in: 0, + gas_out: 0 + }) + end) + end + + defp participants(hash_bytes_list, block_number, include?) do + hash_bytes_list + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> Enum.filter(&include?.(&1, block_number)) + end + + @doc """ + Increments the cached display values of the given address by the given + delta. Only already-cached addresses are updated — live bumps never create + entries, so a cached value always originates from a DB read. + """ + @spec bump_live(binary(), delta()) :: :ok + def bump_live(hash_bytes, delta) do + if :ets.member(@cache_name, {hash_bytes, :transactions_count}) do + gas_delta = + case Helper.fetch_from_ets_cache(@cache_name, {hash_bytes, :gas_direction}) do + :to_address_hash -> delta.gas_in + :from_address_hash -> delta.gas_out + _ -> 0 + end + + :ets.update_counter(@cache_name, {hash_bytes, :transactions_count}, delta.transactions_count) + :ets.update_counter(@cache_name, {hash_bytes, :token_transfers_count}, delta.token_transfers_count) + :ets.update_counter(@cache_name, {hash_bytes, :gas_used}, gas_delta) + end + + :ok + rescue + # the entry was evicted between the membership check and the update + ArgumentError -> :ok + end + + @doc """ + Returns the currently cached display values of the given address or `nil` + when the address is not cached. + """ + @spec snapshot(binary()) :: counters() | nil + def snapshot(hash_bytes) do + with true <- cache_table_exists?(), + [{_, transactions_count}] <- :ets.lookup(@cache_name, {hash_bytes, :transactions_count}), + [{_, token_transfers_count}] <- :ets.lookup(@cache_name, {hash_bytes, :token_transfers_count}), + [{_, gas_used}] <- :ets.lookup(@cache_name, {hash_bytes, :gas_used}) do + %{transactions_count: transactions_count, token_transfers_count: token_transfers_count, gas_used: gas_used} + else + _ -> nil + end + end + + @doc """ + Re-bases the cached display values of the given address on the values just + written to the DB by the consolidator. + + The write is an adjustment (`new value - snapshot`) rather than an + overwrite, so live bumps applied concurrently — always for blocks above the + consolidated range — survive on top of the new base. + """ + @spec refresh_from_consolidation(binary(), :to_address_hash | :from_address_hash, counters(), counters() | nil) :: + :ok + def refresh_from_consolidation(hash_bytes, direction_field, new_values, snapshot) do + if not is_nil(snapshot) and :ets.member(@cache_name, {hash_bytes, :transactions_count}) do + :ets.update_counter( + @cache_name, + {hash_bytes, :transactions_count}, + new_values.transactions_count - snapshot.transactions_count + ) + + :ets.update_counter( + @cache_name, + {hash_bytes, :token_transfers_count}, + new_values.token_transfers_count - snapshot.token_transfers_count + ) + + :ets.update_counter(@cache_name, {hash_bytes, :gas_used}, new_values.gas_used - snapshot.gas_used) + :ets.insert(@cache_name, {{hash_bytes, :gas_direction}, direction_field}) + :ets.insert(@cache_name, {{hash_bytes, :ts}, Helper.current_time()}) + end + + :ok + rescue + # the entry was evicted between the membership check and the update + ArgumentError -> :ok + end + + @doc """ + Drops the cached display values of the given addresses so that the next read + re-seeds them from the DB. Used after out-of-band DB counter corrections + (block re-fetch deltas, deep reorg resets). + """ + @spec invalidate([binary()]) :: :ok + def invalidate(hash_bytes_list) do + if cache_table_exists?() do + Enum.each(hash_bytes_list, &delete_display_entries/1) + end + + :ok + end + + @doc """ + Marks addresses dirty for the consolidator. Accepts a list of + `{hash_bytes, block_number}` pairs; for every address the maximum seen block + number is kept. The update is serialized through the cache process to avoid + losing the maximum on concurrent writes. + + A no-op on nodes without a local consolidator (pure API mode) — markers + would never be drained there. + """ + @spec mark_dirty([{binary(), non_neg_integer()}]) :: :ok + def mark_dirty(pairs) when is_list(pairs) do + if consolidator_local?() do + GenServer.cast(__MODULE__, {:mark_dirty, pairs}) + end + + :ok + end + + @doc """ + Marks the given address dirty at the current maximum block number, asking + the consolidator to (re)calculate its counters from scratch when + `counters_updated_at` is `NULL` or incrementally otherwise. + """ + @spec mark_dirty_at_max_block(Hash.Address.t()) :: :ok + def mark_dirty_at_max_block(address_hash) do + mark_dirty([{hash_bytes(address_hash), BlockNumber.get_max()}]) + end + + @doc """ + Returns `true` when there are no dirty markers (or the cache is not started). + """ + @spec dirty_empty?() :: boolean() + def dirty_empty? do + :ets.whereis(@dirty_cache_name) == :undefined or :ets.info(@dirty_cache_name, :size) == 0 + end + + @doc """ + Returns cache statistics for inspection (the process state itself is empty — + all data lives in the ETS tables): the number of addresses in the display + cache, the number of dirty markers awaiting consolidation, and the memory + held by both tables in bytes. + """ + @spec stats() :: %{ + cached_addresses: non_neg_integer(), + dirty_markers: non_neg_integer(), + memory_bytes: non_neg_integer() + } + def stats do + if cache_table_exists?() do + word_size = :erlang.system_info(:wordsize) + + %{ + cached_addresses: :ets.select_count(@cache_name, [{{{:_, :ts}, :_}, [], [true]}]), + dirty_markers: :ets.info(@dirty_cache_name, :size), + memory_bytes: (:ets.info(@cache_name, :memory) + :ets.info(@dirty_cache_name, :memory)) * word_size + } + else + %{cached_addresses: 0, dirty_markers: 0, memory_bytes: 0} + end + end + + @typedoc """ + Opaque `:ets.select/1` continuation of a dirty-markers traversal (the + `:ets.continuation/0` type is not exported by `:ets`). + """ + @type dirty_markers_continuation :: term() + + @doc """ + Starts (with a limit) or continues (with a previously returned continuation) + a traversal of the dirty markers. Returns `{[{hash_bytes, block_number}], continuation}` + or `:"$end_of_table"`. + """ + @spec select_dirty(pos_integer() | dirty_markers_continuation()) :: + {[{binary(), non_neg_integer()}], dirty_markers_continuation()} | :"$end_of_table" + def select_dirty(limit) when is_integer(limit) do + :ets.select(@dirty_cache_name, [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}], limit) + end + + def select_dirty(continuation) do + :ets.select(continuation) + end + + @doc """ + Deletes the dirty markers of the given addresses unless a concurrent import + bumped them above `safe_block` in the meantime (such markers stay for the + next consolidation cycle). + """ + @spec delete_dirty_markers([binary()], non_neg_integer()) :: :ok + def delete_dirty_markers(hash_bytes_list, safe_block) do + if :ets.whereis(@dirty_cache_name) != :undefined do + Enum.each(hash_bytes_list, fn bytes -> + :ets.select_delete(@dirty_cache_name, [{{bytes, :"$1"}, [{:"=<", :"$1", safe_block}], [true]}]) + end) + end + + :ok + end + + @doc """ + Converts an address hash (or already-extracted hash bytes) to the binary key + used in the cache tables. + """ + @spec hash_bytes(Hash.Address.t() | binary() | nil) :: binary() | nil + def hash_bytes(nil), do: nil + def hash_bytes(%Hash{bytes: bytes}), do: bytes + def hash_bytes(bytes) when is_binary(bytes), do: bytes + + def cache_name, do: @cache_name + def dirty_cache_name, do: @dirty_cache_name + + @impl true + def handle_cast({:mark_dirty, pairs}, state) do + Enum.each(pairs, fn {bytes, block_number} -> insert_dirty_marker(bytes, block_number) end) + + {:noreply, state} + end + + defp insert_dirty_marker(bytes, block_number) do + case :ets.lookup(@dirty_cache_name, bytes) do + [{_, existing}] when existing >= block_number -> + :ok + + [{_, _existing}] -> + :ets.insert(@dirty_cache_name, {bytes, block_number}) + + [] -> + # cap the work list: a dropped marker only delays consolidation until + # the address's next activity (during initial sync all addresses are + # covered by the backfill migration anyway) + if :ets.info(@dirty_cache_name, :size) < max_dirty_markers() do + :ets.insert(@dirty_cache_name, {bytes, block_number}) + end + end + end + + @impl true + def handle_info({:chain_event, :transactions, :realtime, transactions}, state) do + handle_new_data(transactions, [], true) + {:noreply, state} + end + + def handle_info({:chain_event, :token_transfers, :realtime, token_transfers}, state) do + handle_new_data([], token_transfers, true) + {:noreply, state} + end + + def handle_info(:evict, state) do + evict_stale_entries() + schedule_eviction() + {:noreply, state} + end + + def handle_info(_, state), do: {:noreply, state} + + defp seed_from_db(address, bytes) do + if is_nil(address.transactions_count) and is_nil(address.token_transfers_count) and is_nil(address.gas_used) do + mark_dirty([{bytes, BlockNumber.get_max()}]) + + %{transactions_count: 0, token_transfers_count: 0, gas_used: 0} + else + counters = db_counters(address) + + :ets.insert_new(@cache_name, [ + {{bytes, :transactions_count}, counters.transactions_count}, + {{bytes, :token_transfers_count}, counters.token_transfers_count}, + {{bytes, :gas_used}, counters.gas_used}, + {{bytes, :gas_direction}, Counters.gas_usage_direction_field(address)}, + {{bytes, :ts}, Helper.current_time()} + ]) + + counters + end + end + + defp db_counters(address) do + %{ + transactions_count: address.transactions_count || 0, + token_transfers_count: address.token_transfers_count || 0, + gas_used: address.gas_used || 0 + } + end + + defp add_delta(acc, bytes, block_number, contribution) do + acc + |> Map.put_new(bytes, @empty_delta) + |> Map.update!(bytes, fn delta -> + %{ + transactions_count: delta.transactions_count + contribution.transactions_count, + token_transfers_count: delta.token_transfers_count + contribution.token_transfers_count, + gas_in: delta.gas_in + contribution.gas_in, + gas_out: delta.gas_out + contribution.gas_out, + max_block_number: max(delta.max_block_number, block_number) + } + end) + end + + defp delete_display_entries(bytes) do + Enum.each([:transactions_count, :token_transfers_count, :gas_used, :gas_direction, :ts], fn key -> + :ets.delete(@cache_name, {bytes, key}) + end) + end + + defp evict_stale_entries do + threshold = Helper.current_time() - ttl() + + @cache_name + |> :ets.select([{{{:"$1", :ts}, :"$2"}, [{:<, :"$2", threshold}], [:"$1"]}]) + |> Enum.each(&delete_display_entries/1) + end + + defp cache_table_exists? do + :ets.whereis(@cache_name) != :undefined + end + + defp consolidator_local? do + Explorer.mode() in [:indexer, :all] + end + + defp max_dirty_markers do + Application.get_env(:explorer, __MODULE__)[:max_dirty_markers] || 1_000_000 + end + + defp schedule_eviction do + Process.send_after(self(), :evict, max(div(ttl(), 4), :timer.minutes(1))) + end + + defp ttl do + Application.get_env(:explorer, __MODULE__)[:ttl] || :timer.hours(2) + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex new file mode 100644 index 00000000000..e2f1b1d80b9 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex @@ -0,0 +1,721 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do + @moduledoc """ + The single writer of the per-address counters columns + (`addresses.transactions_count`, `addresses.token_transfers_count`, + `addresses.gas_used`). + + Periodically consolidates the counters of the addresses marked dirty in + `Explorer.Chain.Cache.Counters.AddressCounters` (a cycle is skipped entirely + when no address was marked since the last one): + + * an address with a `counters_updated_at` watermark gets three cheap + range-bounded aggregates over `(counters_updated_at, safe_block]` whose + results are added to the columns; + * an address without a watermark (`NULL` — not yet backfilled, or reset by + a deep reorg) gets full aggregates bounded by `safe_block` written as + absolute values. + + `safe_block` stays a configurable lag behind the chain head so that shallow + reorgs never invalidate consolidated ranges, and never advances past the + lowest missing block range or block pending re-fetch counter correction, so + that catchup and block re-fetch cannot make consolidated ranges lose or + double-count rows. + + Each cycle starts by settling pending block re-fetch corrections: for every + block whose old content was subtracted when it was queued for re-fetch (see + `Explorer.Utility.AddressCountersRefetchBlock`) and which has been fully + re-imported since, the new content is added back to the counters of the + addresses whose watermark already covers the block. + + Known accepted limitations: + + * pending transactions are not counted (all aggregates are bounded by block + number) — the legacy on-view counters included them; + * `transactions_count`/`token_transfers_count` are clamped at the int4 + maximum of their columns; + * a reorg deeper than the safety lag that lands while a consolidation cycle + is aggregating the very same address can leave the forked content counted + until the address is recalculated (watermark reset) for another reason. + """ + + use GenServer + + import Ecto.Query + + require Logger + + alias Explorer.Chain.{Address, Block, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.Address.Counters + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.{AddressCounters, Helper} + alias Explorer.Repo + alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} + + @int4_max 2_147_483_647 + @refetch_blocks_chunk_size 100 + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + schedule_next_consolidation(:timer.seconds(10)) + + {:ok, %{cycle_task: nil, cycle_started_at: nil, last_cycle_finished_at: nil}} + end + + # A cycle can run for a long time (draining a large dirty backlog, full + # recalculations of heavy addresses), so it runs in a separate supervised + # task: the process itself stays responsive to system messages + # (`:sys.get_state/1`, observer) and its state reports the running cycle. + # The next cycle is scheduled only once the previous one finished, so cycles + # never overlap. + @impl true + def handle_info(:consolidate, %{cycle_task: nil} = state) do + task = Task.Supervisor.async_nolink(Explorer.TaskSupervisor, &consolidate/0) + + {:noreply, %{state | cycle_task: task, cycle_started_at: DateTime.utc_now()}} + end + + # a stray tick while a cycle is still running — the next one is scheduled on + # its completion + def handle_info(:consolidate, state), do: {:noreply, state} + + def handle_info({ref, _result}, %{cycle_task: %Task{ref: ref}} = state) do + Process.demonitor(ref, [:flush]) + + {:noreply, finish_cycle(state)} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, %{cycle_task: %Task{ref: ref}} = state) do + Logger.error(fn -> ["Address counters consolidation cycle crashed: ", inspect(reason)] end) + + {:noreply, finish_cycle(state)} + end + + def handle_info(_, state), do: {:noreply, state} + + defp finish_cycle(state) do + schedule_next_consolidation(interval()) + + %{state | cycle_task: nil, cycle_started_at: nil, last_cycle_finished_at: DateTime.utc_now()} + end + + @doc """ + Runs one consolidation cycle synchronously. Exposed for tests and manual + runs; the supervised process runs it in a task on every `:consolidate` tick. + """ + @spec consolidate() :: :ok + def consolidate do + apply_pending_refetch_deltas() + + unless AddressCounters.dirty_empty?() do + case safe_block() do + nil -> :ok + safe_block -> consolidate_dirty(AddressCounters.select_dirty(batch_size() * concurrency()), safe_block) + end + end + + :ok + end + + @doc """ + Returns the highest block number consolidated ranges may currently cover or + `nil` when consolidation is not possible yet: the chain head minus the + configured safety lag, additionally capped below the lowest missing block + range and the lowest block pending a re-fetch counter correction. + + While the chain is not yet indexed down to the configured first block — + initial backward sync, before the missing ranges collector has recorded the + unindexed tail — no safe block exists at all: consolidating past blocks that + are absent but not yet registered in `missing_block_ranges` would lose their + contributions forever. + """ + @spec safe_block() :: non_neg_integer() | nil + def safe_block do + if lower_chain_indexed?() do + [lagged_head(), min_missing_block_cap(), min_pending_refetch_cap()] + |> Enum.reject(&is_nil/1) + |> Enum.min(fn -> nil end) + |> case do + safe_block when is_integer(safe_block) and safe_block > 0 -> safe_block + _ -> nil + end + end + end + + defp lower_chain_indexed? do + first_block = Application.get_env(:indexer, :first_block) + + case BlockNumber.get_min() do + min_block_number when is_integer(min_block_number) -> min_block_number <= first_block + _ -> false + end + end + + defp lagged_head do + case BlockNumber.get_max() do + max_block_number when is_integer(max_block_number) -> max_block_number - safe_block_lag() + _ -> nil + end + end + + defp min_missing_block_cap do + case MissingBlockRange.fetch_min_max() do + %{min: min} when is_integer(min) -> min - 1 + _ -> nil + end + end + + defp min_pending_refetch_cap do + case AddressCountersRefetchBlock.min_block_number() do + block_number when is_integer(block_number) -> block_number - 1 + _ -> nil + end + end + + @doc """ + Consolidates the given addresses up to `safe_block` synchronously: full + aggregates for addresses without a `counters_updated_at` watermark, + incremental range aggregates for the rest. Used by + `Explorer.Migrator.BackfillAddressCounters` and available for on-demand + recalculation. + """ + @spec consolidate_addresses([Hash.Address.t()], non_neg_integer()) :: :ok + def consolidate_addresses(address_hashes, safe_block) do + address_hashes + |> Enum.map(& &1.bytes) + |> load_addresses() + |> Enum.reject(&(&1.counters_updated_at && &1.counters_updated_at >= safe_block)) + |> case do + [] -> :ok + addresses -> consolidate_chunk(addresses, safe_block) + end + + :ok + end + + defp consolidate_dirty(:"$end_of_table", _safe_block), do: :ok + + defp consolidate_dirty({entries, continuation}, safe_block) do + process_entries(entries, safe_block) + consolidate_dirty(AddressCounters.select_dirty(continuation), safe_block) + end + + defp process_entries(entries, safe_block) do + entry_bytes = Enum.map(entries, fn {bytes, _marker_block} -> bytes end) + addresses_by_bytes = Map.new(load_addresses(entry_bytes), &{&1.hash.bytes, &1}) + + {found_bytes, missing_bytes} = Enum.split_with(entry_bytes, &Map.has_key?(addresses_by_bytes, &1)) + + # markers without a corresponding address row are garbage + AddressCounters.delete_dirty_markers(missing_bytes, safe_block) + + found_bytes + |> Enum.map(&Map.fetch!(addresses_by_bytes, &1)) + |> Enum.reject(&(&1.counters_updated_at && &1.counters_updated_at >= safe_block)) + |> Enum.chunk_every(batch_size()) + |> Task.async_stream(&consolidate_chunk(&1, safe_block), max_concurrency: concurrency(), timeout: :infinity) + |> Stream.run() + end + + defp consolidate_chunk(addresses, safe_block) do + results = + addresses + |> Enum.map(&compute_address_deltas(&1, safe_block)) + |> Enum.reject(&is_nil/1) + + {increments, inits} = Enum.split_with(results, & &1.address.counters_updated_at) + + updated_rows = apply_increments(increments, safe_block) ++ apply_inits(inits, safe_block) + updated_values_by_bytes = Map.new(updated_rows, &{&1.hash.bytes, &1}) + + updated_results = + Enum.filter(results, fn result -> Map.has_key?(updated_values_by_bytes, result.address.hash.bytes) end) + + Enum.each(updated_results, fn result -> + bytes = result.address.hash.bytes + new_values = Map.fetch!(updated_values_by_bytes, bytes) + + AddressCounters.refresh_from_consolidation( + bytes, + result.direction_field, + %{ + transactions_count: new_values.transactions_count || 0, + token_transfers_count: new_values.token_transfers_count || 0, + gas_used: new_values.gas_used || 0 + }, + result.snapshot + ) + end) + + updated_results + |> Enum.map(& &1.address.hash.bytes) + |> AddressCounters.delete_dirty_markers(safe_block) + end + + defp compute_address_deltas(address, safe_block) do + from_block_number = address.counters_updated_at + direction_field = Counters.gas_usage_direction_field(address) + snapshot = AddressCounters.snapshot(address.hash.bytes) + + transactions_count = + address.hash + |> Counters.address_hash_to_transaction_count_query(from_block_number, safe_block) + |> Repo.aggregate(:count, :hash, timeout: query_timeout()) + + token_transfers_count = + address.hash + |> Counters.address_to_token_transfer_count_query(from_block_number, safe_block) + |> Repo.aggregate(:count, timeout: query_timeout()) + + gas_used = + address.hash + |> Counters.address_to_gas_usage_sum_query(direction_field, from_block_number, safe_block) + |> Repo.aggregate(:sum, :gas_used, timeout: query_timeout()) + |> Helper.gas_to_integer() + + %{ + address: address, + direction_field: direction_field, + snapshot: snapshot, + transactions_count: transactions_count, + token_transfers_count: token_transfers_count, + gas_used: gas_used + } + rescue + error -> + Logger.warning(fn -> + [ + "Failed to consolidate counters for address #{to_string(address.hash)}: ", + Exception.format(:error, error, __STACKTRACE__) + ] + end) + + nil + end + + defp apply_increments([], _safe_block), do: [] + + defp apply_increments(results, safe_block) do + hashes = Enum.map(results, & &1.address.hash.bytes) + expected_froms = Enum.map(results, & &1.address.counters_updated_at) + transactions_deltas = Enum.map(results, & &1.transactions_count) + token_transfers_deltas = Enum.map(results, & &1.token_transfers_count) + gas_deltas = Enum.map(results, & &1.gas_used) + + query = + from( + address in Address, + join: + deltas in fragment( + """ + (SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS expected_from, + unnest(?::bigint[]) AS transactions_delta, unnest(?::bigint[]) AS token_transfers_delta, + unnest(?::bigint[]) AS gas_delta) + """, + ^hashes, + ^expected_froms, + ^transactions_deltas, + ^token_transfers_deltas, + ^gas_deltas + ), + on: address.hash == deltas.hash, + where: address.hash in subquery(addresses_lock_query(hashes)), + where: address.counters_updated_at == deltas.expected_from, + update: [ + set: [ + transactions_count: + fragment("LEAST(COALESCE(?, 0) + ?, ?)", address.transactions_count, deltas.transactions_delta, @int4_max), + token_transfers_count: + fragment( + "LEAST(COALESCE(?, 0) + ?, ?)", + address.token_transfers_count, + deltas.token_transfers_delta, + @int4_max + ), + gas_used: fragment("COALESCE(?, 0) + ?", address.gas_used, deltas.gas_delta), + counters_updated_at: ^safe_block, + updated_at: ^DateTime.utc_now() + ] + ], + select: %{ + hash: address.hash, + transactions_count: address.transactions_count, + token_transfers_count: address.token_transfers_count, + gas_used: address.gas_used + } + ) + + {_count, rows} = Repo.update_all(query, [], timeout: query_timeout()) + + rows + end + + defp apply_inits([], _safe_block), do: [] + + defp apply_inits(results, safe_block) do + hashes = Enum.map(results, & &1.address.hash.bytes) + transactions_counts = Enum.map(results, & &1.transactions_count) + token_transfers_counts = Enum.map(results, & &1.token_transfers_count) + gas_sums = Enum.map(results, & &1.gas_used) + + query = + from( + address in Address, + join: + values in fragment( + """ + (SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS transactions_count, + unnest(?::bigint[]) AS token_transfers_count, unnest(?::bigint[]) AS gas_used) + """, + ^hashes, + ^transactions_counts, + ^token_transfers_counts, + ^gas_sums + ), + on: address.hash == values.hash, + where: address.hash in subquery(addresses_lock_query(hashes)), + where: is_nil(address.counters_updated_at), + update: [ + set: [ + transactions_count: fragment("LEAST(?, ?)", values.transactions_count, @int4_max), + token_transfers_count: fragment("LEAST(?, ?)", values.token_transfers_count, @int4_max), + gas_used: values.gas_used, + counters_updated_at: ^safe_block, + updated_at: ^DateTime.utc_now() + ] + ], + select: %{ + hash: address.hash, + transactions_count: address.transactions_count, + token_transfers_count: address.token_transfers_count, + gas_used: address.gas_used + } + ) + + {_count, rows} = Repo.update_all(query, [], timeout: query_timeout()) + + rows + end + + defp load_addresses(hash_bytes_list) do + hashes = Enum.map(hash_bytes_list, &%Hash{byte_count: 20, bytes: &1}) + + query = + from(address in Address, + where: address.hash in ^hashes, + select: + struct(address, [ + :hash, + :contract_code, + :transactions_count, + :token_transfers_count, + :gas_used, + :counters_updated_at + ]) + ) + + Repo.all(query, timeout: query_timeout()) + end + + ## Block re-fetch corrections (positive side) + + defp apply_pending_refetch_deltas do + # a block is ready to settle once its re-fetch completed: the consensus + # block no longer requires a re-fetch AND the block is no longer covered + # by a missing range (the range row survives until the whole re-import — + # all its stages — succeeded, so old content can no longer be observed) + ready_query = + from(refetch_block in AddressCountersRefetchBlock, + as: :refetch_block, + where: + not exists( + from(block in Block, + where: + block.number == parent_as(:refetch_block).block_number and block.consensus == true and + block.refetch_needed == true, + select: 1 + ) + ), + where: + not exists( + from(range in MissingBlockRange, + where: + range.from_number >= parent_as(:refetch_block).block_number and + range.to_number <= parent_as(:refetch_block).block_number, + select: 1 + ) + ), + select: refetch_block.block_number, + limit: @refetch_blocks_chunk_size + ) + + case Repo.all(ready_query, timeout: query_timeout()) do + [] -> + :ok + + block_numbers -> + settle_refetched_blocks(block_numbers) + # more rows may be ready + apply_pending_refetch_deltas() + end + rescue + error -> + Logger.error(fn -> + ["Failed to settle re-fetched blocks counters: ", Exception.format(:error, error, __STACKTRACE__)] + end) + + :ok + end + + defp settle_refetched_blocks(block_numbers) do + {:ok, updated_bytes} = + Repo.transaction( + fn -> + transactions = + Repo.all( + from(transaction in Transaction, + where: transaction.block_number in ^block_numbers, + select: struct(transaction, [:block_number, :from_address_hash, :to_address_hash, :gas_used]) + ), + timeout: query_timeout() + ) + + token_transfers = + Repo.all( + from(token_transfer in TokenTransfer, + where: token_transfer.block_number in ^block_numbers, + select: struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash]) + ), + timeout: query_timeout() + ) + + updated_bytes = apply_covered_deltas(transactions, token_transfers, :positive) + + AddressCountersRefetchBlock.delete_by_block_numbers(Enum.sort(block_numbers)) + + updated_bytes + end, + timeout: :infinity + ) + + AddressCounters.invalidate(updated_bytes) + end + + @doc """ + Applies the per-address counter deltas of the given transactions and token + transfers directly to the `addresses` columns, restricted per address to the + blocks already covered by its `counters_updated_at` watermark (contributions + above the watermark are left to regular range consolidation). `sign` selects + whether the contributions are added (`:positive` — re-imported content of + re-fetched blocks) or subtracted (`:negative` — old content of blocks queued + for re-fetch, see `Explorer.Chain.Import.Runner.Blocks`). + + Runs on `Explorer.Repo` unless another repo is given. Returns the hash bytes + of the updated addresses. + """ + @spec apply_covered_deltas([map()], [map()], :positive | :negative, Ecto.Repo.t()) :: [binary()] + def apply_covered_deltas(transactions, token_transfers, sign, repo \\ Repo) do + watermarks = + transactions + |> participant_hashes(token_transfers) + |> load_watermarks(repo) + + deltas = + AddressCounters.compute_deltas(transactions, token_transfers, fn bytes, block_number -> + case watermarks[bytes] do + nil -> false + watermark -> block_number <= watermark + end + end) + + if map_size(deltas) == 0 do + [] + else + apply_signed_deltas(deltas, watermarks, sign, repo) + end + end + + @doc """ + Resets the `counters_updated_at` watermark of the given addresses — forcing + a full counters recalculation on their next consolidation — when their + watermark already covers the given block number (a covered range changed + under it: deep reorg, or rows inserted below the watermark by an + out-of-band importer). Returns the hash bytes of the reset addresses; the + caller is responsible for invalidating the display cache and marking them + dirty. + """ + @spec reset_covered_watermarks(%{binary() => non_neg_integer()}, Ecto.Repo.t()) :: [binary()] + def reset_covered_watermarks(min_block_by_hash_bytes, repo \\ Repo) + + def reset_covered_watermarks(min_block_by_hash_bytes, _repo) when map_size(min_block_by_hash_bytes) == 0, do: [] + + def reset_covered_watermarks(min_block_by_hash_bytes, repo) do + {hashes, min_blocks} = min_block_by_hash_bytes |> Enum.sort() |> Enum.unzip() + + query = + from( + address in Address, + join: + affected in fragment( + "(SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS block_number)", + ^hashes, + ^min_blocks + ), + on: address.hash == affected.hash, + where: address.hash in subquery(addresses_lock_query(hashes)), + where: not is_nil(address.counters_updated_at) and address.counters_updated_at >= affected.block_number, + update: [set: [counters_updated_at: nil, updated_at: ^DateTime.utc_now()]], + select: address.hash + ) + + {_count, reset_hashes} = repo.update_all(query, [], timeout: query_timeout()) + + Enum.map(reset_hashes, & &1.bytes) + end + + defp apply_signed_deltas(deltas, watermarks, sign, repo) do + multiplier = if sign == :negative, do: -1, else: 1 + + entries = Enum.to_list(deltas) + hashes = Enum.map(entries, fn {bytes, _delta} -> bytes end) + expected_watermarks = Enum.map(entries, fn {bytes, _delta} -> watermarks[bytes] end) + transactions_deltas = Enum.map(entries, fn {_bytes, delta} -> multiplier * delta.transactions_count end) + token_transfers_deltas = Enum.map(entries, fn {_bytes, delta} -> multiplier * delta.token_transfers_count end) + gas_in_deltas = Enum.map(entries, fn {_bytes, delta} -> multiplier * delta.gas_in end) + gas_out_deltas = Enum.map(entries, fn {_bytes, delta} -> multiplier * delta.gas_out end) + + query = + from( + address in Address, + join: + deltas in fragment( + """ + (SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS expected_watermark, + unnest(?::bigint[]) AS transactions_delta, unnest(?::bigint[]) AS token_transfers_delta, + unnest(?::bigint[]) AS gas_in_delta, unnest(?::bigint[]) AS gas_out_delta) + """, + ^hashes, + ^expected_watermarks, + ^transactions_deltas, + ^token_transfers_deltas, + ^gas_in_deltas, + ^gas_out_deltas + ), + on: address.hash == deltas.hash, + where: address.hash in subquery(addresses_lock_query(hashes)), + # only apply when the watermark is unchanged since the deltas were + # computed; changed addresses are reset below for a full recalculation + where: address.counters_updated_at == deltas.expected_watermark, + update: [ + set: [ + transactions_count: + fragment( + "LEAST(GREATEST(COALESCE(?, 0) + ?, 0), ?)", + address.transactions_count, + deltas.transactions_delta, + @int4_max + ), + token_transfers_count: + fragment( + "LEAST(GREATEST(COALESCE(?, 0) + ?, 0), ?)", + address.token_transfers_count, + deltas.token_transfers_delta, + @int4_max + ), + # incoming gas for smart contracts, outgoing for EOAs — including + # EIP-7702 EOAs with delegated code (code starts with 0xef0100) + gas_used: + fragment( + """ + GREATEST(COALESCE(?, 0) + CASE + WHEN ? IS NULL OR substring(? from 1 for 3) = decode('ef0100', 'hex') THEN ? + ELSE ? + END, 0) + """, + address.gas_used, + address.contract_code, + address.contract_code, + deltas.gas_out_delta, + deltas.gas_in_delta + ), + updated_at: ^DateTime.utc_now() + ] + ], + select: address.hash + ) + + {_count, updated_hashes} = repo.update_all(query, [], timeout: query_timeout()) + + updated_bytes = Enum.map(updated_hashes, & &1.bytes) + + # addresses whose watermark moved between the read and the guarded update + # cannot be corrected by a delta anymore — force a full recalculation + missed_reset_bytes = + (hashes -- updated_bytes) + |> Map.new(&{&1, 0}) + |> reset_covered_watermarks(repo) + + updated_bytes ++ missed_reset_bytes + end + + defp participant_hashes(transactions, token_transfers) do + (transactions ++ token_transfers) + |> Enum.flat_map(fn row -> [row.from_address_hash, row.to_address_hash] end) + |> Enum.reject(&is_nil/1) + |> Enum.map(&AddressCounters.hash_bytes/1) + |> Enum.uniq() + end + + defp load_watermarks(participant_bytes, repo) do + hashes = Enum.map(participant_bytes, &%Hash{byte_count: 20, bytes: &1}) + + from(address in Address, + where: address.hash in ^hashes and not is_nil(address.counters_updated_at), + select: {address.hash, address.counters_updated_at} + ) + |> repo.all(timeout: query_timeout()) + |> Map.new(fn {hash, watermark} -> {hash.bytes, watermark} end) + end + + # Enforce Address ShareLocks order (see docs: sharelocks.md) + defp addresses_lock_query(hashes_bytes) do + hashes = Enum.map(hashes_bytes, &%Hash{byte_count: 20, bytes: &1}) + + from( + address in Address, + where: address.hash in ^hashes, + select: address.hash, + order_by: address.hash, + lock: "FOR NO KEY UPDATE" + ) + end + + defp schedule_next_consolidation(timeout) do + Process.send_after(self(), :consolidate, timeout) + end + + defp interval do + Application.get_env(:explorer, __MODULE__)[:interval] || :timer.minutes(10) + end + + defp batch_size do + Application.get_env(:explorer, __MODULE__)[:batch_size] || 100 + end + + defp concurrency do + Application.get_env(:explorer, __MODULE__)[:concurrency] || 4 + end + + defp safe_block_lag do + Application.get_env(:explorer, __MODULE__)[:safe_block_lag] || 12 + end + + defp query_timeout do + Application.get_env(:explorer, __MODULE__)[:query_timeout] || :timer.minutes(5) + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_token_transfers_count.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_token_transfers_count.ex deleted file mode 100644 index 225b4922886..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/address_token_transfers_count.ex +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.AddressTokenTransfersCount do - @moduledoc """ - Caches Address token transfers count. - """ - use GenServer - use Utils.CompileTimeEnvHelper, enable_consolidation: [:explorer, [__MODULE__, :enable_consolidation]] - - alias Ecto.Changeset - alias Explorer.Chain.Address.Counters - alias Explorer.Chain.Cache.Counters.Helper - alias Explorer.Repo - - @cache_name :address_token_transfers_counter - @last_update_key "last_update" - - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_args) do - Helper.create_cache_table(@cache_name) - - {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} - end - - @impl true - def handle_continue(:ok, %{consolidate?: true} = state) do - {:noreply, state} - end - - @impl true - def handle_continue(:ok, state) do - {:noreply, state} - end - - @impl true - def handle_info(:consolidate, state) do - {:noreply, state} - end - - def fetch(address) do - if cache_expired?(address) do - update_cache(address) - end - - address_hash_string = to_string(address.hash) - fetch_from_cache("hash_#{address_hash_string}") - end - - def cache_name, do: @cache_name - - defp cache_expired?(address) do - cache_period = Application.get_env(:explorer, __MODULE__)[:cache_period] - address_hash_string = to_string(address.hash) - updated_at = fetch_from_cache("hash_#{address_hash_string}_#{@last_update_key}") - - cond do - is_nil(updated_at) -> true - Helper.current_time() - updated_at > cache_period -> true - true -> false - end - end - - defp update_cache(address) do - address_hash_string = to_string(address.hash) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}_#{@last_update_key}", Helper.current_time()) - new_data = Counters.address_to_token_transfer_count(address) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}", new_data) - put_into_db(address, new_data) - end - - defp fetch_from_cache(key) do - Helper.fetch_from_ets_cache(@cache_name, key) - end - - defp put_into_db(address, value) do - address - |> Changeset.change(%{token_transfers_count: value}) - |> Repo.update() - end - - defp enable_consolidation?, do: @enable_consolidation -end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_count.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_count.ex deleted file mode 100644 index 7ff8aece4e1..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_count.ex +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.AddressTransactionsCount do - @moduledoc """ - Caches Address transactions count. - """ - use GenServer - use Utils.CompileTimeEnvHelper, enable_consolidation: [:explorer, [__MODULE__, :enable_consolidation]] - - alias Ecto.Changeset - alias Explorer.Chain.Address.Counters - alias Explorer.Chain.Cache.Counters.Helper - alias Explorer.Repo - - @cache_name :address_transactions_counter - @last_update_key "last_update" - - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_args) do - Helper.create_cache_table(@cache_name) - - {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} - end - - @impl true - def handle_continue(:ok, %{consolidate?: true} = state) do - {:noreply, state} - end - - @impl true - def handle_continue(:ok, state) do - {:noreply, state} - end - - @impl true - def handle_info(:consolidate, state) do - {:noreply, state} - end - - def fetch(address) do - if cache_expired?(address) do - update_cache(address) - end - - address_hash_string = to_string(address.hash) - fetch_from_cache("hash_#{address_hash_string}") - end - - def cache_name, do: @cache_name - - defp cache_expired?(address) do - cache_period = Application.get_env(:explorer, __MODULE__)[:cache_period] - address_hash_string = to_string(address.hash) - updated_at = fetch_from_cache("hash_#{address_hash_string}_#{@last_update_key}") - - cond do - is_nil(updated_at) -> true - Helper.current_time() - updated_at > cache_period -> true - true -> false - end - end - - defp update_cache(address) do - address_hash_string = to_string(address.hash) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}_#{@last_update_key}", Helper.current_time()) - new_data = Counters.address_to_transaction_count(address) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}", new_data) - put_into_db(address, new_data) - end - - defp fetch_from_cache(key) do - Helper.fetch_from_ets_cache(@cache_name, key) - end - - defp put_into_db(address, value) do - address - |> Changeset.change(%{transactions_count: value}) - |> Repo.update() - end - - defp enable_consolidation?, do: @enable_consolidation -end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_gas_usage_sum.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_gas_usage_sum.ex deleted file mode 100644 index fa9bda0ee3e..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/address_transactions_gas_usage_sum.ex +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.AddressTransactionsGasUsageSum do - @moduledoc """ - Caches Address transactions gas usage count. - """ - use GenServer - use Utils.CompileTimeEnvHelper, enable_consolidation: [:explorer, [__MODULE__, :enable_consolidation]] - - alias Ecto.Changeset - alias Explorer.Chain.Address.Counters - alias Explorer.Chain.Cache.Counters.Helper - alias Explorer.Repo - - @cache_name :address_transactions_gas_usage_counter - @last_update_key "last_update" - - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_args) do - Helper.create_cache_table(@cache_name) - - {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} - end - - @impl true - def handle_continue(:ok, %{consolidate?: true} = state) do - {:noreply, state} - end - - @impl true - def handle_continue(:ok, state) do - {:noreply, state} - end - - @impl true - def handle_info(:consolidate, state) do - {:noreply, state} - end - - def fetch(address) do - if cache_expired?(address) do - update_cache(address) - end - - address_hash_string = to_string(address.hash) - fetch_from_cache("hash_#{address_hash_string}") - end - - def cache_name, do: @cache_name - - defp cache_expired?(address) do - cache_period = Application.get_env(:explorer, __MODULE__)[:cache_period] - address_hash_string = to_string(address.hash) - updated_at = fetch_from_cache("hash_#{address_hash_string}_#{@last_update_key}") - - cond do - is_nil(updated_at) -> true - Helper.current_time() - updated_at > cache_period -> true - true -> false - end - end - - defp update_cache(address) do - address_hash_string = to_string(address.hash) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}_#{@last_update_key}", Helper.current_time()) - new_data = Counters.address_to_gas_usage_count(address) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}", new_data) - put_into_db(address, new_data) - end - - defp fetch_from_cache(key) do - Helper.fetch_from_ets_cache(@cache_name, key) - end - - defp put_into_db(_address, value) when is_nil(value), do: :ignore - - defp put_into_db(address, value) do - address - |> Changeset.change(%{gas_used: Decimal.to_integer(value)}) - |> Repo.update() - end - - defp enable_consolidation?, do: @enable_consolidation -end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/helper.ex b/apps/explorer/lib/explorer/chain/cache/counters/helper.ex index 9a74b74b78d..7fee87a4960 100644 --- a/apps/explorer/lib/explorer/chain/cache/counters/helper.ex +++ b/apps/explorer/lib/explorer/chain/cache/counters/helper.ex @@ -35,6 +35,14 @@ defmodule Explorer.Chain.Cache.Counters.Helper do DateTime.to_unix(utc_now, :millisecond) end + @doc """ + Normalizes a gas amount (`nil`, `Decimal` or integer) to an integer. + """ + @spec gas_to_integer(Decimal.t() | non_neg_integer() | nil) :: non_neg_integer() + def gas_to_integer(nil), do: 0 + def gas_to_integer(%Decimal{} = gas), do: Decimal.to_integer(gas) + def gas_to_integer(gas) when is_integer(gas), do: gas + @doc """ Fetches a value from the ETS cache. @@ -86,15 +94,17 @@ defmodule Explorer.Chain.Cache.Counters.Helper do ## Parameters - cache_name: The name of the cache table to be created. + - extra_opts: Additional `:ets.new/2` options appended to the default ones + (e.g. `[write_concurrency: true]`). ## Returns - The table identifier if the table is created. - `nil` if the table already exists. """ - @spec create_cache_table(atom()) :: any() - def create_cache_table(cache_name) do + @spec create_cache_table(atom(), list()) :: any() + def create_cache_table(cache_name, extra_opts \\ []) do if :ets.whereis(cache_name) == :undefined do - :ets.new(cache_name, @ets_opts) + :ets.new(cache_name, @ets_opts ++ extra_opts) end end diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex index 4fbd82ce206..00658705d91 100644 --- a/apps/explorer/lib/explorer/chain/import.ex +++ b/apps/explorer/lib/explorer/chain/import.ex @@ -7,6 +7,8 @@ defmodule Explorer.Chain.Import do alias Ecto.{Changeset, Multi} alias Explorer.Account.Notify alias Explorer.Chain.{Block, Import} + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} alias Explorer.Chain.Events.Publisher alias Explorer.Chain.Import.Stage alias Explorer.Repo @@ -155,11 +157,64 @@ defmodule Explorer.Chain.Import do {:ok, runner_to_changes_list} <- runner_to_changes_list(valid_runner_option_pairs), {:ok, data} <- insert_runner_to_changes_list(runner_to_changes_list, options) do Notify.async(data[:transactions]) + update_address_counters(data, Map.get(options, :broadcast, false)) Publisher.broadcast(Map.drop(data, @not_broadcasted_runners), Map.get(options, :broadcast, false)) {:ok, data} end end + # Registers imported transactions and token transfers in the incremental + # address counters, whatever the import source (block fetcher, on-demand and + # async fetchers, migrators): marks the involved addresses dirty for + # `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` and live-bumps + # the display cache for realtime imports. Token transfers inserted below an + # address's consolidation watermark (derived asynchronously — e.g. from + # internal transactions — or backfilled by migrators) cannot be covered by + # incremental range aggregates anymore, so such addresses get their + # watermark reset for a full recalculation. + # + # Runs after the import transaction committed, so failures here (e.g. a + # transient DB error in the watermark reset) must not fail the import or + # prevent the subsequent event broadcast — they only cost counter freshness. + defp update_address_counters(data, broadcast_type) do + transactions = Map.get(data, :transactions, []) + token_transfers = Map.get(data, :token_transfers, []) + + AddressCounters.handle_new_data(transactions, token_transfers, broadcast_type == :realtime) + + if token_transfers != [] and Explorer.mode() in [:indexer, :all] do + reset_watermarks_covering_token_transfers(token_transfers) + end + + :ok + rescue + error -> + Logger.error(fn -> + ["Could not update address counters for imported data: ", Exception.format(:error, error, __STACKTRACE__)] + end) + + :ok + end + + defp reset_watermarks_covering_token_transfers(token_transfers) do + reset_bytes = + token_transfers + |> Enum.flat_map(&token_transfer_participants/1) + |> Enum.reduce(%{}, fn {bytes, block_number}, acc -> + Map.update(acc, bytes, block_number, &min(&1, block_number)) + end) + |> AddressCountersConsolidator.reset_covered_watermarks() + + AddressCounters.invalidate(reset_bytes) + AddressCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + end + + defp token_transfer_participants(token_transfer) do + for address_hash <- [token_transfer.from_address_hash, token_transfer.to_address_hash], + not is_nil(address_hash) and not is_nil(token_transfer.block_number), + do: {address_hash.bytes, token_transfer.block_number} + end + @doc """ Prepares a bulk import transaction without executing it. diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index e05ab9dcff2..375f88feef3 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -30,6 +30,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Explorer.Chain.Block.Reward alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances alias Explorer.Chain.Import.Runner.{Addresses, TokenInstances, Tokens} @@ -37,7 +38,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Explorer.Chain.Zilliqa.Zrc2.TokenTransfer, as: Zrc2TokenTransfer alias Explorer.Prometheus.Instrumenter alias Explorer.Repo, as: ExplorerRepo - alias Explorer.Utility.MissingBlockRange + alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} alias Explorer.Chain.Celo.AggregatedElectionReward, as: CeloAggregatedElectionReward alias Explorer.Chain.Celo.ElectionReward, as: CeloElectionReward @@ -95,6 +96,17 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :lose_consensus ) end) + |> Multi.run(:address_counters_refetched_block_numbers, fn repo, _ -> + Instrumenter.block_import_stage_runner( + fn -> + # Note, needs to be executed before `blocks` which resets `refetch_needed` + address_counters_refetched_block_numbers(repo, changes_list, insert_options) + end, + :address_referencing, + :blocks, + :address_counters_refetched_block_numbers + ) + end) |> Multi.run(:blocks, fn repo, _ -> Instrumenter.block_import_stage_runner( fn -> @@ -159,6 +171,18 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :derive_transaction_forks ) end) + |> Multi.run(:address_counters_corrections, fn repo, + %{ + address_counters_refetched_block_numbers: refetched_block_numbers, + fork_transactions: forked_transactions + } -> + Instrumenter.block_import_stage_runner( + fn -> address_counters_corrections(repo, refetched_block_numbers, forked_transactions, insert_options) end, + :address_referencing, + :blocks, + :address_counters_corrections + ) + end) |> Multi.run(:delete_address_token_balances, fn repo, %{lose_consensus: non_consensus_blocks} -> Instrumenter.block_import_stage_runner( fn -> delete_address_token_balances(repo, non_consensus_blocks, insert_options) end, @@ -279,6 +303,137 @@ defmodule Explorer.Chain.Import.Runner.Blocks do @impl Runner def timeout, do: @timeout + # Detects which of the incoming blocks are re-imports of blocks queued for a + # re-fetch: their stored row still has `refetch_needed` set (the upcoming + # `blocks` upsert resets it). Blocks already present in the + # `address_counters_refetch_blocks` queue are skipped — their current content + # was never added to the counters, so there is nothing to subtract. + defp address_counters_refetched_block_numbers(repo, blocks_changes, %{timeout: timeout}) do + hashes = blocks_changes |> Enum.map(& &1.hash) |> Enum.uniq() + + refetched_numbers = + repo.all( + from(block in Block, + where: block.hash in ^hashes and block.refetch_needed == true, + select: block.number + ), + timeout: timeout + ) + + case refetched_numbers do + [] -> + {:ok, []} + + _ -> + pending_numbers = + repo.all( + from(refetch_block in AddressCountersRefetchBlock, + where: refetch_block.block_number in ^refetched_numbers, + select: refetch_block.block_number + ), + timeout: timeout + ) + + {:ok, refetched_numbers -- pending_numbers} + end + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error}} + end + + # Corrects the incremental address counters for content changing under + # already-consolidated ranges: + # + # * deep reorgs — participants of forked transactions and of their token + # transfers whose `counters_updated_at` watermark covers the forked block + # get their watermark reset for a full recalculation (reorgs within the + # consolidation safety lag never satisfy the condition); + # * block re-fetches — the old content of the re-imported blocks is + # subtracted from the already-covered counters while it is still in the + # DB, and the block numbers are queued in `address_counters_refetch_blocks` + # so that `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` adds + # the re-imported content back once the re-fetch completes. Content of + # forked transactions is excluded (`block_number` already nulled) — those + # addresses are covered by the watermark reset. + defp address_counters_corrections(repo, refetched_block_numbers, forked_transactions, %{ + timeout: timeout, + timestamps: timestamps + }) do + reset_bytes = reset_watermarks_for_forked(repo, forked_transactions, timeout) + + subtracted_bytes = subtract_refetched_blocks_content(repo, refetched_block_numbers, timeout, timestamps) + + AddressCounters.invalidate(reset_bytes ++ subtracted_bytes) + AddressCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + + {:ok, %{reset: reset_bytes, refetched_block_numbers: refetched_block_numbers}} + rescue + postgrex_error in Postgrex.Error -> + {:error, %{exception: postgrex_error}} + end + + defp reset_watermarks_for_forked(_repo, [], _timeout), do: [] + + defp reset_watermarks_for_forked(repo, forked_transactions, timeout) do + forked_hashes = Enum.map(forked_transactions, & &1.hash) + + forked_token_transfers = + repo.all( + from(token_transfer in TokenTransfer, + where: token_transfer.transaction_hash in ^forked_hashes, + select: struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash]) + ), + timeout: timeout + ) + + (forked_transactions ++ forked_token_transfers) + |> Enum.flat_map(fn row -> + for address_hash <- [row.from_address_hash, row.to_address_hash], + not is_nil(address_hash) and not is_nil(row.block_number), + do: {address_hash.bytes, row.block_number} + end) + |> Enum.reduce(%{}, fn {bytes, block_number}, acc -> + Map.update(acc, bytes, block_number, &min(&1, block_number)) + end) + |> AddressCountersConsolidator.reset_covered_watermarks(repo) + end + + defp subtract_refetched_blocks_content(_repo, [], _timeout, _timestamps), do: [] + + defp subtract_refetched_blocks_content(repo, block_numbers, timeout, timestamps) do + transactions = + repo.all( + from(transaction in Transaction, + where: transaction.block_number in ^block_numbers, + select: struct(transaction, [:block_number, :from_address_hash, :to_address_hash, :gas_used]) + ), + timeout: timeout + ) + + token_transfers = + repo.all( + from(token_transfer in TokenTransfer, + where: token_transfer.block_number in ^block_numbers, + select: struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash]) + ), + timeout: timeout + ) + + updated_bytes = AddressCountersConsolidator.apply_covered_deltas(transactions, token_transfers, :negative, repo) + + # Enforce AddressCountersRefetchBlock ShareLocks order (see docs: sharelocks.md) + repo.insert_all( + AddressCountersRefetchBlock, + block_numbers + |> Enum.sort() + |> Enum.map(&%{block_number: &1, inserted_at: timestamps.inserted_at, updated_at: timestamps.updated_at}), + on_conflict: :nothing, + timeout: timeout + ) + + updated_bytes + end + defp fork_transactions(%{ repo: repo, timeout: timeout, diff --git a/apps/explorer/lib/explorer/migrator/backfill_address_counters.ex b/apps/explorer/lib/explorer/migrator/backfill_address_counters.ex new file mode 100644 index 00000000000..f1ff5a7b586 --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/backfill_address_counters.ex @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.BackfillAddressCounters do + @moduledoc """ + Backfills the incremental address counters: computes + `addresses.transactions_count`, `addresses.token_transfers_count` and + `addresses.gas_used` from scratch for every address that was never + consolidated (`counters_updated_at IS NULL`) and stamps its consolidation + watermark. + + Pre-existing column values were written by the legacy on-view counters at + unknown block heights, so they cannot seed incremental updates and are + recalculated. The migration walks the `addresses` primary key with a keyset + cursor stored in the migration `meta`, so it resumes where it left off after + a restart. Addresses consolidated concurrently (via the dirty-marker path of + `Explorer.Chain.Cache.Counters.AddressCountersConsolidator`) are skipped by + the `counters_updated_at IS NULL` guard. + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + require Logger + + alias Explorer.Chain.Address + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Chain.Cache.Counters.AddressCountersConsolidator + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + @migration_name "backfill_address_counters" + + # extra cursor sweeps picking up addresses whose aggregates failed on a + # previous pass (they keep a NULL watermark but the cursor already passed + # them) + @max_retry_passes 3 + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers(state) do + limit = batch_size() * concurrency() + + query = + from(address in Address, + where: is_nil(address.counters_updated_at), + order_by: [asc: address.hash], + limit: ^limit, + select: address.hash + ) + + hashes = + state + |> Map.get("max_processed_hash") + |> case do + nil -> query + cursor -> where(query, [address], address.hash > ^cursor) + end + |> Repo.all(timeout: :infinity) + + case List.last(hashes) do + nil -> maybe_retry_pass(state) + last_hash -> {hashes, Map.put(state, "max_processed_hash", to_string(last_hash))} + end + end + + defp maybe_retry_pass(state) do + retry_passes = Map.get(state, "retry_passes", 0) + + if retry_passes < @max_retry_passes and Map.has_key?(state, "max_processed_hash") and + Repo.exists?(from(address in Address, where: is_nil(address.counters_updated_at))) do + last_unprocessed_identifiers(%{"retry_passes" => retry_passes + 1}) + else + {[], state} + end + end + + @impl FillingMigration + def unprocessed_data_query, do: nil + + @impl FillingMigration + def update_batch(address_hashes) do + AddressCountersConsolidator.consolidate_addresses(address_hashes, await_safe_block()) + end + + @impl FillingMigration + def update_cache do + BackgroundMigrations.set_backfill_address_counters_finished(true) + end + + # Consolidation is not possible while the chain head is unknown or blocks + # below it are still missing or pending a re-fetch counter correction. Wait + # instead of skipping: the cursor must not advance past unprocessed + # addresses. + defp await_safe_block do + case AddressCountersConsolidator.safe_block() do + nil -> + Logger.info("#{@migration_name} migration is waiting for a safe block to consolidate up to") + Process.sleep(:timer.seconds(30)) + await_safe_block() + + safe_block -> + safe_block + end + end +end diff --git a/apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex b/apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex new file mode 100644 index 00000000000..893cc10cbdc --- /dev/null +++ b/apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Utility.AddressCountersRefetchBlock do + @moduledoc """ + Keeps block numbers whose old content was already subtracted from the + incremental address counters (`addresses.transactions_count`, + `addresses.token_transfers_count`, `addresses.gas_used`) because the blocks + were queued for re-fetch (see `Explorer.Chain.Block.full_refetch/1`). + + A row means "the negative counters delta for this block was applied, the + positive delta from its re-imported content is still pending". Rows are + consumed by `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` once + the corresponding block no longer requires a re-fetch. While a row exists, + the consolidator does not advance any address consolidation watermark past + this block number. + """ + + use Explorer.Schema + + alias Explorer.Repo + + @primary_key false + typed_schema "address_counters_refetch_blocks" do + field(:block_number, :integer, primary_key: true) + + timestamps() + end + + @doc false + def changeset(refetch_block \\ %__MODULE__{}, params) do + cast(refetch_block, params, [:block_number]) + end + + @doc """ + Returns the minimum pending block number or `nil` when the table is empty. + """ + @spec min_block_number() :: non_neg_integer() | nil + def min_block_number do + __MODULE__ + |> select([r], min(r.block_number)) + |> Repo.one() + end + + @doc """ + Deletes the rows for the given block numbers. + """ + @spec delete_by_block_numbers([non_neg_integer()]) :: {non_neg_integer(), nil} + def delete_by_block_numbers(block_numbers) do + __MODULE__ + |> where([r], r.block_number in ^block_numbers) + |> Repo.delete_all() + end +end diff --git a/apps/explorer/priv/repo/migrations/20260831000000_add_counters_updated_at_to_addresses.exs b/apps/explorer/priv/repo/migrations/20260831000000_add_counters_updated_at_to_addresses.exs new file mode 100644 index 00000000000..6c1f72c7040 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20260831000000_add_counters_updated_at_to_addresses.exs @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Repo.Migrations.AddCountersUpdatedAtToAddresses do + use Ecto.Migration + + def change do + alter table(:addresses) do + add(:counters_updated_at, :bigint, null: true) + end + end +end diff --git a/apps/explorer/priv/repo/migrations/20260831000001_create_address_counters_refetch_blocks.exs b/apps/explorer/priv/repo/migrations/20260831000001_create_address_counters_refetch_blocks.exs new file mode 100644 index 00000000000..794293ef3ee --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20260831000001_create_address_counters_refetch_blocks.exs @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Repo.Migrations.CreateAddressCountersRefetchBlocks do + use Ecto.Migration + + def change do + create table(:address_counters_refetch_blocks, primary_key: false) do + add(:block_number, :bigint, primary_key: true) + + timestamps(null: false, type: :utc_datetime_usec) + end + end +end diff --git a/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs b/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs new file mode 100644 index 00000000000..389e8c35549 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Address + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} + alias Explorer.Repo + alias Explorer.Utility.AddressCountersRefetchBlock + + describe "consolidate_addresses/2" do + test "incrementally consolidates an address with a watermark" do + address = + insert(:address, transactions_count: 7, token_transfers_count: 2, gas_used: 10, counters_updated_at: 100) + + in_range_block = insert(:block, number: 150) + above_range_block = insert(:block, number: 300) + + in_range_transaction = :transaction |> insert(from_address: address) |> with_block(in_range_block) + :transaction |> insert(to_address: address) |> with_block(above_range_block) + + another_transaction = :transaction |> insert() |> with_block(in_range_block) + + insert(:token_transfer, + from_address: address, + transaction: another_transaction, + block: in_range_block, + block_number: in_range_block.number + ) + + AddressCountersConsolidator.consolidate_addresses([address.hash], 200) + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 8 + assert reloaded.token_transfers_count == 3 + assert reloaded.gas_used == 10 + Decimal.to_integer(in_range_transaction.gas_used) + assert reloaded.counters_updated_at == 200 + end + + test "recalculates an address without a watermark from scratch, replacing stale values" do + address = insert(:address, transactions_count: 1000, token_transfers_count: 1000, gas_used: 1000) + + block = insert(:block, number: 50) + transaction = :transaction |> insert(from_address: address) |> with_block(block) + + AddressCountersConsolidator.consolidate_addresses([address.hash], 200) + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 1 + assert reloaded.token_transfers_count == 0 + assert reloaded.gas_used == Decimal.to_integer(transaction.gas_used) + assert reloaded.counters_updated_at == 200 + end + + test "skips addresses already consolidated at or above the safe block" do + address = insert(:address, transactions_count: 7, counters_updated_at: 300) + + block = insert(:block, number: 150) + :transaction |> insert(from_address: address) |> with_block(block) + + AddressCountersConsolidator.consolidate_addresses([address.hash], 200) + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 7 + assert reloaded.counters_updated_at == 300 + end + + test "accumulates incoming gas for contracts and outgoing gas for EOAs with delegated code" do + contract = insert(:contract_address, counters_updated_at: 100) + + eoa_with_code = + insert(:address, + contract_code: "0xef01001111111111111111111111111111111111111111", + counters_updated_at: 100 + ) + + block = insert(:block, number: 150) + + incoming_to_contract = :transaction |> insert(to_address: contract) |> with_block(block) + :transaction |> insert(from_address: contract) |> with_block(insert(:block, number: 151)) + + incoming_to_eoa = :transaction |> insert(to_address: eoa_with_code) |> with_block(insert(:block, number: 152)) + outgoing_from_eoa = :transaction |> insert(from_address: eoa_with_code) |> with_block(insert(:block, number: 153)) + + _ = incoming_to_eoa + + AddressCountersConsolidator.consolidate_addresses([contract.hash, eoa_with_code.hash], 200) + + assert Repo.get(Address, contract.hash).gas_used == Decimal.to_integer(incoming_to_contract.gas_used) + assert Repo.get(Address, eoa_with_code.hash).gas_used == Decimal.to_integer(outgoing_from_eoa.gas_used) + end + end + + describe "safe_block/0" do + setup do + initial_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + + Application.put_env( + :explorer, + AddressCountersConsolidator, + Keyword.merge(initial_env, safe_block_lag: 0) + ) + + on_exit(fn -> Application.put_env(:explorer, AddressCountersConsolidator, initial_env) end) + + :ok + end + + test "returns the lagged chain head" do + insert(:block, number: 0) + insert(:block, number: 100) + + assert AddressCountersConsolidator.safe_block() == 100 + end + + test "is nil while the chain is not indexed down to the first block" do + insert(:block, number: 100) + + assert AddressCountersConsolidator.safe_block() == nil + end + + test "is capped below the lowest missing block range" do + insert(:block, number: 0) + insert(:block, number: 100) + insert(:missing_block_range, from_number: 60, to_number: 50) + + assert AddressCountersConsolidator.safe_block() == 49 + end + + test "is capped below the lowest block pending a re-fetch counter correction" do + insert(:block, number: 0) + insert(:block, number: 100) + Repo.insert!(%AddressCountersRefetchBlock{block_number: 30}) + + assert AddressCountersConsolidator.safe_block() == 29 + end + end + + describe "consolidation cycle process" do + test "the process stays responsive while a cycle runs and reports it in its state" do + pid = start_supervised!(AddressCountersConsolidator) + + send(pid, :consolidate) + + # the cycle runs in a task, so the process answers immediately + assert %{cycle_task: _, cycle_started_at: _, last_cycle_finished_at: _} = :sys.get_state(pid, 1_000) + + wait_for_results(fn -> + case :sys.get_state(pid, 1_000) do + %{cycle_task: nil, last_cycle_finished_at: %DateTime{}} -> :ok + _ -> raise Ecto.NoResultsError, queryable: "cycle completion" + end + end) + end + end + + describe "consolidate/0" do + setup do + initial_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + + Application.put_env( + :explorer, + AddressCountersConsolidator, + Keyword.merge(initial_env, safe_block_lag: 0) + ) + + on_exit(fn -> Application.put_env(:explorer, AddressCountersConsolidator, initial_env) end) + + start_supervised!(AddressCounters) + + :ok + end + + test "consolidates dirty addresses and clears their markers" do + address = insert(:address, transactions_count: 1, token_transfers_count: 0, gas_used: 0, counters_updated_at: 10) + + insert(:block, number: 0) + block = insert(:block, number: 100) + :transaction |> insert(from_address: address) |> with_block(block) + + AddressCounters.mark_dirty([{address.hash.bytes, 100}]) + :sys.get_state(AddressCounters) + + AddressCountersConsolidator.consolidate() + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 2 + assert reloaded.counters_updated_at == 100 + assert AddressCounters.dirty_empty?() + end + + test "keeps markers above the safe block" do + address = insert(:address, transactions_count: 1, token_transfers_count: 0, gas_used: 0, counters_updated_at: 10) + + insert(:block, number: 0) + insert(:block, number: 100) + + AddressCounters.mark_dirty([{address.hash.bytes, 500}]) + :sys.get_state(AddressCounters) + + AddressCountersConsolidator.consolidate() + + assert Repo.get(Address, address.hash).counters_updated_at == 100 + refute AddressCounters.dirty_empty?() + end + + test "settles re-fetched blocks: adds the re-imported content below the watermark and drops the queue rows" do + address = insert(:address, transactions_count: 5, token_transfers_count: 0, gas_used: 0, counters_updated_at: 100) + + reimported_block = insert(:block, number: 50, refetch_needed: false) + transaction = :transaction |> insert(from_address: address) |> with_block(reimported_block) + + insert(:block, number: 100) + Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + + AddressCountersConsolidator.consolidate() + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 6 + assert reloaded.gas_used == Decimal.to_integer(transaction.gas_used) + # settling a covered block does not move the watermark + assert reloaded.counters_updated_at == 100 + assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 0 + end + + test "does not settle blocks still awaiting their re-fetch" do + insert(:block, number: 50, refetch_needed: true) + insert(:block, number: 100) + Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + + AddressCountersConsolidator.consolidate() + + assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + end + + test "does not settle blocks still covered by a missing range" do + # refetch_needed already flipped, but the re-import has not fully + # completed yet (the missing range row is only removed afterwards) + insert(:block, number: 50, refetch_needed: false) + insert(:block, number: 100) + insert(:missing_block_range, from_number: 50, to_number: 50) + Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + + AddressCountersConsolidator.consolidate() + + assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + end + end + + describe "apply_covered_deltas/4" do + test "applies negative deltas only for blocks covered by the address watermark" do + address = + insert(:address, transactions_count: 5, token_transfers_count: 3, gas_used: 100, counters_updated_at: 100) + + transactions = [ + %{block_number: 50, from_address_hash: address.hash, to_address_hash: nil, gas_used: 30}, + # above the watermark: never counted, must not be subtracted + %{block_number: 150, from_address_hash: address.hash, to_address_hash: nil, gas_used: 30} + ] + + token_transfers = [ + %{block_number: 50, from_address_hash: address.hash, to_address_hash: nil} + ] + + [updated_bytes] = AddressCountersConsolidator.apply_covered_deltas(transactions, token_transfers, :negative) + + assert updated_bytes == address.hash.bytes + + reloaded = Repo.get(Address, address.hash) + + assert reloaded.transactions_count == 4 + assert reloaded.token_transfers_count == 2 + assert reloaded.gas_used == 70 + assert reloaded.counters_updated_at == 100 + end + + test "skips addresses without a watermark" do + address = insert(:address, transactions_count: 5) + + transactions = [%{block_number: 50, from_address_hash: address.hash, to_address_hash: nil, gas_used: 30}] + + assert AddressCountersConsolidator.apply_covered_deltas(transactions, [], :positive) == [] + assert Repo.get(Address, address.hash).transactions_count == 5 + end + end +end diff --git a/apps/explorer/test/explorer/chain/cache/counters/address_counters_test.exs b/apps/explorer/test/explorer/chain/cache/counters/address_counters_test.exs new file mode 100644 index 00000000000..259d3b6c2ef --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/counters/address_counters_test.exs @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.AddressCountersTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.Counters.AddressCounters + + setup do + start_supervised!(AddressCounters) + + :ok + end + + describe "fetch/1" do + test "returns DB column values and seeds the cache" do + address = insert(:address, transactions_count: 5, token_transfers_count: 3, gas_used: 100) + + assert %{transactions_count: 5, token_transfers_count: 3, gas_used: 100} = AddressCounters.fetch(address) + + # the second read is served from ETS: a live bump is visible on top of the seed + AddressCounters.bump_live(address.hash.bytes, delta(transactions_count: 2)) + + assert %{transactions_count: 7, token_transfers_count: 3, gas_used: 100} = AddressCounters.fetch(address) + end + + test "returns zeros and marks the address dirty when the counters were never calculated" do + address = insert(:address) + + assert %{transactions_count: 0, token_transfers_count: 0, gas_used: 0} = AddressCounters.fetch(address) + + :sys.get_state(AddressCounters) + + assert {[{bytes, _block_number}], _continuation} = AddressCounters.select_dirty(10) + assert bytes == address.hash.bytes + end + end + + describe "compute_deltas/3" do + test "computes per-address contributions of transactions and token transfers" do + address_a = build(:address) + address_b = build(:address) + + transactions = [ + %{ + block_number: 100, + from_address_hash: address_a.hash, + to_address_hash: address_b.hash, + gas_used: Decimal.new(21_000) + }, + # pending transactions are skipped + %{block_number: nil, from_address_hash: address_a.hash, to_address_hash: address_b.hash, gas_used: nil} + ] + + token_transfers = [ + %{block_number: 101, from_address_hash: address_b.hash, to_address_hash: address_a.hash} + ] + + deltas = AddressCounters.compute_deltas(transactions, token_transfers) + + assert %{ + transactions_count: 1, + token_transfers_count: 1, + gas_in: 0, + gas_out: 21_000, + max_block_number: 101 + } = deltas[address_a.hash.bytes] + + assert %{ + transactions_count: 1, + token_transfers_count: 1, + gas_in: 21_000, + gas_out: 0, + max_block_number: 101 + } = deltas[address_b.hash.bytes] + end + + test "counts a self-send once and fills both gas buckets" do + address = build(:address) + + deltas = + AddressCounters.compute_deltas( + [%{block_number: 5, from_address_hash: address.hash, to_address_hash: address.hash, gas_used: 100}], + [] + ) + + assert %{transactions_count: 1, gas_in: 100, gas_out: 100} = deltas[address.hash.bytes] + end + + test "the include? predicate excludes single contributions" do + address_a = build(:address) + address_b = build(:address) + + transactions = [ + %{block_number: 10, from_address_hash: address_a.hash, to_address_hash: address_b.hash, gas_used: 1}, + %{block_number: 20, from_address_hash: address_a.hash, to_address_hash: address_b.hash, gas_used: 1} + ] + + watermarks = %{address_a.hash.bytes => 15} + + deltas = + AddressCounters.compute_deltas(transactions, [], fn bytes, block_number -> + case watermarks[bytes] do + nil -> false + watermark -> block_number <= watermark + end + end) + + assert %{transactions_count: 1, max_block_number: 10} = deltas[address_a.hash.bytes] + refute Map.has_key?(deltas, address_b.hash.bytes) + end + end + + describe "bump_live/2" do + test "never creates entries for addresses that are not cached" do + address = insert(:address, transactions_count: 1, token_transfers_count: 1, gas_used: 1) + + AddressCounters.bump_live(address.hash.bytes, delta(transactions_count: 5)) + + # the first read seeds from the DB columns: the bump was not applied anywhere + assert %{transactions_count: 1} = AddressCounters.fetch(address) + end + + test "applies the gas bucket matching the cached gas direction" do + contract = insert(:contract_address, transactions_count: 0, token_transfers_count: 0, gas_used: 0) + AddressCounters.fetch(contract) + + AddressCounters.bump_live(contract.hash.bytes, delta(gas_in: 30, gas_out: 7)) + + assert %{gas_used: 30} = AddressCounters.fetch(contract) + end + end + + describe "handle_new_data/3" do + test "bumps cached entries for live data and marks addresses dirty" do + address = insert(:address, transactions_count: 10, token_transfers_count: 0, gas_used: 0) + AddressCounters.fetch(address) + + transactions = [ + %{block_number: 300, from_address_hash: address.hash, to_address_hash: nil, gas_used: 55} + ] + + AddressCounters.handle_new_data(transactions, [], true) + :sys.get_state(AddressCounters) + + assert %{transactions_count: 11, gas_used: 55} = AddressCounters.fetch(address) + assert {[{bytes, 300}], _continuation} = AddressCounters.select_dirty(10) + assert bytes == address.hash.bytes + end + + test "only marks addresses dirty for catchup data" do + address = insert(:address, transactions_count: 10, token_transfers_count: 0, gas_used: 0) + AddressCounters.fetch(address) + + transactions = [ + %{block_number: 300, from_address_hash: address.hash, to_address_hash: nil, gas_used: 55} + ] + + AddressCounters.handle_new_data(transactions, [], false) + :sys.get_state(AddressCounters) + + assert %{transactions_count: 10, gas_used: 0} = AddressCounters.fetch(address) + assert {[{_bytes, 300}], _continuation} = AddressCounters.select_dirty(10) + end + end + + describe "mark_dirty/1" do + test "is a no-op on api-only nodes" do + initial_mode = Application.get_env(:explorer, :mode) + Application.put_env(:explorer, :mode, :api) + on_exit(fn -> Application.put_env(:explorer, :mode, initial_mode) end) + + AddressCounters.mark_dirty([{build(:address).hash.bytes, 10}]) + :sys.get_state(AddressCounters) + + assert AddressCounters.dirty_empty?() + end + + test "does not create new markers above the configured cap" do + initial_env = Application.get_env(:explorer, AddressCounters) || [] + Application.put_env(:explorer, AddressCounters, Keyword.merge(initial_env, max_dirty_markers: 1)) + on_exit(fn -> Application.put_env(:explorer, AddressCounters, initial_env) end) + + address_a = build(:address) + address_b = build(:address) + + AddressCounters.mark_dirty([{address_a.hash.bytes, 10}]) + AddressCounters.mark_dirty([{address_b.hash.bytes, 20}]) + # already-marked addresses can still be bumped + AddressCounters.mark_dirty([{address_a.hash.bytes, 30}]) + :sys.get_state(AddressCounters) + + assert {[{bytes, 30}], _continuation} = AddressCounters.select_dirty(10) + assert bytes == address_a.hash.bytes + end + end + + describe "stats/0" do + test "reports cached addresses and dirty markers" do + address = insert(:address, transactions_count: 5, token_transfers_count: 0, gas_used: 0) + AddressCounters.fetch(address) + + AddressCounters.mark_dirty([{build(:address).hash.bytes, 10}]) + :sys.get_state(AddressCounters) + + assert %{cached_addresses: 1, dirty_markers: 1, memory_bytes: memory_bytes} = AddressCounters.stats() + assert memory_bytes > 0 + end + end + + describe "delete_dirty_markers/2" do + test "keeps markers above the safe block" do + address_a = build(:address) + address_b = build(:address) + + AddressCounters.mark_dirty([{address_a.hash.bytes, 100}, {address_b.hash.bytes, 200}]) + :sys.get_state(AddressCounters) + + AddressCounters.delete_dirty_markers([address_a.hash.bytes, address_b.hash.bytes], 150) + + assert {[{bytes, 200}], _continuation} = AddressCounters.select_dirty(10) + assert bytes == address_b.hash.bytes + end + end + + describe "eviction" do + test "drops entries not touched within the TTL" do + initial_env = Application.get_env(:explorer, AddressCounters) || [] + Application.put_env(:explorer, AddressCounters, Keyword.merge(initial_env, ttl: 0)) + on_exit(fn -> Application.put_env(:explorer, AddressCounters, initial_env) end) + + address = insert(:address, transactions_count: 5, token_transfers_count: 0, gas_used: 0) + AddressCounters.fetch(address) + + Process.sleep(1) + send(Process.whereis(AddressCounters), :evict) + :sys.get_state(AddressCounters) + + assert is_nil(AddressCounters.snapshot(address.hash.bytes)) + end + end + + defp delta(overrides) do + Enum.into(overrides, %{transactions_count: 0, token_transfers_count: 0, gas_in: 0, gas_out: 0, max_block_number: 0}) + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index f316123d7c7..dd03ca2ce8e 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -15,7 +15,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Explorer.Chain.{Address, Block, Transaction, PendingBlockOperation} alias Explorer.Chain.Cache.BlockNumber alias Explorer.{Chain, Repo} - alias Explorer.Utility.MissingBlockRange + alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} describe "run/1" do setup do @@ -89,6 +89,98 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do "Tuple was written even though it is not distinct" end + test "re-import of a block queued for re-fetch subtracts its old content from the address counters", %{ + consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, + options: options + } do + consensus_block = insert(:block, %{hash: block_hash, number: block_number, refetch_needed: true}) + + transaction = + :transaction + |> insert() + |> with_block(consensus_block) + + gas_used = Decimal.to_integer(transaction.gas_used) + + from_address = Repo.get(Address, transaction.from_address_hash) + + from_address + |> Ecto.Changeset.change(%{ + transactions_count: 5, + token_transfers_count: 2, + gas_used: gas_used + 100, + counters_updated_at: block_number + 10 + }) + |> Repo.update!() + + block_params = + params_for(:block, hash: block_hash, miner_hash: miner_hash, number: block_number, consensus: true) + + %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, block_params) + + assert {:ok, %{address_counters_refetched_block_numbers: [^block_number]}} = + Multi.new() + |> Blocks.run([block_changes], options) + |> Repo.transaction() + + reloaded = Repo.get(Address, transaction.from_address_hash) + + assert reloaded.transactions_count == 4 + assert reloaded.token_transfers_count == 2 + assert reloaded.gas_used == 100 + assert reloaded.counters_updated_at == block_number + 10 + + assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + + # a second import of the same block applies no further deltas: the + # re-fetch was already settled into the queue + assert {:ok, %{address_counters_refetched_block_numbers: []}} = + Multi.new() + |> Blocks.run([block_changes], options) + |> Repo.transaction() + + assert Repo.get(Address, transaction.from_address_hash).transactions_count == 4 + end + + test "a reorg below the consolidation watermark resets it to force a full recalculation", %{ + consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, + options: options + } do + consensus_block = insert(:block, %{hash: block_hash, number: block_number}) + + transaction = + :transaction + |> insert() + |> with_block(consensus_block) + + consolidated_address = + Address + |> Repo.get(transaction.from_address_hash) + |> Ecto.Changeset.change(%{counters_updated_at: block_number + 10}) + |> Repo.update!() + + fresh_address = + Address + |> Repo.get(transaction.to_address_hash) + |> Ecto.Changeset.change(%{counters_updated_at: block_number - 1}) + |> Repo.update!() + + block_params = + params_for(:block, hash: block_hash, miner_hash: miner_hash, number: block_number, consensus: false) + + %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, block_params) + + assert {:ok, %{address_counters_corrections: %{reset: reset_bytes}}} = + Multi.new() + |> Blocks.run([block_changes], options) + |> Repo.transaction() + + assert reset_bytes == [consolidated_address.hash.bytes] + assert is_nil(Repo.get(Address, consolidated_address.hash).counters_updated_at) + # the forked transaction was above this watermark, so it stays intact + assert Repo.get(Address, fresh_address.hash).counters_updated_at == block_number - 1 + end + test "coin balances are deleted and new balances are derived if some blocks lost consensus", %{consensus_block: %{number: block_number} = block, options: options} do %{hash: address_hash} = address = insert(:address, fetched_coin_balance_block_number: block_number) diff --git a/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs b/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs new file mode 100644 index 00000000000..5f0a3e7f8f5 --- /dev/null +++ b/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.BackfillAddressCountersTest do + use Explorer.DataCase, async: false + + import Ecto.Query + + alias Explorer.Chain.Address + alias Explorer.Chain.Cache.Counters.AddressCountersConsolidator + alias Explorer.Migrator.{BackfillAddressCounters, MigrationStatus} + alias Explorer.Repo + + setup do + initial_consolidator_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + + Application.put_env( + :explorer, + AddressCountersConsolidator, + Keyword.merge(initial_consolidator_env, safe_block_lag: 0) + ) + + initial_migrator_env = Application.get_env(:explorer, BackfillAddressCounters) || [] + + Application.put_env( + :explorer, + BackfillAddressCounters, + Keyword.merge(initial_migrator_env, batch_size: 2, concurrency: 2, timeout: 0) + ) + + on_exit(fn -> + Application.put_env(:explorer, AddressCountersConsolidator, initial_consolidator_env) + Application.put_env(:explorer, BackfillAddressCounters, initial_migrator_env) + end) + + :ok + end + + test "backfills counters for addresses that were never consolidated" do + insert(:block, number: 0) + block = insert(:block, number: 100) + + active_address = insert(:address, transactions_count: 999, token_transfers_count: 999, gas_used: 999) + transaction = :transaction |> insert(from_address: active_address) |> with_block(block) + + dormant_address = insert(:address) + + already_consolidated_address = + insert(:address, transactions_count: 42, token_transfers_count: 42, gas_used: 42, counters_updated_at: 77) + + assert MigrationStatus.get_status("backfill_address_counters") == nil + + BackfillAddressCounters.start_link([]) + + wait_for_results(fn -> + Repo.one!( + from(ms in MigrationStatus, + where: ms.migration_name == ^"backfill_address_counters" and ms.status == "completed" + ) + ) + end) + + safe_block = AddressCountersConsolidator.safe_block() + + active_reloaded = Repo.get(Address, active_address.hash) + assert active_reloaded.transactions_count == 1 + assert active_reloaded.token_transfers_count == 0 + assert active_reloaded.gas_used == Decimal.to_integer(transaction.gas_used) + assert active_reloaded.counters_updated_at == safe_block + + dormant_reloaded = Repo.get(Address, dormant_address.hash) + assert dormant_reloaded.transactions_count == 0 + assert dormant_reloaded.token_transfers_count == 0 + assert dormant_reloaded.gas_used == 0 + assert dormant_reloaded.counters_updated_at == safe_block + + # addresses already consolidated are left untouched + consolidated_reloaded = Repo.get(Address, already_consolidated_address.hash) + assert consolidated_reloaded.transactions_count == 42 + assert consolidated_reloaded.counters_updated_at == 77 + + # nothing is left to process + assert Repo.one(from(a in Address, where: is_nil(a.counters_updated_at), select: count())) == 0 + end + + test "resumes from the cursor stored in the migration meta" do + insert(:block, number: 0) + insert(:block, number: 100) + + addresses = Enum.map(1..5, fn _ -> insert(:address) end) + max_hash = addresses |> Enum.map(&to_string(&1.hash)) |> Enum.max() + + BackfillAddressCounters.start_link([]) + + wait_for_results(fn -> + Repo.one!( + from(ms in MigrationStatus, + where: ms.migration_name == ^"backfill_address_counters" and ms.status == "completed" + ) + ) + end) + + %{meta: meta} = MigrationStatus.fetch("backfill_address_counters") + + assert meta["max_processed_hash"] >= max_hash + end +end diff --git a/config/runtime.exs b/config/runtime.exs index 2181b9bc333..31c37f6b987 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -430,8 +430,17 @@ config :explorer, Explorer.Chain.Cache.Counters.Rootstock.LockedBTCCount, config :explorer, Explorer.Chain.Cache.OptimismFinalizationPeriod, enabled: ConfigHelper.chain_type() == :optimism -config :explorer, Explorer.Chain.Cache.Counters.AddressTransactionsGasUsageSum, - cache_period: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TRANSACTIONS_GAS_USAGE_COUNTER_PERIOD", "30m") +config :explorer, Explorer.Chain.Cache.Counters.AddressCounters, + ttl: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_COUNTERS_TTL", "2h"), + max_dirty_markers: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_MAX_DIRTY_MARKERS", 1_000_000, min: 1) + +config :explorer, Explorer.Chain.Cache.Counters.AddressCountersConsolidator, + enabled: !ConfigHelper.parse_bool_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_DISABLED"), + interval: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_INTERVAL", "10m"), + batch_size: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_BATCH_SIZE", 100, min: 1), + concurrency: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_CONCURRENCY", 4, min: 1), + safe_block_lag: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG", 12, min: 0), + query_timeout: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT", "5m") config :explorer, Explorer.Chain.Cache.Counters.TokenHoldersCount, cache_period: ConfigHelper.parse_time_env_var("CACHE_TOKEN_HOLDERS_COUNTER_PERIOD", "1h") @@ -454,15 +463,9 @@ config :explorer, Explorer.Stats.HotSmartContractsCache, %{ "3h" => ConfigHelper.parse_time_env_var("CACHE_HOT_SMART_CONTRACTS_3H_PERIOD", "18m") } -config :explorer, Explorer.Chain.Cache.Counters.AddressTransactionsCount, - cache_period: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TRANSACTIONS_COUNTER_PERIOD", "1h") - config :explorer, Explorer.Chain.Cache.Counters.AddressTokensUsdSum, cache_period: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TOKENS_USD_SUM_PERIOD", "1h") -config :explorer, Explorer.Chain.Cache.Counters.AddressTokenTransfersCount, - cache_period: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TOKEN_TRANSFERS_COUNTER_PERIOD", "1h") - config :explorer, Explorer.Chain.Cache.Counters.Optimism.LastOutputRootSizeCount, enabled: ConfigHelper.chain_type() == :optimism, enable_consolidation: ConfigHelper.chain_type() == :optimism, @@ -948,6 +951,12 @@ config :explorer, Explorer.Migrator.FillInternalTransactionsAddressIds, concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_FILL_INTERNAL_TRANSACTIONS_ADDRESS_IDS_CONCURRENCY", 10), timeout: ConfigHelper.parse_time_env_var("MIGRATION_FILL_INTERNAL_TRANSACTIONS_ADDRESS_IDS_TIMEOUT", "5s") +config :explorer, Explorer.Migrator.BackfillAddressCounters, + enabled: !ConfigHelper.parse_bool_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED"), + batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE", 10), + concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY", 2), + timeout: ConfigHelper.parse_time_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT", "500ms") + config :explorer, Explorer.Chain.BridgedToken, eth_omni_bridge_mediator: System.get_env("BRIDGED_TOKENS_ETH_OMNI_BRIDGE_MEDIATOR"), bsc_omni_bridge_mediator: System.get_env("BRIDGED_TOKENS_BSC_OMNI_BRIDGE_MEDIATOR"), diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 3284431f5d9..9608cc3522b 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -121,15 +121,24 @@ RELEASE_LINK= # CACHE_BLOCK_COUNT_PERIOD=7200 # CACHE_TXS_COUNT_PERIOD=7200 # CACHE_ADDRESS_SUM_PERIOD=3600 -# CACHE_ADDRESS_TRANSACTIONS_GAS_USAGE_COUNTER_PERIOD=1800 # CACHE_TOKEN_HOLDERS_COUNTER_PERIOD=3600 # CACHE_TOKEN_TRANSFERS_COUNTER_PERIOD=3600 # CACHE_ADDRESS_COUNT_PERIOD=1800 # CACHE_AVERAGE_BLOCK_PERIOD=1800 # CACHE_MARKET_HISTORY_PERIOD=21600 -# CACHE_ADDRESS_TRANSACTIONS_COUNTER_PERIOD=3600 # CACHE_ADDRESS_TOKENS_USD_SUM_PERIOD=3600 -# CACHE_ADDRESS_TOKEN_TRANSFERS_COUNTER_PERIOD=3600 +# CACHE_ADDRESS_COUNTERS_TTL=2h +# CACHE_ADDRESS_COUNTERS_MAX_DIRTY_MARKERS=1000000 +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_DISABLED=false +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_INTERVAL=10m +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_BATCH_SIZE=100 +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_CONCURRENCY=4 +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG=12 +# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT=5m +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED=false +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE=10 +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY=2 +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT=500ms # CACHE_HOT_SMART_CONTRACTS_5M_PERIOD=30s # CACHE_HOT_SMART_CONTRACTS_1H_PERIOD=6m # CACHE_HOT_SMART_CONTRACTS_3H_PERIOD=18m From 42b94e6e244b7ee1d0a1c4a62b59bd7a26783e41 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Fri, 4 Sep 2026 14:52:02 +0300 Subject: [PATCH 15/52] perf: Switch token counters to incremental consolidation (#14773) --- .../controllers/api/v2/token_controller.ex | 4 +- .../controllers/tokens/token_controller.ex | 10 +- .../api/v2/token_controller_test.exs | 10 +- apps/explorer/config/config.exs | 11 +- apps/explorer/config/runtime/test.exs | 3 + apps/explorer/lib/explorer/application.ex | 5 +- .../chain/cache/background_migrations.ex | 9 + .../counters/address_counters_consolidator.ex | 175 +---- .../chain/cache/counters/consolidation.ex | 224 +++++++ .../chain/cache/counters/token_counters.ex | 411 ++++++++++++ .../counters/token_counters_consolidator.ex | 602 ++++++++++++++++++ .../cache/counters/token_holders_count.ex | 101 --- .../cache/counters/token_transfers_count.ex | 101 --- apps/explorer/lib/explorer/chain/import.ex | 68 +- .../runner/address/current_token_balances.ex | 267 ++++---- .../explorer/chain/import/runner/blocks.ex | 87 +-- .../explorer/chain/import/runner/tokens.ex | 10 +- apps/explorer/lib/explorer/chain/token.ex | 149 ++--- .../lib/explorer/chain/token_transfer.ex | 22 + .../migrator/backfill_token_counters.ex | 105 +++ ...tch_block.ex => counters_refetch_block.ex} | 25 +- ...0002_add_counters_updated_at_to_tokens.exs | 10 + ...rename_address_counters_refetch_blocks.exs | 15 + .../address_counters_consolidator_test.exs | 59 +- .../token_counters_consolidator_test.exs | 272 ++++++++ .../cache/counters/token_counters_test.exs | 187 ++++++ .../address/current_token_balances_test.exs | 92 ++- .../chain/import/runner/blocks_test.exs | 10 +- .../backfill_address_counters_test.exs | 10 +- .../migrator/backfill_token_counters_test.exs | 114 ++++ .../indexer/fetcher/token_counters_updater.ex | 38 +- config/runtime.exs | 23 +- cspell.json | 1 + docker-compose/envs/common-blockscout.env | 15 +- 34 files changed, 2532 insertions(+), 713 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/counters/consolidation.ex create mode 100644 apps/explorer/lib/explorer/chain/cache/counters/token_counters.ex create mode 100644 apps/explorer/lib/explorer/chain/cache/counters/token_counters_consolidator.ex delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/token_holders_count.ex delete mode 100644 apps/explorer/lib/explorer/chain/cache/counters/token_transfers_count.ex create mode 100644 apps/explorer/lib/explorer/migrator/backfill_token_counters.ex rename apps/explorer/lib/explorer/utility/{address_counters_refetch_block.ex => counters_refetch_block.ex} (55%) create mode 100644 apps/explorer/priv/repo/migrations/20260831000002_add_counters_updated_at_to_tokens.exs create mode 100644 apps/explorer/priv/repo/migrations/20260831000003_rename_address_counters_refetch_blocks.exs create mode 100644 apps/explorer/test/explorer/chain/cache/counters/token_counters_consolidator_test.exs create mode 100644 apps/explorer/test/explorer/chain/cache/counters/token_counters_test.exs create mode 100644 apps/explorer/test/explorer/migrator/backfill_token_counters_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index 42d4716f8ea..09c0a1f3447 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -148,8 +148,8 @@ defmodule BlockScoutWeb.API.V2.TokenController do def counters(conn, %{address_hash_param: address_hash_string} = params) do with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)}, {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params), - {:not_found, true} <- {:not_found, Token.by_contract_address_hash_exists?(address_hash, @api_true)} do - {transfers_count, holders_count} = Token.fetch_token_counters(address_hash, 5_000) + {:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash, @api_true)} do + {transfers_count, holders_count} = Token.fetch_token_counters(token) json(conn, %{transfers_count: to_string(transfers_count), token_holders_count: to_string(holders_count)}) end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex index c7661debd32..e8e8af5c5b3 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/token_controller.ex @@ -13,12 +13,12 @@ defmodule BlockScoutWeb.Tokens.TokenController do end def token_counters(conn, %{"id" => address_hash_string}) do - case Chain.string_to_address_hash(address_hash_string) do - {:ok, address_hash} -> - {transfers_count, holders_count} = Token.fetch_token_counters(address_hash, 30_000) - - json(conn, %{transfer_count: transfers_count, token_holder_count: holders_count}) + with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), + %Token{} = token <- Token.get_by_contract_address_hash(address_hash, []) do + {transfers_count, holders_count} = Token.fetch_token_counters(token) + json(conn, %{transfer_count: transfers_count, token_holder_count: holders_count}) + else _ -> not_found(conn) end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/token_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/token_controller_test.exs index ecb3a77c381..6162c188542 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/token_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/token_controller_test.exs @@ -10,8 +10,9 @@ defmodule BlockScoutWeb.API.V2.TokenControllerTest do alias Explorer.{Repo, TestHelper} - alias Explorer.Chain.{Address, Token, Token.Instance, TokenTransfer} + alias Explorer.Chain.{Address, Block, Token, Token.Instance, TokenTransfer} alias Explorer.Chain.Address.CurrentTokenBalance + alias Explorer.Chain.Cache.Counters.TokenCountersConsolidator alias Explorer.Chain.Events.Subscriber alias Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetch, as: TokenInstanceMetadataRefetchOnDemand @@ -98,6 +99,8 @@ defmodule BlockScoutWeb.API.V2.TokenControllerTest do 3, :token_transfer, transaction: transaction, + block: transaction.block, + block_number: transaction.block_number, token_contract_address: contract_token_address ) @@ -111,10 +114,9 @@ defmodule BlockScoutWeb.API.V2.TokenControllerTest do ) ) - request = get(conn, "/api/v2/tokens/#{token.contract_address.hash}/counters") - assert json_response(request, 200) + safe_block = Repo.aggregate(Block, :max, :number) + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], safe_block) - Process.sleep(500) request = get(conn, "/api/v2/tokens/#{token.contract_address.hash}/counters") assert response = json_response(request, 200) diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 11dfd949dc5..15a2236a0a5 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -84,13 +84,9 @@ config :explorer, Explorer.Chain.Cache.Counters.Blackfort.ValidatorsCount, config :explorer, Explorer.Market.Fetcher.Token, enabled: true -config :explorer, Explorer.Chain.Cache.Counters.TokenHoldersCount, - enabled: true, - enable_consolidation: true +config :explorer, Explorer.Chain.Cache.Counters.TokenCounters, enabled: true -config :explorer, Explorer.Chain.Cache.Counters.TokenTransfersCount, - enabled: true, - enable_consolidation: true +config :explorer, Explorer.Chain.Cache.Counters.TokenCountersConsolidator, enabled: true config :explorer, Explorer.Chain.Cache.Counters.BlockBurntFeeCount, enabled: true, @@ -139,7 +135,8 @@ for migrator <- [ Explorer.Migrator.ReindexBlocksWithUncatalogedTokenTransfers, Explorer.Migrator.EmptyInternalTransactionsData, Explorer.Migrator.FillInternalTransactionsAddressIds, - Explorer.Migrator.BackfillAddressCounters + Explorer.Migrator.BackfillAddressCounters, + Explorer.Migrator.BackfillTokenCounters ] do config :explorer, migrator, enabled: true end diff --git a/apps/explorer/config/runtime/test.exs b/apps/explorer/config/runtime/test.exs index 84e068f714e..264a1f12e1e 100644 --- a/apps/explorer/config/runtime/test.exs +++ b/apps/explorer/config/runtime/test.exs @@ -17,6 +17,8 @@ config :explorer, Explorer.Chain.Transaction.History.Historian, enabled: false config :explorer, Explorer.Chain.Cache.Counters.AddressCounters, enabled: false config :explorer, Explorer.Chain.Cache.Counters.AddressCountersConsolidator, enabled: false +config :explorer, Explorer.Chain.Cache.Counters.TokenCounters, enabled: false +config :explorer, Explorer.Chain.Cache.Counters.TokenCountersConsolidator, enabled: false for counter <- [ Explorer.Chain.Cache.Counters.AddressesCount, @@ -85,6 +87,7 @@ for migrator <- [ Explorer.Migrator.EmptyInternalTransactionsData, Explorer.Migrator.FillInternalTransactionsAddressIds, Explorer.Migrator.BackfillAddressCounters, + Explorer.Migrator.BackfillTokenCounters, # Heavy DB index operations Explorer.Migrator.HeavyDbIndexOperation.CreateLogsBlockHashIndex, diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index ee94f58bba9..528b161cc3f 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -155,8 +155,8 @@ defmodule Explorer.Application do configure(Explorer.Chain.Cache.Counters.AddressCounters), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressCountersConsolidator, :indexer), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.AddressTokensUsdSum, :api), - configure(Explorer.Chain.Cache.Counters.TokenHoldersCount), - configure(Explorer.Chain.Cache.Counters.TokenTransfersCount), + configure(Explorer.Chain.Cache.Counters.TokenCounters), + configure_mode_dependent_process(Explorer.Chain.Cache.Counters.TokenCountersConsolidator, :indexer), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.BlockBurntFeeCount, :api), configure_mode_dependent_process(Explorer.Chain.Cache.Counters.BlockPriorityFeeCount, :api), configure(Explorer.Chain.Cache.Counters.AverageBlockTime), @@ -190,6 +190,7 @@ defmodule Explorer.Application do configure_mode_dependent_process(Explorer.Migrator.RestoreOmittedWETHTransfers, :indexer), configure_mode_dependent_process(Explorer.Migrator.FilecoinPendingAddressOperations, :indexer), configure_mode_dependent_process(Explorer.Migrator.BackfillAddressCounters, :indexer), + configure_mode_dependent_process(Explorer.Migrator.BackfillTokenCounters, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloL2Epochs, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloAccounts, :indexer), configure_mode_dependent_process(Explorer.Migrator.CeloAggregatedElectionRewards, :indexer), diff --git a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex index 8b3339ff0fe..7b70f201d45 100644 --- a/apps/explorer/lib/explorer/chain/cache/background_migrations.ex +++ b/apps/explorer/lib/explorer/chain/cache/background_migrations.ex @@ -71,6 +71,7 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do key: :heavy_indexes_create_address_ids_internal_transactions_indexes_finished, key: :fill_internal_transactions_address_ids_finished, key: :backfill_address_counters_finished, + key: :backfill_token_counters_finished, key: :heavy_indexes_create_logs_address_hash_first_topic_second_topic_block_number_index_finished, key: :heavy_indexes_create_address_current_token_balances_address_hash_block_number_index_finished @@ -82,6 +83,7 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do ArbitrumDaRecordsNormalization, BackfillAddressCounters, BackfillMultichainSearchDB, + BackfillTokenCounters, EmptyInternalTransactionsData, FillInternalTransactionsAddressIds, SanitizeDuplicatedLogIndexLogs, @@ -435,6 +437,13 @@ defmodule Explorer.Chain.Cache.BackgroundMigrations do ) end + defp handle_fallback(:backfill_token_counters_finished) do + set_and_return_migration_status( + BackfillTokenCounters, + &set_backfill_token_counters_finished/1 + ) + end + defp handle_fallback(:heavy_indexes_create_logs_address_hash_first_topic_second_topic_block_number_index_finished) do set_and_return_migration_status( CreateLogsAddressHashFirstTopicSecondTopicBlockNumberIndex, diff --git a/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex b/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex index e2f1b1d80b9..76d2b7f35bc 100644 --- a/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex +++ b/apps/explorer/lib/explorer/chain/cache/counters/address_counters_consolidator.ex @@ -22,11 +22,12 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do that catchup and block re-fetch cannot make consolidated ranges lose or double-count rows. - Each cycle starts by settling pending block re-fetch corrections: for every - block whose old content was subtracted when it was queued for re-fetch (see - `Explorer.Utility.AddressCountersRefetchBlock`) and which has been fully - re-imported since, the new content is added back to the counters of the - addresses whose watermark already covers the block. + Each cycle starts by settling pending block re-fetch corrections via + `Explorer.Chain.Cache.Counters.Consolidation.settle_pending_refetch_blocks/0`: + for every block whose old content was subtracted when it was queued for + re-fetch (see `Explorer.Utility.CountersRefetchBlock`) and which has been + fully re-imported since, the new content is added back to the counters + covered by the consolidation watermarks. Known accepted limitations: @@ -45,15 +46,13 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do require Logger - alias Explorer.Chain.{Address, Block, Hash, TokenTransfer, Transaction} + alias Explorer.Chain.{Address, Hash} alias Explorer.Chain.Address.Counters alias Explorer.Chain.Cache.BlockNumber - alias Explorer.Chain.Cache.Counters.{AddressCounters, Helper} + alias Explorer.Chain.Cache.Counters.{AddressCounters, Consolidation, Helper} alias Explorer.Repo - alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} @int4_max 2_147_483_647 - @refetch_blocks_chunk_size 100 @spec start_link(term()) :: GenServer.on_start() def start_link(_) do @@ -110,7 +109,7 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do """ @spec consolidate() :: :ok def consolidate do - apply_pending_refetch_deltas() + Consolidation.settle_pending_refetch_blocks() unless AddressCounters.dirty_empty?() do case safe_block() do @@ -123,59 +122,9 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do end @doc """ - Returns the highest block number consolidated ranges may currently cover or - `nil` when consolidation is not possible yet: the chain head minus the - configured safety lag, additionally capped below the lowest missing block - range and the lowest block pending a re-fetch counter correction. - - While the chain is not yet indexed down to the configured first block — - initial backward sync, before the missing ranges collector has recorded the - unindexed tail — no safe block exists at all: consolidating past blocks that - are absent but not yet registered in `missing_block_ranges` would lose their - contributions forever. + See `Explorer.Chain.Cache.Counters.Consolidation.safe_block/0`. """ - @spec safe_block() :: non_neg_integer() | nil - def safe_block do - if lower_chain_indexed?() do - [lagged_head(), min_missing_block_cap(), min_pending_refetch_cap()] - |> Enum.reject(&is_nil/1) - |> Enum.min(fn -> nil end) - |> case do - safe_block when is_integer(safe_block) and safe_block > 0 -> safe_block - _ -> nil - end - end - end - - defp lower_chain_indexed? do - first_block = Application.get_env(:indexer, :first_block) - - case BlockNumber.get_min() do - min_block_number when is_integer(min_block_number) -> min_block_number <= first_block - _ -> false - end - end - - defp lagged_head do - case BlockNumber.get_max() do - max_block_number when is_integer(max_block_number) -> max_block_number - safe_block_lag() - _ -> nil - end - end - - defp min_missing_block_cap do - case MissingBlockRange.fetch_min_max() do - %{min: min} when is_integer(min) -> min - 1 - _ -> nil - end - end - - defp min_pending_refetch_cap do - case AddressCountersRefetchBlock.min_block_number() do - block_number when is_integer(block_number) -> block_number - 1 - _ -> nil - end - end + defdelegate safe_block, to: Consolidation @doc """ Consolidates the given addresses up to `safe_block` synchronously: full @@ -422,90 +371,6 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do Repo.all(query, timeout: query_timeout()) end - ## Block re-fetch corrections (positive side) - - defp apply_pending_refetch_deltas do - # a block is ready to settle once its re-fetch completed: the consensus - # block no longer requires a re-fetch AND the block is no longer covered - # by a missing range (the range row survives until the whole re-import — - # all its stages — succeeded, so old content can no longer be observed) - ready_query = - from(refetch_block in AddressCountersRefetchBlock, - as: :refetch_block, - where: - not exists( - from(block in Block, - where: - block.number == parent_as(:refetch_block).block_number and block.consensus == true and - block.refetch_needed == true, - select: 1 - ) - ), - where: - not exists( - from(range in MissingBlockRange, - where: - range.from_number >= parent_as(:refetch_block).block_number and - range.to_number <= parent_as(:refetch_block).block_number, - select: 1 - ) - ), - select: refetch_block.block_number, - limit: @refetch_blocks_chunk_size - ) - - case Repo.all(ready_query, timeout: query_timeout()) do - [] -> - :ok - - block_numbers -> - settle_refetched_blocks(block_numbers) - # more rows may be ready - apply_pending_refetch_deltas() - end - rescue - error -> - Logger.error(fn -> - ["Failed to settle re-fetched blocks counters: ", Exception.format(:error, error, __STACKTRACE__)] - end) - - :ok - end - - defp settle_refetched_blocks(block_numbers) do - {:ok, updated_bytes} = - Repo.transaction( - fn -> - transactions = - Repo.all( - from(transaction in Transaction, - where: transaction.block_number in ^block_numbers, - select: struct(transaction, [:block_number, :from_address_hash, :to_address_hash, :gas_used]) - ), - timeout: query_timeout() - ) - - token_transfers = - Repo.all( - from(token_transfer in TokenTransfer, - where: token_transfer.block_number in ^block_numbers, - select: struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash]) - ), - timeout: query_timeout() - ) - - updated_bytes = apply_covered_deltas(transactions, token_transfers, :positive) - - AddressCountersRefetchBlock.delete_by_block_numbers(Enum.sort(block_numbers)) - - updated_bytes - end, - timeout: :infinity - ) - - AddressCounters.invalidate(updated_bytes) - end - @doc """ Applies the per-address counter deltas of the given transactions and token transfers directly to the `addresses` columns, restricted per address to the @@ -545,9 +410,9 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do a full counters recalculation on their next consolidation — when their watermark already covers the given block number (a covered range changed under it: deep reorg, or rows inserted below the watermark by an - out-of-band importer). Returns the hash bytes of the reset addresses; the - caller is responsible for invalidating the display cache and marking them - dirty. + out-of-band importer). Reset addresses are marked dirty so the recalculation + is actually scheduled. Returns the hash bytes of the reset addresses; the + caller is responsible for invalidating the display cache. """ @spec reset_covered_watermarks(%{binary() => non_neg_integer()}, Ecto.Repo.t()) :: [binary()] def reset_covered_watermarks(min_block_by_hash_bytes, repo \\ Repo) @@ -575,7 +440,13 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do {_count, reset_hashes} = repo.update_all(query, [], timeout: query_timeout()) - Enum.map(reset_hashes, & &1.bytes) + reset_bytes = Enum.map(reset_hashes, & &1.bytes) + + # a spurious marker on transaction rollback is harmless (the guarded + # consolidation of a non-reset address is a no-op) + AddressCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + + reset_bytes end defp apply_signed_deltas(deltas, watermarks, sign, repo) do @@ -711,10 +582,6 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidator do Application.get_env(:explorer, __MODULE__)[:concurrency] || 4 end - defp safe_block_lag do - Application.get_env(:explorer, __MODULE__)[:safe_block_lag] || 12 - end - defp query_timeout do Application.get_env(:explorer, __MODULE__)[:query_timeout] || :timer.minutes(5) end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/consolidation.ex b/apps/explorer/lib/explorer/chain/cache/counters/consolidation.ex new file mode 100644 index 00000000000..155f8c55d44 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/counters/consolidation.ex @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.Consolidation do + @moduledoc """ + Plumbing shared by the incremental counter consolidators + (`Explorer.Chain.Cache.Counters.AddressCountersConsolidator` and + `Explorer.Chain.Cache.Counters.TokenCountersConsolidator`): + + * `safe_block/0` — the highest block number consolidated ranges may cover; + * `settle_pending_refetch_blocks/0` — applies the pending positive counter + corrections for re-fetched blocks (see `Explorer.Utility.CountersRefetchBlock`) + to both the address and the token counters. + + Both consolidators call the settle at the start of their cycles; a + transaction-scoped advisory lock guarantees the corrections are applied + exactly once even when the cycles overlap. + """ + + import Ecto.Query + + require Logger + + alias Explorer.Chain.{Block, TokenTransfer, Transaction} + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator, TokenCounters} + alias Explorer.Chain.Cache.Counters.TokenCountersConsolidator + alias Explorer.Repo + alias Explorer.Utility.{CountersRefetchBlock, MissingBlockRange} + + @refetch_blocks_chunk_size 100 + + # arbitrary constant identifying the settle critical section among + # `pg_advisory_xact_lock` users + @settle_advisory_lock_key 7_235_622_386_001 + + @doc """ + Returns the highest block number consolidated ranges may currently cover or + `nil` when consolidation is not possible yet: the chain head minus the + configured safety lag, additionally capped below the lowest missing block + range and the lowest block pending a re-fetch counter correction. + + While the chain is not yet indexed down to the configured first block — + initial backward sync, before the missing ranges collector has recorded the + unindexed tail — no safe block exists at all: consolidating past blocks that + are absent but not yet registered in `missing_block_ranges` would lose their + contributions forever. + """ + @spec safe_block() :: non_neg_integer() | nil + def safe_block do + if lower_chain_indexed?() do + [lagged_head(), min_missing_block_cap(), min_pending_refetch_cap()] + |> Enum.reject(&is_nil/1) + |> Enum.min(fn -> nil end) + |> case do + safe_block when is_integer(safe_block) and safe_block > 0 -> safe_block + _ -> nil + end + end + end + + @doc """ + Returns the configured number of blocks the consolidation watermarks stay + behind the chain head. + """ + @spec safe_block_lag() :: non_neg_integer() + def safe_block_lag do + Application.get_env(:explorer, __MODULE__)[:safe_block_lag] || 12 + end + + @doc """ + Settles the pending block re-fetch corrections: for every queued block whose + re-fetch fully completed, adds the counter contributions of the re-imported + content back to the addresses and tokens whose watermarks already cover the + block, then drops the queue rows. Blocks whose re-import is still in flight + (the consensus block still requires a re-fetch, or the block is still + covered by a missing range) are left queued. + """ + @spec settle_pending_refetch_blocks() :: :ok + def settle_pending_refetch_blocks do + case Repo.all(ready_to_settle_query(), timeout: query_timeout()) do + [] -> + :ok + + block_numbers -> + case settle_refetched_blocks(block_numbers) do + # another settle is in progress on this or another node + :skipped -> :ok + # more rows may be ready + :ok -> settle_pending_refetch_blocks() + end + end + rescue + error -> + Logger.error(fn -> + ["Failed to settle re-fetched blocks counters: ", Exception.format(:error, error, __STACKTRACE__)] + end) + + :ok + end + + # a block is ready to settle once its re-fetch completed: the consensus + # block no longer requires a re-fetch AND the block is no longer covered + # by a missing range (the range row survives until the whole re-import — + # all its stages — succeeded, so old content can no longer be observed) + defp ready_to_settle_query do + from(refetch_block in CountersRefetchBlock, + as: :refetch_block, + where: + not exists( + from(block in Block, + where: + block.number == parent_as(:refetch_block).block_number and block.consensus == true and + block.refetch_needed == true, + select: 1 + ) + ), + where: + not exists( + from(range in MissingBlockRange, + where: + range.from_number >= parent_as(:refetch_block).block_number and + range.to_number <= parent_as(:refetch_block).block_number, + select: 1 + ) + ), + select: refetch_block.block_number, + limit: @refetch_blocks_chunk_size + ) + end + + defp settle_refetched_blocks(block_numbers) do + {:ok, result} = + Repo.transaction( + fn -> + if advisory_lock_acquired?() do + transactions = + Repo.all( + from(transaction in Transaction, + where: transaction.block_number in ^block_numbers, + select: struct(transaction, [:block_number, :from_address_hash, :to_address_hash, :gas_used]) + ), + timeout: query_timeout() + ) + + token_transfers = + Repo.all( + from(token_transfer in TokenTransfer, + where: token_transfer.block_number in ^block_numbers, + select: + struct(token_transfer, [ + :block_number, + :from_address_hash, + :to_address_hash, + :token_contract_address_hash + ]) + ), + timeout: query_timeout() + ) + + updated_address_bytes = + AddressCountersConsolidator.apply_covered_deltas(transactions, token_transfers, :positive) + + updated_token_bytes = TokenCountersConsolidator.apply_covered_transfer_deltas(token_transfers, :positive) + + CountersRefetchBlock.delete_by_block_numbers(Enum.sort(block_numbers)) + + {updated_address_bytes, updated_token_bytes} + else + :skipped + end + end, + timeout: :infinity + ) + + case result do + :skipped -> + :skipped + + {updated_address_bytes, updated_token_bytes} -> + AddressCounters.invalidate(updated_address_bytes) + TokenCounters.invalidate(updated_token_bytes) + :ok + end + end + + defp advisory_lock_acquired? do + %{rows: [[acquired]]} = Repo.query!("SELECT pg_try_advisory_xact_lock($1)", [@settle_advisory_lock_key]) + + acquired + end + + defp lower_chain_indexed? do + first_block = Application.get_env(:indexer, :first_block) + + case BlockNumber.get_min() do + min_block_number when is_integer(min_block_number) -> min_block_number <= first_block + _ -> false + end + end + + defp lagged_head do + case BlockNumber.get_max() do + max_block_number when is_integer(max_block_number) -> max_block_number - safe_block_lag() + _ -> nil + end + end + + defp min_missing_block_cap do + case MissingBlockRange.fetch_min_max() do + %{min: min} when is_integer(min) -> min - 1 + _ -> nil + end + end + + defp min_pending_refetch_cap do + case CountersRefetchBlock.min_block_number() do + block_number when is_integer(block_number) -> block_number - 1 + _ -> nil + end + end + + defp query_timeout do + Application.get_env(:explorer, __MODULE__)[:query_timeout] || :timer.minutes(5) + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/token_counters.ex b/apps/explorer/lib/explorer/chain/cache/counters/token_counters.ex new file mode 100644 index 00000000000..a18f16943f5 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/counters/token_counters.ex @@ -0,0 +1,411 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.TokenCounters do + @moduledoc """ + Shared ETS cache for the per-token transfers counter. + + The durable value lives in the `tokens.transfer_count` column and is + advanced incrementally by + `Explorer.Chain.Cache.Counters.TokenCountersConsolidator` up to the block + number stored in `tokens.counters_updated_at`. The per-token holders counter + (`tokens.holder_count`) is intentionally NOT cached here: it is maintained + live by the import-time deltas of the current token balances runner, so the + column itself is always current. + + This module keeps two ETS tables: + + * `:token_counters` — the display cache consumed by the API. Entries are + seeded from the `transfer_count` column on read (`fetch/1`), incremented + in place with the deltas of newly imported token transfers + (`handle_new_data/2`), re-based from the DB by the consolidator + (`refresh_from_consolidation/3`) and evicted a configurable TTL after + their last authoritative base, so accumulated live-bump drift is bounded + by the TTL. Live bumps never create entries. + + * `:token_counters_dirty` — the consolidator work list: token contract + address hash bytes mapped to the maximum block number seen for the token + since its last consolidation. Markers are written on nodes where the + consolidator runs (`:indexer`/`:all` modes) and are capped at a + configurable size — a dropped marker only delays consolidation until the + token's next activity. + + On pure API nodes of split deployments the display cache is kept warm by the + realtime `:token_transfers` chain events (dirty markers are not written + there); on indexer and all-mode nodes `Explorer.Chain.Import` calls + `handle_new_data/2` directly, so chain events are ignored there to avoid + double counting. + """ + + use GenServer + + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.Helper + alias Explorer.Chain.Events.Subscriber + alias Explorer.Chain.Hash + + @cache_name :token_counters + @dirty_cache_name :token_counters_dirty + + @typedoc """ + Per-token contribution of a set of imported (or deleted) token transfers. + """ + @type delta :: %{ + transfer_count: non_neg_integer(), + max_block_number: non_neg_integer() + } + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + Helper.create_cache_table(@cache_name, write_concurrency: true) + Helper.create_cache_table(@dirty_cache_name, write_concurrency: true) + + if Explorer.mode() == :api do + Subscriber.to(:token_transfers, :realtime) + end + + schedule_eviction() + + {:ok, %{}} + end + + @doc """ + Returns the transfers counter for the given token: the cached value when + present, the `tokens.transfer_count` column of the given struct otherwise + (seeding the cache along the way). A token whose counter was never + calculated (`NULL` column) is reported as zero and marked dirty so that the + consolidator computes its value ahead of the background backfill (on nodes + with a local consolidator). + """ + @spec fetch(Explorer.Chain.Token.t()) :: non_neg_integer() + def fetch(token) do + if cache_table_exists?() do + bytes = hash_bytes(token.contract_address_hash) + + case snapshot(bytes) do + nil -> seed_from_db(token, bytes) + transfer_count -> transfer_count + end + else + token.transfer_count || 0 + end + end + + @doc """ + Registers newly imported `token_transfers` in the cache: marks the involved + tokens dirty for the consolidator and, when `live?` is `true` (realtime + imports), increments the already-cached display values. + """ + @spec handle_new_data([map()], boolean()) :: :ok + def handle_new_data(token_transfers, live?) do + if cache_table_exists?() do + token_transfers + |> compute_transfer_deltas() + |> apply_new_deltas(live?) + end + + :ok + end + + defp apply_new_deltas(deltas, _live?) when map_size(deltas) == 0, do: :ok + + defp apply_new_deltas(deltas, live?) do + if live? do + Enum.each(deltas, fn {bytes, delta} -> bump_live(bytes, delta) end) + end + + deltas + |> Enum.map(fn {bytes, delta} -> {bytes, delta.max_block_number} end) + |> mark_dirty() + end + + @doc """ + Computes per-token transfer-count deltas for the given token transfers. Rows + without a block number are skipped. The optional `include?` predicate + receives the token contract hash bytes and the row block number and can + exclude single contributions (used to restrict deltas to blocks at or below + a token consolidation watermark). + """ + @spec compute_transfer_deltas([map()], (binary(), non_neg_integer() -> boolean())) :: %{binary() => delta()} + def compute_transfer_deltas(token_transfers, include? \\ fn _hash_bytes, _block_number -> true end) do + Enum.reduce(token_transfers, %{}, &add_transfer_delta(&2, &1, include?)) + end + + defp add_transfer_delta(acc, token_transfer, include?) do + bytes = hash_bytes(token_transfer.token_contract_address_hash) + block_number = token_transfer.block_number + + if is_nil(bytes) or is_nil(block_number) or not include?.(bytes, block_number) do + acc + else + acc + |> Map.put_new(bytes, %{transfer_count: 0, max_block_number: 0}) + |> Map.update!(bytes, fn delta -> + %{ + transfer_count: delta.transfer_count + 1, + max_block_number: max(delta.max_block_number, block_number) + } + end) + end + end + + @doc """ + Increments the cached display value of the given token by the given delta. + Only already-cached tokens are updated — live bumps never create entries. + """ + @spec bump_live(binary(), delta()) :: :ok + def bump_live(hash_bytes, delta) do + if :ets.member(@cache_name, {hash_bytes, :transfer_count}) do + :ets.update_counter(@cache_name, {hash_bytes, :transfer_count}, delta.transfer_count) + end + + :ok + rescue + # the entry was evicted between the membership check and the update + ArgumentError -> :ok + end + + @doc """ + Returns the currently cached transfers counter of the given token or `nil` + when the token is not cached. + """ + @spec snapshot(binary()) :: non_neg_integer() | nil + def snapshot(hash_bytes) do + if cache_table_exists?() do + Helper.fetch_from_ets_cache(@cache_name, {hash_bytes, :transfer_count}) + end + end + + @doc """ + Re-bases the cached display value of the given token on the value just + written to the DB by the consolidator. The write is an adjustment + (`new value - snapshot`) rather than an overwrite, so live bumps applied + concurrently — always for blocks above the consolidated range — survive on + top of the new base. + """ + @spec refresh_from_consolidation(binary(), non_neg_integer(), non_neg_integer() | nil) :: :ok + def refresh_from_consolidation(hash_bytes, new_transfer_count, snapshot) do + if not is_nil(snapshot) and :ets.member(@cache_name, {hash_bytes, :transfer_count}) do + :ets.update_counter(@cache_name, {hash_bytes, :transfer_count}, new_transfer_count - snapshot) + :ets.insert(@cache_name, {{hash_bytes, :ts}, Helper.current_time()}) + end + + :ok + rescue + # the entry was evicted between the membership check and the update + ArgumentError -> :ok + end + + @doc """ + Drops the cached display values of the given tokens so that the next read + re-seeds them from the DB. Used after out-of-band DB counter corrections + (block re-fetch deltas, watermark resets). + """ + @spec invalidate([binary()]) :: :ok + def invalidate(hash_bytes_list) do + if cache_table_exists?() do + Enum.each(hash_bytes_list, &delete_display_entries/1) + end + + :ok + end + + @doc """ + Marks tokens dirty for the consolidator. Accepts a list of + `{hash_bytes, block_number}` pairs; for every token the maximum seen block + number is kept. The update is serialized through the cache process to avoid + losing the maximum on concurrent writes. + + A no-op on nodes without a local consolidator (pure API mode). + """ + @spec mark_dirty([{binary(), non_neg_integer()}]) :: :ok + def mark_dirty(pairs) when is_list(pairs) do + if consolidator_local?() do + GenServer.cast(__MODULE__, {:mark_dirty, pairs}) + end + + :ok + end + + @doc """ + Marks the given token dirty at the current maximum block number, asking the + consolidator to (re)calculate its counters from scratch when + `counters_updated_at` is `NULL` or incrementally otherwise. + """ + @spec mark_dirty_at_max_block(Hash.Address.t()) :: :ok + def mark_dirty_at_max_block(contract_address_hash) do + mark_dirty([{hash_bytes(contract_address_hash), BlockNumber.get_max()}]) + end + + @doc """ + Returns `true` when there are no dirty markers (or the cache is not started). + """ + @spec dirty_empty?() :: boolean() + def dirty_empty? do + :ets.whereis(@dirty_cache_name) == :undefined or :ets.info(@dirty_cache_name, :size) == 0 + end + + @doc """ + Returns cache statistics for inspection (the process state itself is empty — + all data lives in the ETS tables): the number of tokens in the display + cache, the number of dirty markers awaiting consolidation, and the memory + held by both tables in bytes. + """ + @spec stats() :: %{ + cached_tokens: non_neg_integer(), + dirty_markers: non_neg_integer(), + memory_bytes: non_neg_integer() + } + def stats do + if cache_table_exists?() do + word_size = :erlang.system_info(:wordsize) + + %{ + cached_tokens: :ets.select_count(@cache_name, [{{{:_, :ts}, :_}, [], [true]}]), + dirty_markers: :ets.info(@dirty_cache_name, :size), + memory_bytes: (:ets.info(@cache_name, :memory) + :ets.info(@dirty_cache_name, :memory)) * word_size + } + else + %{cached_tokens: 0, dirty_markers: 0, memory_bytes: 0} + end + end + + @typedoc """ + Opaque `:ets.select/1` continuation of a dirty-markers traversal. + """ + @type dirty_markers_continuation :: term() + + @doc """ + Starts (with a limit) or continues (with a previously returned continuation) + a traversal of the dirty markers. Returns `{[{hash_bytes, block_number}], continuation}` + or `:"$end_of_table"`. + """ + @spec select_dirty(pos_integer() | dirty_markers_continuation()) :: + {[{binary(), non_neg_integer()}], dirty_markers_continuation()} | :"$end_of_table" + def select_dirty(limit) when is_integer(limit) do + :ets.select(@dirty_cache_name, [{{:"$1", :"$2"}, [], [{{:"$1", :"$2"}}]}], limit) + end + + def select_dirty(continuation) do + :ets.select(continuation) + end + + @doc """ + Deletes the dirty markers of the given tokens unless a concurrent import + bumped them above `safe_block` in the meantime (such markers stay for the + next consolidation cycle). + """ + @spec delete_dirty_markers([binary()], non_neg_integer()) :: :ok + def delete_dirty_markers(hash_bytes_list, safe_block) do + if :ets.whereis(@dirty_cache_name) != :undefined do + Enum.each(hash_bytes_list, fn bytes -> + :ets.select_delete(@dirty_cache_name, [{{bytes, :"$1"}, [{:"=<", :"$1", safe_block}], [true]}]) + end) + end + + :ok + end + + @doc """ + Converts a token contract address hash (or already-extracted hash bytes) to + the binary key used in the cache tables. + """ + @spec hash_bytes(Hash.Address.t() | binary() | nil) :: binary() | nil + def hash_bytes(nil), do: nil + def hash_bytes(%Hash{bytes: bytes}), do: bytes + def hash_bytes(bytes) when is_binary(bytes), do: bytes + + def cache_name, do: @cache_name + def dirty_cache_name, do: @dirty_cache_name + + @impl true + def handle_cast({:mark_dirty, pairs}, state) do + Enum.each(pairs, fn {bytes, block_number} -> insert_dirty_marker(bytes, block_number) end) + + {:noreply, state} + end + + defp insert_dirty_marker(bytes, block_number) do + case :ets.lookup(@dirty_cache_name, bytes) do + [{_, existing}] when existing >= block_number -> + :ok + + [{_, _existing}] -> + :ets.insert(@dirty_cache_name, {bytes, block_number}) + + [] -> + # cap the work list: a dropped marker only delays consolidation until + # the token's next activity (during initial sync all tokens are + # covered by the backfill migration anyway) + if :ets.info(@dirty_cache_name, :size) < max_dirty_markers() do + :ets.insert(@dirty_cache_name, {bytes, block_number}) + end + end + end + + @impl true + def handle_info({:chain_event, :token_transfers, :realtime, token_transfers}, state) do + handle_new_data(token_transfers, true) + {:noreply, state} + end + + def handle_info(:evict, state) do + evict_stale_entries() + schedule_eviction() + {:noreply, state} + end + + def handle_info(_, state), do: {:noreply, state} + + defp seed_from_db(token, bytes) do + if is_nil(token.transfer_count) do + mark_dirty([{bytes, BlockNumber.get_max()}]) + + 0 + else + :ets.insert_new(@cache_name, [ + {{bytes, :transfer_count}, token.transfer_count}, + {{bytes, :ts}, Helper.current_time()} + ]) + + token.transfer_count + end + end + + defp delete_display_entries(bytes) do + Enum.each([:transfer_count, :ts], fn key -> + :ets.delete(@cache_name, {bytes, key}) + end) + end + + defp evict_stale_entries do + threshold = Helper.current_time() - ttl() + + @cache_name + |> :ets.select([{{{:"$1", :ts}, :"$2"}, [{:<, :"$2", threshold}], [:"$1"]}]) + |> Enum.each(&delete_display_entries/1) + end + + defp cache_table_exists? do + :ets.whereis(@cache_name) != :undefined + end + + defp consolidator_local? do + Explorer.mode() in [:indexer, :all] + end + + defp max_dirty_markers do + Application.get_env(:explorer, __MODULE__)[:max_dirty_markers] || 200_000 + end + + defp schedule_eviction do + Process.send_after(self(), :evict, max(div(ttl(), 4), :timer.minutes(1))) + end + + defp ttl do + Application.get_env(:explorer, __MODULE__)[:ttl] || :timer.hours(2) + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/token_counters_consolidator.ex b/apps/explorer/lib/explorer/chain/cache/counters/token_counters_consolidator.ex new file mode 100644 index 00000000000..4d94e0fb6d2 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/counters/token_counters_consolidator.ex @@ -0,0 +1,602 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.TokenCountersConsolidator do + @moduledoc """ + The single writer of the per-token counters columns (`tokens.transfer_count` + and — on the initialization path only — `tokens.holder_count`). + + Periodically consolidates the counters of the tokens marked dirty in + `Explorer.Chain.Cache.Counters.TokenCounters` (a cycle is skipped entirely + when no token was marked since the last one): + + * a token with a `counters_updated_at` watermark gets a cheap range-bounded + `COUNT(*)` of its token transfers over `(counters_updated_at, safe_block]` + added to `transfer_count`; + * a token without a watermark (`NULL` — not yet backfilled, or reset because + covered content changed) gets a full transfer count bounded by + `safe_block`, and its `holder_count` re-established from a full recount. + + `holder_count` is otherwise maintained live by the import-time deltas of the + current token balances runner (state count — it cannot be recomputed from a + block range), so the recount here is written as a snapshot-relative + adjustment: the token is first "armed" (`holder_count` set to `0` when + `NULL`, enabling the delta guard), then the current count and the pre-count + column value are measured in one statement (one snapshot), and finally + `holder_count = holder_count - measured_old + recount` is applied — deltas + committed between the measurement and the guarded update survive on top of + the recount (READ COMMITTED re-reads the locked row, while the measured + values keep the measurement snapshot). + + `safe_block` (shared with the address counters consolidation — see + `Explorer.Chain.Cache.Counters.Consolidation`) keeps watermarks a + configurable lag behind the chain head and never advances past missing block + ranges or blocks pending re-fetch corrections. Token transfers of forked + blocks keep their physical rows (only `block_consensus` flips), and the + counter counts physical rows — legacy parity — so forks need no token + watermark resets; re-imported rows landing below a watermark are handled by + the reset valve in `Explorer.Chain.Import`. + + Known accepted limitations mirror the address consolidator: int4 clamping, + and content changes below a watermark landing while the very same token is + being consolidated can leave stale counts until the next watermark reset. + A one-off migrator deleting token transfer rows invalidates consolidated + transfer counts — reset `tokens.counters_updated_at` to `NULL` afterwards. + """ + + use GenServer + + import Ecto.Query + import Explorer.Chain.SmartContract, only: [burn_address_hash_string: 0] + + require Logger + + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Cache.Counters.{Consolidation, TokenCounters} + alias Explorer.Chain.{Hash, Token, TokenTransfer} + alias Explorer.Repo + + @int4_max 2_147_483_647 + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + def init(_args) do + schedule_next_consolidation(:timer.seconds(10)) + + {:ok, %{cycle_task: nil, cycle_started_at: nil, last_cycle_finished_at: nil}} + end + + # A cycle can run for a long time (draining a large dirty backlog, full + # recalculations of heavy tokens), so it runs in a separate supervised task: + # the process itself stays responsive to system messages (`:sys.get_state/1`, + # observer) and its state reports the running cycle. The next cycle is + # scheduled only once the previous one finished, so cycles never overlap. + @impl true + def handle_info(:consolidate, %{cycle_task: nil} = state) do + task = Task.Supervisor.async_nolink(Explorer.TaskSupervisor, &consolidate/0) + + {:noreply, %{state | cycle_task: task, cycle_started_at: DateTime.utc_now()}} + end + + # a stray tick while a cycle is still running — the next one is scheduled on + # its completion + def handle_info(:consolidate, state), do: {:noreply, state} + + def handle_info({ref, _result}, %{cycle_task: %Task{ref: ref}} = state) do + Process.demonitor(ref, [:flush]) + + {:noreply, finish_cycle(state)} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, %{cycle_task: %Task{ref: ref}} = state) do + Logger.error(fn -> ["Token counters consolidation cycle crashed: ", inspect(reason)] end) + + {:noreply, finish_cycle(state)} + end + + def handle_info(_, state), do: {:noreply, state} + + defp finish_cycle(state) do + schedule_next_consolidation(interval()) + + %{state | cycle_task: nil, cycle_started_at: nil, last_cycle_finished_at: DateTime.utc_now()} + end + + @doc """ + Runs one consolidation cycle synchronously. Exposed for tests and manual + runs; the supervised process runs it in a task on every `:consolidate` tick. + """ + @spec consolidate() :: :ok + def consolidate do + Consolidation.settle_pending_refetch_blocks() + + unless TokenCounters.dirty_empty?() do + case Consolidation.safe_block() do + nil -> :ok + safe_block -> consolidate_dirty(TokenCounters.select_dirty(batch_size() * concurrency()), safe_block) + end + end + + :ok + end + + @doc """ + Consolidates the given tokens up to `safe_block` synchronously: full + recalculation (transfer count and holder recount) for tokens without a + `counters_updated_at` watermark, incremental transfer-count range aggregates + for the rest. Used by `Explorer.Migrator.BackfillTokenCounters` and + available for on-demand recalculation. + """ + @spec consolidate_tokens([Hash.Address.t()], non_neg_integer()) :: :ok + def consolidate_tokens(contract_address_hashes, safe_block) do + contract_address_hashes + |> Enum.map(& &1.bytes) + |> load_tokens() + |> Enum.reject(&(&1.counters_updated_at && &1.counters_updated_at >= safe_block)) + |> case do + [] -> :ok + tokens -> consolidate_chunk(tokens, safe_block) + end + + :ok + end + + defp consolidate_dirty(:"$end_of_table", _safe_block), do: :ok + + defp consolidate_dirty({entries, continuation}, safe_block) do + process_entries(entries, safe_block) + consolidate_dirty(TokenCounters.select_dirty(continuation), safe_block) + end + + defp process_entries(entries, safe_block) do + entry_bytes = Enum.map(entries, fn {bytes, _marker_block} -> bytes end) + tokens_by_bytes = Map.new(load_tokens(entry_bytes), &{&1.contract_address_hash.bytes, &1}) + + {found_bytes, missing_bytes} = Enum.split_with(entry_bytes, &Map.has_key?(tokens_by_bytes, &1)) + + # markers without a corresponding token row are garbage + TokenCounters.delete_dirty_markers(missing_bytes, safe_block) + + found_bytes + |> Enum.map(&Map.fetch!(tokens_by_bytes, &1)) + |> Enum.reject(&(&1.counters_updated_at && &1.counters_updated_at >= safe_block)) + |> Enum.chunk_every(batch_size()) + |> Task.async_stream(&consolidate_chunk(&1, safe_block), max_concurrency: concurrency(), timeout: :infinity) + |> Stream.run() + end + + defp consolidate_chunk(tokens, safe_block) do + {incremental_tokens, inits} = Enum.split_with(tokens, & &1.counters_updated_at) + + updated_rows = + consolidate_incremental_tokens(incremental_tokens, safe_block) ++ consolidate_inits(inits, safe_block) + + Enum.each(updated_rows, fn %{token: token, transfer_count: new_transfer_count, snapshot: snapshot} -> + TokenCounters.refresh_from_consolidation(token.contract_address_hash.bytes, new_transfer_count || 0, snapshot) + end) + + updated_rows + |> Enum.map(& &1.token.contract_address_hash.bytes) + |> TokenCounters.delete_dirty_markers(safe_block) + end + + ## Incremental path + + defp consolidate_incremental_tokens([], _safe_block), do: [] + + defp consolidate_incremental_tokens(tokens, safe_block) do + results = + tokens + |> Enum.map(&compute_transfer_range_count(&1, safe_block)) + |> Enum.reject(&is_nil/1) + + apply_increments(results, safe_block) + end + + defp compute_transfer_range_count(token, safe_block) do + snapshot = TokenCounters.snapshot(token.contract_address_hash.bytes) + + transfer_count_delta = + token.contract_address_hash + |> TokenTransfer.count_token_transfers_from_token_hash_query(token.counters_updated_at, safe_block) + |> Repo.aggregate(:count, timeout: query_timeout()) + + %{token: token, snapshot: snapshot, transfer_count_delta: transfer_count_delta} + rescue + error -> + Logger.warning(fn -> + [ + "Failed to consolidate counters for token #{to_string(token.contract_address_hash)}: ", + Exception.format(:error, error, __STACKTRACE__) + ] + end) + + nil + end + + defp apply_increments([], _safe_block), do: [] + + defp apply_increments(results, safe_block) do + hashes = Enum.map(results, & &1.token.contract_address_hash.bytes) + expected_froms = Enum.map(results, & &1.token.counters_updated_at) + transfer_deltas = Enum.map(results, & &1.transfer_count_delta) + + query = + from( + token in Token, + join: + deltas in fragment( + "(SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS expected_from, unnest(?::bigint[]) AS transfer_delta)", + ^hashes, + ^expected_froms, + ^transfer_deltas + ), + on: token.contract_address_hash == deltas.hash, + where: token.contract_address_hash in subquery(tokens_lock_query(hashes)), + where: token.counters_updated_at == deltas.expected_from, + update: [ + set: [ + transfer_count: + fragment("LEAST(COALESCE(?, 0) + ?, ?)", token.transfer_count, deltas.transfer_delta, @int4_max), + counters_updated_at: ^safe_block, + updated_at: ^DateTime.utc_now() + ] + ], + select: %{hash: token.contract_address_hash, transfer_count: token.transfer_count} + ) + + {_count, rows} = Repo.update_all(query, [], timeout: query_timeout()) + + rows_by_bytes = Map.new(rows, &{&1.hash.bytes, &1}) + + results + |> Enum.filter(&Map.has_key?(rows_by_bytes, &1.token.contract_address_hash.bytes)) + |> Enum.map(fn result -> + %{ + token: result.token, + snapshot: result.snapshot, + transfer_count: Map.fetch!(rows_by_bytes, result.token.contract_address_hash.bytes).transfer_count + } + end) + end + + ## Initialization path (NULL watermark): arm -> measure -> apply + + defp consolidate_inits([], _safe_block), do: [] + + defp consolidate_inits(tokens, safe_block) do + hashes = Enum.map(tokens, & &1.contract_address_hash.bytes) + + arm_holder_counts(hashes) + + results = + tokens + |> Enum.map(&measure_token(&1, safe_block)) + |> Enum.reject(&is_nil/1) + + apply_inits(results, safe_block) + end + + # Sets `holder_count` to 0 where it is NULL so that the import-time delta + # machinery (guarded by `not is_nil(holder_count)`) accumulates every + # transition committed after this point; the measured recount then lands as + # an adjustment on top (see the moduledoc). + defp arm_holder_counts(hashes) do + query = + from(token in Token, + where: token.contract_address_hash in subquery(tokens_lock_query(hashes)), + where: is_nil(token.holder_count), + update: [set: [holder_count: 0, updated_at: ^DateTime.utc_now()]] + ) + + Repo.update_all(query, [], timeout: query_timeout()) + end + + defp measure_token(token, safe_block) do + # one statement = one snapshot: the pre-recount column value and the + # recount are consistent with each other + {old_holder_count, holder_recount} = + Repo.one!( + from(t in Token, + where: t.contract_address_hash == ^token.contract_address_hash, + select: { + coalesce(t.holder_count, 0), + fragment( + """ + (SELECT COUNT(DISTINCT ctb.address_hash) + FROM address_current_token_balances ctb + WHERE ctb.token_contract_address_hash = ? + AND ctb.address_hash <> ? + AND (ctb.value > 0 OR ctb.token_type = 'ERC-7984')) + """, + t.contract_address_hash, + ^burn_address_bytes() + ) + } + ), + timeout: query_timeout() + ) + + transfer_count = + token.contract_address_hash + |> TokenTransfer.count_token_transfers_from_token_hash_query(nil, safe_block) + |> Repo.aggregate(:count, timeout: query_timeout()) + + snapshot = TokenCounters.snapshot(token.contract_address_hash.bytes) + + %{ + token: token, + snapshot: snapshot, + old_holder_count: old_holder_count, + holder_recount: holder_recount, + transfer_count: transfer_count + } + rescue + error -> + Logger.warning(fn -> + [ + "Failed to recalculate counters for token #{to_string(token.contract_address_hash)}: ", + Exception.format(:error, error, __STACKTRACE__) + ] + end) + + nil + end + + defp apply_inits([], _safe_block), do: [] + + defp apply_inits(results, safe_block) do + hashes = Enum.map(results, & &1.token.contract_address_hash.bytes) + old_holder_counts = Enum.map(results, & &1.old_holder_count) + holder_recounts = Enum.map(results, & &1.holder_recount) + transfer_counts = Enum.map(results, & &1.transfer_count) + + query = + from( + token in Token, + join: + values in fragment( + """ + (SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS old_holder_count, + unnest(?::bigint[]) AS holder_recount, unnest(?::bigint[]) AS transfer_count) + """, + ^hashes, + ^old_holder_counts, + ^holder_recounts, + ^transfer_counts + ), + on: token.contract_address_hash == values.hash, + where: token.contract_address_hash in subquery(tokens_lock_query(hashes)), + where: is_nil(token.counters_updated_at), + update: [ + set: [ + # snapshot-relative: deltas committed since the measurement stay + holder_count: + fragment( + "LEAST(GREATEST(COALESCE(?, 0) - ? + ?, 0), ?)", + token.holder_count, + values.old_holder_count, + values.holder_recount, + @int4_max + ), + transfer_count: fragment("LEAST(?, ?)", values.transfer_count, @int4_max), + counters_updated_at: ^safe_block, + updated_at: ^DateTime.utc_now() + ] + ], + select: %{hash: token.contract_address_hash, transfer_count: token.transfer_count} + ) + + {_count, rows} = Repo.update_all(query, [], timeout: query_timeout()) + + rows_by_bytes = Map.new(rows, &{&1.hash.bytes, &1}) + + results + |> Enum.filter(&Map.has_key?(rows_by_bytes, &1.token.contract_address_hash.bytes)) + |> Enum.map(fn result -> + %{ + token: result.token, + snapshot: result.snapshot, + transfer_count: Map.fetch!(rows_by_bytes, result.token.contract_address_hash.bytes).transfer_count + } + end) + end + + ## Covered deltas (block re-fetch corrections) + + @doc """ + Applies the per-token transfer-count deltas of the given token transfers + directly to `tokens.transfer_count`, restricted per token to the blocks + already covered by its `counters_updated_at` watermark (contributions above + the watermark are left to regular range consolidation). `sign` selects + whether the contributions are added (`:positive` — re-imported content of + re-fetched blocks) or subtracted (`:negative` — old content of blocks queued + for re-fetch, see `Explorer.Chain.Import.Runner.Blocks`). + + Runs on `Explorer.Repo` unless another repo is given. Returns the hash bytes + of the updated tokens. + """ + @spec apply_covered_transfer_deltas([map()], :positive | :negative, Ecto.Repo.t()) :: [binary()] + def apply_covered_transfer_deltas(token_transfers, sign, repo \\ Repo) do + watermarks = + token_transfers + |> Enum.map(&TokenCounters.hash_bytes(&1.token_contract_address_hash)) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> load_watermarks(repo) + + deltas = + TokenCounters.compute_transfer_deltas(token_transfers, fn bytes, block_number -> + case watermarks[bytes] do + nil -> false + watermark -> block_number <= watermark + end + end) + + if map_size(deltas) == 0 do + [] + else + apply_signed_transfer_deltas(deltas, watermarks, sign, repo) + end + end + + @doc """ + Resets the `counters_updated_at` watermark of the given tokens — forcing a + full counters recalculation on their next consolidation — when their + watermark already covers the given block number (a covered range changed + under it: rows inserted below the watermark, or a missed correction guard). + Reset tokens are marked dirty so the recalculation is actually scheduled. + Returns the hash bytes of the reset tokens; the caller is responsible for + invalidating the display cache. + """ + @spec reset_covered_watermarks(%{binary() => non_neg_integer()}, Ecto.Repo.t()) :: [binary()] + def reset_covered_watermarks(min_block_by_hash_bytes, repo \\ Repo) + + def reset_covered_watermarks(min_block_by_hash_bytes, _repo) when map_size(min_block_by_hash_bytes) == 0, do: [] + + def reset_covered_watermarks(min_block_by_hash_bytes, repo) do + {hashes, min_blocks} = min_block_by_hash_bytes |> Enum.sort() |> Enum.unzip() + + query = + from( + token in Token, + join: + affected in fragment( + "(SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS block_number)", + ^hashes, + ^min_blocks + ), + on: token.contract_address_hash == affected.hash, + where: token.contract_address_hash in subquery(tokens_lock_query(hashes)), + where: not is_nil(token.counters_updated_at) and token.counters_updated_at >= affected.block_number, + update: [set: [counters_updated_at: nil, updated_at: ^DateTime.utc_now()]], + select: token.contract_address_hash + ) + + {_count, reset_hashes} = repo.update_all(query, [], timeout: query_timeout()) + + reset_bytes = Enum.map(reset_hashes, & &1.bytes) + + # a spurious marker on transaction rollback is harmless (the guarded + # consolidation of a non-reset token is a no-op) + TokenCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + + reset_bytes + end + + defp apply_signed_transfer_deltas(deltas, watermarks, sign, repo) do + multiplier = if sign == :negative, do: -1, else: 1 + + entries = Enum.to_list(deltas) + hashes = Enum.map(entries, fn {bytes, _delta} -> bytes end) + expected_watermarks = Enum.map(entries, fn {bytes, _delta} -> watermarks[bytes] end) + transfer_deltas = Enum.map(entries, fn {_bytes, delta} -> multiplier * delta.transfer_count end) + + query = + from( + token in Token, + join: + deltas in fragment( + "(SELECT unnest(?::bytea[]) AS hash, unnest(?::bigint[]) AS expected_watermark, unnest(?::bigint[]) AS transfer_delta)", + ^hashes, + ^expected_watermarks, + ^transfer_deltas + ), + on: token.contract_address_hash == deltas.hash, + where: token.contract_address_hash in subquery(tokens_lock_query(hashes)), + # only apply when the watermark is unchanged since the deltas were + # computed; changed tokens are reset below for a full recalculation + where: token.counters_updated_at == deltas.expected_watermark, + update: [ + set: [ + transfer_count: + fragment( + "LEAST(GREATEST(COALESCE(?, 0) + ?, 0), ?)", + token.transfer_count, + deltas.transfer_delta, + @int4_max + ), + updated_at: ^DateTime.utc_now() + ] + ], + select: token.contract_address_hash + ) + + {_count, updated_hashes} = repo.update_all(query, [], timeout: query_timeout()) + + updated_bytes = Enum.map(updated_hashes, & &1.bytes) + + # tokens whose watermark moved between the read and the guarded update + # cannot be corrected by a delta anymore — force a full recalculation + missed_reset_bytes = + (hashes -- updated_bytes) + |> Map.new(&{&1, 0}) + |> reset_covered_watermarks(repo) + + updated_bytes ++ missed_reset_bytes + end + + defp load_watermarks(token_bytes_list, repo) do + hashes = Enum.map(token_bytes_list, &%Hash{byte_count: 20, bytes: &1}) + + from(token in Token, + where: token.contract_address_hash in ^hashes and not is_nil(token.counters_updated_at), + select: {token.contract_address_hash, token.counters_updated_at} + ) + |> repo.all(timeout: query_timeout()) + |> Map.new(fn {hash, watermark} -> {hash.bytes, watermark} end) + end + + defp load_tokens(hash_bytes_list) do + hashes = Enum.map(hash_bytes_list, &%Hash{byte_count: 20, bytes: &1}) + + query = + from(token in Token, + where: token.contract_address_hash in ^hashes, + select: struct(token, [:contract_address_hash, :holder_count, :transfer_count, :counters_updated_at]) + ) + + Repo.all(query, timeout: query_timeout()) + end + + # Enforce Token ShareLocks order (see docs: sharelocks.md) + defp tokens_lock_query(hashes_bytes) do + hashes = Enum.map(hashes_bytes, &%Hash{byte_count: 20, bytes: &1}) + + from( + token in Token, + where: token.contract_address_hash in ^hashes, + select: token.contract_address_hash, + order_by: token.contract_address_hash, + lock: "FOR NO KEY UPDATE" + ) + end + + defp burn_address_bytes do + {:ok, burn_address_hash} = Hash.Address.cast(burn_address_hash_string()) + + burn_address_hash.bytes + end + + defp schedule_next_consolidation(timeout) do + Process.send_after(self(), :consolidate, timeout) + end + + defp interval do + Application.get_env(:explorer, __MODULE__)[:interval] || :timer.minutes(10) + end + + defp batch_size do + Application.get_env(:explorer, __MODULE__)[:batch_size] || 100 + end + + defp concurrency do + Application.get_env(:explorer, __MODULE__)[:concurrency] || 2 + end + + defp query_timeout do + Application.get_env(:explorer, __MODULE__)[:query_timeout] || :timer.minutes(5) + end +end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/token_holders_count.ex b/apps/explorer/lib/explorer/chain/cache/counters/token_holders_count.ex deleted file mode 100644 index 75860cef8d6..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/token_holders_count.ex +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.TokenHoldersCount do - @moduledoc """ - Caches Token holders count. - """ - use GenServer - use Utils.CompileTimeEnvHelper, enable_consolidation: [:explorer, [__MODULE__, :enable_consolidation]] - - alias Explorer.Chain.Address.CurrentTokenBalance - alias Explorer.Chain.Cache.Counters.Helper - alias Explorer.Chain.{Hash, Token} - - @api_true [api?: true] - @cache_name :token_holders_count - @ets_last_update_key "last_update" - - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_args) do - Helper.create_cache_table(@cache_name) - - {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} - end - - @impl true - def handle_continue(:ok, %{consolidate?: true} = state) do - {:noreply, state} - end - - @impl true - def handle_continue(:ok, state) do - {:noreply, state} - end - - @impl true - def handle_info(:consolidate, state) do - {:noreply, state} - end - - def fetch(address_hash) do - if cache_expired?(address_hash) do - update_cache(address_hash) - end - - fetch_count_from_cache(address_hash) - end - - def cache_name, do: @cache_name - - defp cache_expired?(address_hash) do - cache_period = Application.get_env(:explorer, __MODULE__)[:cache_period] - updated_at = fetch_updated_at_from_cache(address_hash, @cache_name) - - cond do - is_nil(updated_at) -> true - Helper.current_time() - updated_at > cache_period -> true - true -> false - end - end - - defp update_cache(address_hash) do - address_hash_string = to_string(address_hash) - new_data = CurrentTokenBalance.count_token_holders_from_token_hash(address_hash) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}", new_data) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}_#{@ets_last_update_key}", Helper.current_time()) - put_into_db_cache(address_hash, new_data) - end - - @doc """ - Fetches the token holders count from the cache or database. - """ - @spec fetch_count_from_cache(Hash.Address.t()) :: integer() - def fetch_count_from_cache(address_hash) do - address_hash_string = to_string(address_hash) - key = "hash_#{address_hash_string}" - - Helper.fetch_from_ets_cache(@cache_name, key) || fetch_from_db_cache(address_hash) - end - - defp fetch_updated_at_from_cache(address_hash, cache_name) do - address_hash_string = to_string(address_hash) - key = "hash_#{address_hash_string}_#{@ets_last_update_key}" - - Helper.fetch_from_ets_cache(cache_name, key) - end - - defp fetch_from_db_cache(address_hash) do - token = Token.get_by_contract_address_hash(address_hash, @api_true) - (token && token.holder_count) || 0 - end - - defp put_into_db_cache(address_hash, count) do - Token.update_token_holder_count(address_hash, count) - end - - defp enable_consolidation?, do: @enable_consolidation -end diff --git a/apps/explorer/lib/explorer/chain/cache/counters/token_transfers_count.ex b/apps/explorer/lib/explorer/chain/cache/counters/token_transfers_count.ex deleted file mode 100644 index c70d6a38fa8..00000000000 --- a/apps/explorer/lib/explorer/chain/cache/counters/token_transfers_count.ex +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Chain.Cache.Counters.TokenTransfersCount do - @moduledoc """ - Caches Token transfers counter. - """ - use GenServer - use Utils.CompileTimeEnvHelper, enable_consolidation: [:explorer, [__MODULE__, :enable_consolidation]] - - alias Explorer.Chain - alias Explorer.Chain.Cache.Counters.Helper - alias Explorer.Chain.{Hash, Token} - - @api_true [api?: true] - @cache_name :token_transfers_counter - @ets_last_update_key "last_update" - - @spec start_link(term()) :: GenServer.on_start() - def start_link(_) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @impl true - def init(_args) do - Helper.create_cache_table(@cache_name) - - {:ok, %{consolidate?: enable_consolidation?()}, {:continue, :ok}} - end - - @impl true - def handle_continue(:ok, %{consolidate?: true} = state) do - {:noreply, state} - end - - @impl true - def handle_continue(:ok, state) do - {:noreply, state} - end - - @impl true - def handle_info(:consolidate, state) do - {:noreply, state} - end - - def fetch(address_hash) do - if cache_expired?(address_hash) do - update_cache(address_hash) - end - - fetch_count_from_cache(address_hash) - end - - def cache_name, do: @cache_name - - defp cache_expired?(address_hash) do - cache_period = Application.get_env(:explorer, __MODULE__)[:cache_period] - updated_at = fetch_updated_at_from_cache(address_hash, @cache_name) - - cond do - is_nil(updated_at) -> true - Helper.current_time() - updated_at > cache_period -> true - true -> false - end - end - - defp update_cache(address_hash) do - address_hash_string = to_string(address_hash) - new_data = Chain.count_token_transfers_from_token_hash(address_hash) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}", new_data) - Helper.put_into_ets_cache(@cache_name, "hash_#{address_hash_string}_#{@ets_last_update_key}", Helper.current_time()) - put_into_db_cache(address_hash, new_data) - end - - @doc """ - Fetches the token transfers count from the cache or database. - """ - @spec fetch_count_from_cache(Hash.Address.t()) :: integer() - def fetch_count_from_cache(address_hash) do - address_hash_string = to_string(address_hash) - key = "hash_#{address_hash_string}" - - Helper.fetch_from_ets_cache(@cache_name, key) || fetch_from_db_cache(address_hash) - end - - defp fetch_updated_at_from_cache(address_hash, cache_name) do - address_hash_string = to_string(address_hash) - key = "hash_#{address_hash_string}_#{@ets_last_update_key}" - - Helper.fetch_from_ets_cache(cache_name, key) - end - - defp fetch_from_db_cache(address_hash) do - token = Token.get_by_contract_address_hash(address_hash, @api_true) - (token && token.transfer_count) || 0 - end - - defp put_into_db_cache(address_hash, count) do - Token.update_token_transfer_count(address_hash, count) - end - - defp enable_consolidation?, do: @enable_consolidation -end diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex index 00658705d91..ffae635ab3f 100644 --- a/apps/explorer/lib/explorer/chain/import.ex +++ b/apps/explorer/lib/explorer/chain/import.ex @@ -8,7 +8,15 @@ defmodule Explorer.Chain.Import do alias Explorer.Account.Notify alias Explorer.Chain.{Block, Import} alias Explorer.Chain.Cache.BlockNumber - alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} + + alias Explorer.Chain.Cache.Counters.{ + AddressCounters, + AddressCountersConsolidator, + Consolidation, + TokenCounters, + TokenCountersConsolidator + } + alias Explorer.Chain.Events.Publisher alias Explorer.Chain.Import.Stage alias Explorer.Repo @@ -157,30 +165,31 @@ defmodule Explorer.Chain.Import do {:ok, runner_to_changes_list} <- runner_to_changes_list(valid_runner_option_pairs), {:ok, data} <- insert_runner_to_changes_list(runner_to_changes_list, options) do Notify.async(data[:transactions]) - update_address_counters(data, Map.get(options, :broadcast, false)) + update_counters(data, Map.get(options, :broadcast, false)) Publisher.broadcast(Map.drop(data, @not_broadcasted_runners), Map.get(options, :broadcast, false)) {:ok, data} end end # Registers imported transactions and token transfers in the incremental - # address counters, whatever the import source (block fetcher, on-demand and - # async fetchers, migrators): marks the involved addresses dirty for - # `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` and live-bumps - # the display cache for realtime imports. Token transfers inserted below an - # address's consolidation watermark (derived asynchronously — e.g. from - # internal transactions — or backfilled by migrators) cannot be covered by - # incremental range aggregates anymore, so such addresses get their - # watermark reset for a full recalculation. + # address and token counters, whatever the import source (block fetcher, + # on-demand and async fetchers, migrators): marks the involved addresses and + # tokens dirty for their consolidators and live-bumps the display caches for + # realtime imports. Token transfers inserted below a consolidation watermark + # (derived asynchronously — e.g. from internal transactions — or backfilled + # by migrators) cannot be covered by incremental range aggregates anymore, + # so the affected addresses and tokens get their watermark reset for a full + # recalculation. # # Runs after the import transaction committed, so failures here (e.g. a # transient DB error in the watermark reset) must not fail the import or # prevent the subsequent event broadcast — they only cost counter freshness. - defp update_address_counters(data, broadcast_type) do + defp update_counters(data, broadcast_type) do transactions = Map.get(data, :transactions, []) token_transfers = Map.get(data, :token_transfers, []) AddressCounters.handle_new_data(transactions, token_transfers, broadcast_type == :realtime) + TokenCounters.handle_new_data(token_transfers, broadcast_type == :realtime) if token_transfers != [] and Explorer.mode() in [:indexer, :all] do reset_watermarks_covering_token_transfers(token_transfers) @@ -190,13 +199,30 @@ defmodule Explorer.Chain.Import do rescue error -> Logger.error(fn -> - ["Could not update address counters for imported data: ", Exception.format(:error, error, __STACKTRACE__)] + ["Could not update counters for imported data: ", Exception.format(:error, error, __STACKTRACE__)] end) :ok end defp reset_watermarks_covering_token_transfers(token_transfers) do + # a covered watermark is at most the lagged head, so transfers above it can + # never land below any watermark — pure-realtime imports skip the DB round + # trips entirely + lagged_head = BlockNumber.get_max() - Consolidation.safe_block_lag() + + covered_candidates = + Enum.filter(token_transfers, &(not is_nil(&1.block_number) and &1.block_number <= lagged_head)) + + if covered_candidates != [] do + reset_address_watermarks(covered_candidates) + reset_token_watermarks(covered_candidates) + end + + :ok + end + + defp reset_address_watermarks(token_transfers) do reset_bytes = token_transfers |> Enum.flat_map(&token_transfer_participants/1) @@ -206,7 +232,23 @@ defmodule Explorer.Chain.Import do |> AddressCountersConsolidator.reset_covered_watermarks() AddressCounters.invalidate(reset_bytes) - AddressCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + end + + defp reset_token_watermarks(token_transfers) do + reset_bytes = + token_transfers + |> Enum.reduce(%{}, fn token_transfer, acc -> + case token_transfer.token_contract_address_hash do + nil -> + acc + + token_hash -> + Map.update(acc, token_hash.bytes, token_transfer.block_number, &min(&1, token_transfer.block_number)) + end + end) + |> TokenCountersConsolidator.reset_covered_watermarks() + + TokenCounters.invalidate(reset_bytes) end defp token_transfer_participants(token_transfer) do diff --git a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex index e2d69283ed4..f4737da3229 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/address/current_token_balances.ex @@ -22,68 +22,52 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do @type imported :: [CurrentTokenBalance.t()] - @spec to_holder_address_hash_set_by_token_contract_address_hash([CurrentTokenBalance.t()]) :: %{ - token_contract_address_hash => MapSet.t(holder_address_hash) + @typedoc """ + A current token balance row shape carrying enough fields to decide whether + it makes its address a holder of its token: `value > 0`, or any row of an + `ERC-7984` token (confidential balances), except for the burn address — + matching `Explorer.Chain.Address.CurrentTokenBalance.token_holders_query_for_count/1`. + """ + @type holder_row :: %{ + required(:address_hash) => Hash.Address.t(), + required(:token_contract_address_hash) => Hash.Address.t(), + required(:token_id) => Decimal.t() | nil, + required(:token_type) => String.t() | nil, + required(:value) => Decimal.t() | nil, + optional(atom()) => any() } - when token_contract_address_hash: Hash.Address.t(), holder_address_hash: Hash.Address.t() - def to_holder_address_hash_set_by_token_contract_address_hash(address_current_token_balances) - when is_list(address_current_token_balances) do - address_current_token_balances - |> Stream.filter(fn %{value: value} -> valid_holder?(value) end) - |> Enum.reduce(%{}, fn %{token_contract_address_hash: token_contract_address_hash, address_hash: address_hash}, - acc_holder_address_hash_set_by_token_contract_address_hash -> - updated_holder_address_hash_set = - acc_holder_address_hash_set_by_token_contract_address_hash - |> Map.get_lazy(token_contract_address_hash, &MapSet.new/0) - |> MapSet.put(address_hash) - - Map.put( - acc_holder_address_hash_set_by_token_contract_address_hash, - token_contract_address_hash, - updated_holder_address_hash_set - ) - end) - end - @spec token_holder_count_deltas(%{deleted: [current_token_balance], inserted: [current_token_balance]}) :: [ + @doc """ + Computes the holder-count deltas produced by a reorg: `deleted` are the + current token balance rows removed with the non-consensus blocks, `inserted` + the rows re-derived from the historical token balances. A `{token, address}` + pair contributes ±1 only when its holder-ness actually flipped, taking the + pair's untouched rows (other `token_id`s still in the DB) into account. + """ + @spec token_holder_count_deltas(Repo.t(), %{deleted: [holder_row()], inserted: [holder_row()]}, timeout()) :: [ Tokens.token_holder_count_delta() ] - when current_token_balance: %{ - address_hash: Hash.Address.t(), - token_contract_address_hash: Hash.Address.t(), - value: Decimal.t() - } - def token_holder_count_deltas(%{deleted: deleted, inserted: inserted}) when is_list(deleted) and is_list(inserted) do - deleted_holder_address_hash_set_by_token_contract_address_hash = - to_holder_address_hash_set_by_token_contract_address_hash(deleted) - - inserted_holder_address_hash_set_by_token_contract_address_hash = - to_holder_address_hash_set_by_token_contract_address_hash(inserted) - - ordered_token_contract_address_hashes = - ordered_token_contract_address_hashes([ - deleted_holder_address_hash_set_by_token_contract_address_hash, - inserted_holder_address_hash_set_by_token_contract_address_hash - ]) - - Enum.flat_map(ordered_token_contract_address_hashes, fn token_contract_address_hash -> - holder_count_delta = - holder_count_delta(%{ - deleted_holder_address_hash_set_by_token_contract_address_hash: - deleted_holder_address_hash_set_by_token_contract_address_hash, - inserted_holder_address_hash_set_by_token_contract_address_hash: - inserted_holder_address_hash_set_by_token_contract_address_hash, - token_contract_address_hash: token_contract_address_hash - }) - - case holder_count_delta do - 0 -> - [] - - _ -> - [%{contract_address_hash: token_contract_address_hash, delta: holder_count_delta}] - end - end) + def token_holder_count_deltas(repo, %{deleted: deleted, inserted: inserted}, timeout \\ @timeout) + when is_list(deleted) and is_list(inserted) do + deleted = Enum.reject(deleted, &burn_address?(&1.address_hash)) + inserted = Enum.reject(inserted, &burn_address?(&1.address_hash)) + + before_by_pair = rows_qualification_by_pair(deleted) + after_by_pair = rows_qualification_by_pair(inserted) + + pair_states = + (Map.keys(before_by_pair) ++ Map.keys(after_by_pair)) + |> Enum.uniq() + |> Map.new(fn pair -> + {pair, %{before?: Map.get(before_by_pair, pair, false), after?: Map.get(after_by_pair, pair, false)}} + end) + + # the re-inserted rows are the only reorg-touched rows still present in + # the DB (deleted-but-not-reinserted rows are gone), so excluding the + # inserted triples leaves exactly the pair rows untouched by the reorg + batch_triples = MapSet.new(inserted, &row_triple/1) + + pair_transition_deltas(repo, pair_states, batch_triples, timeout) end @impl Import.Runner @@ -126,7 +110,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do :filter_params ) end) - |> Multi.run(:address_current_token_balances, fn repo, %{filter_params: filtered_changes_list} -> + |> Multi.run(:address_current_token_balances, fn repo, %{filter_params: {filtered_changes_list, _existing_keys}} -> Instrumenter.block_import_stage_runner( fn -> insert(repo, filtered_changes_list, insert_options) end, :block_following, @@ -137,11 +121,12 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do |> Multi.run(:address_current_token_balances_update_token_holder_counts, fn repo, %{ address_current_token_balances: - upserted_balances + upserted_balances, + filter_params: {_, existing_keys} } -> Instrumenter.block_import_stage_runner( fn -> - token_holder_count_deltas = upserted_balances_to_holder_count_deltas(upserted_balances) + token_holder_count_deltas = upserted_balances_to_holder_count_deltas(repo, upserted_balances, existing_keys) # ShareLocks order already enforced by `acquire_contract_address_tokens` (see docs: sharelocks.md) Tokens.update_holder_counts_with_deltas( @@ -164,18 +149,78 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do not is_nil(value) and Decimal.compare(value, 0) == :gt end - # Assumes existence of old_value field with previous value or nil - defp upserted_balances_to_holder_count_deltas(upserted_balances) do - upserted_balances - |> Enum.map(fn %{token_contract_address_hash: contract_address_hash, value: value, old_value: old_value} -> - delta = - cond do - not valid_holder?(old_value) and valid_holder?(value) -> 1 - valid_holder?(old_value) and not valid_holder?(value) -> -1 - true -> 0 - end + # matches the semantics of `CurrentTokenBalance.token_holders_query_for_count/1`: + # any row of an ERC-7984 token counts (confidential balances), otherwise value > 0 + defp row_qualifies?(%{token_type: "ERC-7984"}), do: true + defp row_qualifies?(%{value: value}), do: valid_holder?(value) + + # whether the row made its address a holder BEFORE this upsert: freshly + # inserted rows did not exist; updated ERC-7984 rows existed (any row + # counts); otherwise the previous value decides (`old_value` is set to the + # pre-upsert value by the on_conflict clause) + defp old_row_qualifies?(row, existing_keys) do + if MapSet.member?(existing_keys, row_triple(row)) do + case row.token_type do + "ERC-7984" -> true + _ -> valid_holder?(row.old_value) + end + else + false + end + end + + defp rows_qualification_by_pair(rows) do + rows + |> Enum.group_by(&{&1.token_contract_address_hash, &1.address_hash}) + |> Map.new(fn {pair, pair_rows} -> {pair, Enum.any?(pair_rows, &row_qualifies?/1)} end) + end + + defp burn_address?(%Hash{bytes: bytes}), do: bytes == <<0::160>> + + defp row_triple(row) do + {row.address_hash.bytes, row.token_contract_address_hash.bytes, normalize_token_id(row.token_id)} + end + + defp normalize_token_id(nil), do: nil + defp normalize_token_id(%Decimal{} = token_id), do: Decimal.to_integer(token_id) + defp normalize_token_id(token_id) when is_integer(token_id), do: token_id + + # Computes the holder-count deltas of the freshly upserted current token + # balance rows: a {token, address} pair contributes ±1 only when its + # holder-ness actually flipped with this batch, taking the pair's other rows + # (untouched token_ids of ERC-1155/404 balances) into account. + defp upserted_balances_to_holder_count_deltas(repo, upserted_balances, existing_keys) do + pair_states = + upserted_balances + |> Enum.reject(&burn_address?(&1.address_hash)) + |> Enum.group_by(&{&1.token_contract_address_hash, &1.address_hash}) + |> Map.new(fn {pair, rows} -> + {pair, + %{ + before?: Enum.any?(rows, &old_row_qualifies?(&1, existing_keys)), + after?: Enum.any?(rows, &row_qualifies?/1) + }} + end) - %{contract_address_hash: contract_address_hash, delta: delta} + batch_triples = MapSet.new(upserted_balances, &row_triple/1) + + pair_transition_deltas(repo, pair_states, batch_triples, @timeout) + end + + # For every {token, address} pair whose holder-ness flipped within the + # batch, checks whether the pair has other qualifying rows outside the batch + # (then its holder-ness did not actually change) and folds the surviving + # flips into per-token deltas. + defp pair_transition_deltas(repo, pair_states, batch_triples, timeout) do + flipped = for {pair, %{before?: before?, after?: after?}} <- pair_states, before? != after?, do: {pair, after?} + + pairs_with_other_rows = + pairs_with_other_qualifying_rows(repo, Enum.map(flipped, fn {pair, _after?} -> pair end), batch_triples, timeout) + + flipped + |> Enum.reject(fn {pair, _after?} -> MapSet.member?(pairs_with_other_rows, pair) end) + |> Enum.map(fn {{token_contract_address_hash, _address_hash}, after?} -> + %{contract_address_hash: token_contract_address_hash, delta: if(after?, do: 1, else: -1)} end) |> Enum.group_by(& &1.contract_address_hash, & &1.delta) |> Enum.map(fn {contract_address_hash, deltas} -> @@ -185,34 +230,32 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do |> Enum.sort_by(& &1.contract_address_hash) end - defp holder_count_delta(%{ - deleted_holder_address_hash_set_by_token_contract_address_hash: - deleted_holder_address_hash_set_by_token_contract_address_hash, - inserted_holder_address_hash_set_by_token_contract_address_hash: - inserted_holder_address_hash_set_by_token_contract_address_hash, - token_contract_address_hash: token_contract_address_hash - }) do - case {deleted_holder_address_hash_set_by_token_contract_address_hash[token_contract_address_hash], - inserted_holder_address_hash_set_by_token_contract_address_hash[token_contract_address_hash]} do - {deleted_holder_address_hash_set, nil} -> - -1 * Enum.count(deleted_holder_address_hash_set) - - {nil, inserted_holder_address_hash_set} -> - Enum.count(inserted_holder_address_hash_set) - - {deleted_holder_address_hash_set, inserted_holder_address_hash_set} -> - inserted_holder_address_hash_count = - inserted_holder_address_hash_set - |> MapSet.difference(deleted_holder_address_hash_set) - |> Enum.count() - - deleted_holder_address_hash_count = - deleted_holder_address_hash_set - |> MapSet.difference(inserted_holder_address_hash_set) - |> Enum.count() - - inserted_holder_address_hash_count - deleted_holder_address_hash_count - end + defp pairs_with_other_qualifying_rows(_repo, [], _batch_triples, _timeout), do: MapSet.new() + + defp pairs_with_other_qualifying_rows(repo, pairs, batch_triples, timeout) do + ids = + Enum.map(pairs, fn {token_contract_address_hash, address_hash} -> + {address_hash.bytes, token_contract_address_hash.bytes} + end) + + query = + from(ctb in CurrentTokenBalance, + where: ^QueryHelper.tuple_in([:address_hash, :token_contract_address_hash], ids), + where: ctb.value > 0 or ctb.token_type == "ERC-7984", + select: {ctb.token_contract_address_hash, ctb.address_hash, ctb.token_id} + ) + + query + |> repo.all(timeout: timeout) + |> Enum.reduce(MapSet.new(), fn {token_contract_address_hash, address_hash, token_id}, acc -> + triple = {address_hash.bytes, token_contract_address_hash.bytes, normalize_token_id(token_id)} + + if MapSet.member?(batch_triples, triple) do + acc + else + MapSet.put(acc, {token_contract_address_hash, address_hash}) + end + end) end defp filter_params(repo, changes_list) do @@ -229,15 +272,25 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do %{block_number: ctb.block_number, value: ctb.value, value_fetched_at: ctb.value_fetched_at}} end) + existing_keys = + existing_ctb_without_token_id + |> Enum.concat(existing_ctb_with_token_id) + |> MapSet.new(fn ctb -> + {hash_bytes(ctb.address_hash), hash_bytes(ctb.token_contract_address_hash), normalize_token_id(ctb.token_id)} + end) + filtered_ctbs = Enum.filter(changes_list, fn ctb -> existing_ctb = existing_ctb_map[{ctb[:address_hash], ctb[:token_contract_address_hash], ctb[:token_id]}] should_update?(Map.put_new(ctb, :value_fetched_at, nil), existing_ctb) end) - {:ok, filtered_ctbs} + {:ok, {filtered_ctbs, existing_keys}} end + defp hash_bytes(%Hash{bytes: bytes}), do: bytes + defp hash_bytes(bytes) when is_binary(bytes), do: bytes + defp select_existing_current_token_balances(_repo, [], _with_token_id?), do: [] defp select_existing_current_token_balances(repo, params, false) do @@ -396,16 +449,4 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalances do fragment("? < EXCLUDED.value_fetched_at", current_token_balance.value_fetched_at))) ) end - - defp ordered_token_contract_address_hashes(holder_address_hash_set_by_token_contract_address_hash_list) - when is_list(holder_address_hash_set_by_token_contract_address_hash_list) do - holder_address_hash_set_by_token_contract_address_hash_list - |> Enum.reduce(MapSet.new(), fn holder_address_hash_set_by_token_contract_address_hash, acc -> - holder_address_hash_set_by_token_contract_address_hash - |> Map.keys() - |> MapSet.new() - |> MapSet.union(acc) - end) - |> Enum.sort() - end end diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index 375f88feef3..b91b23ae20e 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -30,7 +30,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Explorer.Chain.Block.Reward alias Explorer.Chain.Cache.BlockNumber - alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator, TokenCounters} + alias Explorer.Chain.Cache.Counters.TokenCountersConsolidator alias Explorer.Chain.Import.Runner alias Explorer.Chain.Import.Runner.Address.CurrentTokenBalances alias Explorer.Chain.Import.Runner.{Addresses, TokenInstances, Tokens} @@ -38,7 +39,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do alias Explorer.Chain.Zilliqa.Zrc2.TokenTransfer, as: Zrc2TokenTransfer alias Explorer.Prometheus.Instrumenter alias Explorer.Repo, as: ExplorerRepo - alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} + alias Explorer.Utility.{CountersRefetchBlock, MissingBlockRange} alias Explorer.Chain.Celo.AggregatedElectionReward, as: CeloAggregatedElectionReward alias Explorer.Chain.Celo.ElectionReward, as: CeloElectionReward @@ -96,15 +97,15 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :lose_consensus ) end) - |> Multi.run(:address_counters_refetched_block_numbers, fn repo, _ -> + |> Multi.run(:counters_refetched_block_numbers, fn repo, _ -> Instrumenter.block_import_stage_runner( fn -> # Note, needs to be executed before `blocks` which resets `refetch_needed` - address_counters_refetched_block_numbers(repo, changes_list, insert_options) + counters_refetched_block_numbers(repo, changes_list, insert_options) end, :address_referencing, :blocks, - :address_counters_refetched_block_numbers + :counters_refetched_block_numbers ) end) |> Multi.run(:blocks, fn repo, _ -> @@ -171,16 +172,16 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :derive_transaction_forks ) end) - |> Multi.run(:address_counters_corrections, fn repo, - %{ - address_counters_refetched_block_numbers: refetched_block_numbers, - fork_transactions: forked_transactions - } -> + |> Multi.run(:counters_corrections, fn repo, + %{ + counters_refetched_block_numbers: refetched_block_numbers, + fork_transactions: forked_transactions + } -> Instrumenter.block_import_stage_runner( - fn -> address_counters_corrections(repo, refetched_block_numbers, forked_transactions, insert_options) end, + fn -> counters_corrections(repo, refetched_block_numbers, forked_transactions, insert_options) end, :address_referencing, :blocks, - :address_counters_corrections + :counters_corrections ) end) |> Multi.run(:delete_address_token_balances, fn repo, %{lose_consensus: non_consensus_blocks} -> @@ -275,7 +276,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do } -> Instrumenter.block_import_stage_runner( fn -> - deltas = CurrentTokenBalances.token_holder_count_deltas(%{deleted: deleted, inserted: inserted}) + deltas = CurrentTokenBalances.token_holder_count_deltas(repo, %{deleted: deleted, inserted: inserted}) Tokens.update_holder_counts_with_deltas(repo, deltas, insert_options) end, :address_referencing, @@ -306,9 +307,9 @@ defmodule Explorer.Chain.Import.Runner.Blocks do # Detects which of the incoming blocks are re-imports of blocks queued for a # re-fetch: their stored row still has `refetch_needed` set (the upcoming # `blocks` upsert resets it). Blocks already present in the - # `address_counters_refetch_blocks` queue are skipped — their current content + # `counters_refetch_blocks` queue are skipped — their current content # was never added to the counters, so there is nothing to subtract. - defp address_counters_refetched_block_numbers(repo, blocks_changes, %{timeout: timeout}) do + defp counters_refetched_block_numbers(repo, blocks_changes, %{timeout: timeout}) do hashes = blocks_changes |> Enum.map(& &1.hash) |> Enum.uniq() refetched_numbers = @@ -327,7 +328,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do _ -> pending_numbers = repo.all( - from(refetch_block in AddressCountersRefetchBlock, + from(refetch_block in CountersRefetchBlock, where: refetch_block.block_number in ^refetched_numbers, select: refetch_block.block_number ), @@ -341,30 +342,35 @@ defmodule Explorer.Chain.Import.Runner.Blocks do {:error, %{exception: postgrex_error}} end - # Corrects the incremental address counters for content changing under - # already-consolidated ranges: + # Corrects the incremental address and token counters for content changing + # under already-consolidated ranges: # # * deep reorgs — participants of forked transactions and of their token # transfers whose `counters_updated_at` watermark covers the forked block # get their watermark reset for a full recalculation (reorgs within the - # consolidation safety lag never satisfy the condition); + # consolidation safety lag never satisfy the condition). Token watermarks + # need no fork handling: token transfer rows of forked blocks stay + # physically present (only `block_consensus` flips) and the token counter + # counts physical rows; # * block re-fetches — the old content of the re-imported blocks is - # subtracted from the already-covered counters while it is still in the - # DB, and the block numbers are queued in `address_counters_refetch_blocks` - # so that `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` adds - # the re-imported content back once the re-fetch completes. Content of - # forked transactions is excluded (`block_number` already nulled) — those - # addresses are covered by the watermark reset. - defp address_counters_corrections(repo, refetched_block_numbers, forked_transactions, %{ + # subtracted from the already-covered address and token counters while it + # is still in the DB, and the block numbers are queued in + # `counters_refetch_blocks` so that + # `Explorer.Chain.Cache.Counters.Consolidation` adds the re-imported + # content back once the re-fetch completes. Content of forked transactions + # is excluded (`block_number` already nulled) — those addresses are + # covered by the watermark reset. + defp counters_corrections(repo, refetched_block_numbers, forked_transactions, %{ timeout: timeout, timestamps: timestamps }) do reset_bytes = reset_watermarks_for_forked(repo, forked_transactions, timeout) - subtracted_bytes = subtract_refetched_blocks_content(repo, refetched_block_numbers, timeout, timestamps) + {subtracted_address_bytes, subtracted_token_bytes} = + subtract_refetched_blocks_content(repo, refetched_block_numbers, timeout, timestamps) - AddressCounters.invalidate(reset_bytes ++ subtracted_bytes) - AddressCounters.mark_dirty(Enum.map(reset_bytes, &{&1, BlockNumber.get_max()})) + AddressCounters.invalidate(reset_bytes ++ subtracted_address_bytes) + TokenCounters.invalidate(subtracted_token_bytes) {:ok, %{reset: reset_bytes, refetched_block_numbers: refetched_block_numbers}} rescue @@ -398,7 +404,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do |> AddressCountersConsolidator.reset_covered_watermarks(repo) end - defp subtract_refetched_blocks_content(_repo, [], _timeout, _timestamps), do: [] + defp subtract_refetched_blocks_content(_repo, [], _timeout, _timestamps), do: {[], []} defp subtract_refetched_blocks_content(repo, block_numbers, timeout, timestamps) do transactions = @@ -414,16 +420,20 @@ defmodule Explorer.Chain.Import.Runner.Blocks do repo.all( from(token_transfer in TokenTransfer, where: token_transfer.block_number in ^block_numbers, - select: struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash]) + select: + struct(token_transfer, [:block_number, :from_address_hash, :to_address_hash, :token_contract_address_hash]) ), timeout: timeout ) - updated_bytes = AddressCountersConsolidator.apply_covered_deltas(transactions, token_transfers, :negative, repo) + updated_address_bytes = + AddressCountersConsolidator.apply_covered_deltas(transactions, token_transfers, :negative, repo) + + updated_token_bytes = TokenCountersConsolidator.apply_covered_transfer_deltas(token_transfers, :negative, repo) - # Enforce AddressCountersRefetchBlock ShareLocks order (see docs: sharelocks.md) + # Enforce CountersRefetchBlock ShareLocks order (see docs: sharelocks.md) repo.insert_all( - AddressCountersRefetchBlock, + CountersRefetchBlock, block_numbers |> Enum.sort() |> Enum.map(&%{block_number: &1, inserted_at: timestamps.inserted_at, updated_at: timestamps.updated_at}), @@ -431,7 +441,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do timeout: timeout ) - updated_bytes + {updated_address_bytes, updated_token_bytes} end defp fork_transactions(%{ @@ -984,6 +994,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :address_hash, :token_contract_address_hash, :token_id, + :token_type, :value ]), where: ctb.block_number in ^non_consensus_block_numbers @@ -1017,6 +1028,7 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :address_hash, :token_contract_address_hash, :token_id, + :token_type, # Used to determine if `address_hash` was a holder of `token_contract_address_hash` before # `address_current_token_balance` is deleted in `update_tokens_holder_count`. @@ -1084,7 +1096,10 @@ defmodule Explorer.Chain.Import.Runner.Blocks do ) derived_address_current_token_balances = - Enum.map(result, &Map.take(&1, [:address_hash, :token_contract_address_hash, :token_id, :block_number, :value])) + Enum.map( + result, + &Map.take(&1, [:address_hash, :token_contract_address_hash, :token_id, :token_type, :block_number, :value]) + ) {:ok, derived_address_current_token_balances} end diff --git a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex index ef77f04be35..ccec1b64b04 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/tokens.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/tokens.ex @@ -59,7 +59,7 @@ defmodule Explorer.Chain.Import.Runner.Tokens do where: not is_nil(token.holder_count), update: [ set: [ - holder_count: token.holder_count + deltas.delta, + holder_count: fragment("LEAST(GREATEST(? + ?, 0), 2147483647)", token.holder_count, deltas.delta), updated_at: ^updated_at ] ], @@ -199,8 +199,8 @@ defmodule Explorer.Chain.Import.Runner.Tokens do cataloged: fragment("COALESCE(EXCLUDED.cataloged, ?)", token.cataloged), bridged: fragment("COALESCE(EXCLUDED.bridged, ?)", token.bridged), skip_metadata: fragment("COALESCE(EXCLUDED.skip_metadata, ?)", token.skip_metadata), - # `holder_count` and `transfer_count` are not updated as a pre-existing token means these counts are already initialized OR - # need to be migrated with `priv/repo/migrations/scripts/update_new_tokens_holder_count_in_batches.sql.exs` + # `holder_count`, `transfer_count` and `counters_updated_at` are not updated: pre-existing tokens are maintained by + # the incremental counters machinery (import-time holder deltas and the token counters consolidator) # Don't update `contract_address_hash` as it is the primary key and used for the conflict target inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token.updated_at) @@ -233,8 +233,8 @@ defmodule Explorer.Chain.Import.Runner.Tokens do type: fragment("COALESCE(EXCLUDED.type, ?)", token.type), cataloged: fragment("COALESCE(EXCLUDED.cataloged, ?)", token.cataloged), skip_metadata: fragment("COALESCE(EXCLUDED.skip_metadata, ?)", token.skip_metadata), - # `holder_count` is not updated as a pre-existing token means the `holder_count` is already initialized OR - # need to be migrated with `priv/repo/migrations/scripts/update_new_tokens_holder_count_in_batches.sql.exs` + # `holder_count`, `transfer_count` and `counters_updated_at` are not updated: pre-existing tokens are maintained by + # the incremental counters machinery (import-time holder deltas and the token counters consolidator) # Don't update `contract_address_hash` as it is the primary key and used for the conflict target inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", token.inserted_at), updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", token.updated_at) diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index bd283659ae6..d290a3b4a93 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -37,6 +37,7 @@ defmodule Explorer.Chain.Token.Schema do field(:volume_24h, FiatValue) field(:circulating_supply, :decimal) field(:transfer_count, :integer) + field(:counters_updated_at, :integer) belongs_to( :contract_address, @@ -95,15 +96,12 @@ defmodule Explorer.Chain.Token do alias Explorer.{Chain, SortingHelper} alias Explorer.Chain.{Address, BridgedToken, Hash, Search, Token} alias Explorer.Chain.Cache.{BackgroundMigrations, BlockNumber} - alias Explorer.Chain.Cache.Counters.{TokenHoldersCount, TokenTransfersCount} + alias Explorer.Chain.Cache.Counters.TokenCounters alias Explorer.Chain.Import.Runner alias Explorer.Helper, as: ExplorerHelper alias Explorer.Repo alias Explorer.SmartContract.Helper - # milliseconds - @timeout 60_000 - @default_sorting [ desc_nulls_last: :circulating_market_cap, desc_nulls_last: :fiat_value, @@ -233,6 +231,27 @@ defmodule Explorer.Chain.Token do |> Repo.stream_reduce(initial, reducer) end + @doc """ + Streams cataloged tokens with their current counter values (holders and + transfers counts) for the periodic counters export. Tokens whose counters + were never consolidated (`counters_updated_at IS NULL`) are skipped rather + than exported as zeros. + """ + @spec stream_cataloged_tokens_for_counters( + initial :: accumulator, + reducer :: (entry :: __MODULE__.t(), accumulator -> accumulator), + limited? :: boolean() + ) :: {:ok, accumulator} + when accumulator: term() + def stream_cataloged_tokens_for_counters(initial, reducer, limited? \\ false) when is_function(reducer, 2) do + __MODULE__ + |> where([token], token.cataloged == true and not is_nil(token.counters_updated_at)) + |> select([token], struct(token, [:contract_address_hash, :holder_count, :transfer_count])) + |> Chain.add_fetcher_limit(limited?) + |> order_by(asc: :contract_address_hash) + |> Repo.stream_reduce(initial, reducer) + end + @doc """ Update a new `t:Token.t/0` record. @@ -483,49 +502,6 @@ defmodule Explorer.Chain.Token do (last_address_hash && where(query, [token], token.contract_address_hash > ^last_address_hash)) || query end - @doc """ - Updates token_holder_count for a given contract_address_hash. - It used by Explorer.Chain.Cache.Counters.TokenHoldersCount module. - """ - @spec update_token_holder_count(Hash.Address.t(), integer()) :: {non_neg_integer(), nil} - def update_token_holder_count(contract_address_hash, holders_count) when not is_nil(holders_count) do - now = DateTime.utc_now() - - Repo.update_all( - from(t in __MODULE__, - where: t.contract_address_hash == ^contract_address_hash, - update: [set: [holder_count: ^holders_count, updated_at: ^now]] - ), - [], - timeout: @timeout - ) - end - - @doc """ - Updates `transfer_count` field for a given `contract_address_hash`. - Used by the `Explorer.Chain.Cache.Counters.TokenTransfersCount` module. - - ## Parameters - - `contract_address_hash`: The address of the token contract. - - `transfer_count`: The updated counter value. - - ## Returns - - `{updated_count, nil}` tuple where `updated_count` is the number of updated rows in the db table. - """ - @spec update_token_transfer_count(Hash.Address.t(), non_neg_integer()) :: {non_neg_integer(), nil} - def update_token_transfer_count(contract_address_hash, transfer_count) when not is_nil(transfer_count) do - now = DateTime.utc_now() - - Repo.update_all( - from(t in __MODULE__, - where: t.contract_address_hash == ^contract_address_hash, - update: [set: [transfer_count: ^transfer_count, updated_at: ^now]] - ), - [], - timeout: @timeout - ) - end - @doc """ Drops token info for the given token: Sets is_verified_via_admin_panel to false, icon_url to nil, symbol to nil, name to nil. @@ -589,75 +565,32 @@ defmodule Explorer.Chain.Token do end @doc """ - Fetches token counters (transfers count and holders count) for a given token address. + Fetches the token counters (transfers count and holders count) for a given + token. - This function spawns two async tasks to fetch the token transfers count and - token holders count concurrently. If a task times out or exits, it falls back - to fetching the cached value. + The transfers count is served from the incremental counter cache + (`Explorer.Chain.Cache.Counters.TokenCounters`) falling back to the + `transfer_count` column; the holders count comes straight from the + `holder_count` column, which is maintained live by the import-time deltas. + No aggregates run in the calling process — a token whose counters were never + calculated reports zeros and is marked for consolidation. ## Parameters - - `address_hash`: The contract address hash of the token - - `timeout`: The timeout in milliseconds for the async tasks + - `token`: The token to fetch the counters for ## Returns - - A tuple `{transfers_count, holders_count}` where each value is an integer or nil + - A tuple `{transfers_count, holders_count}` of non-negative integers """ - @spec fetch_token_counters(Hash.Address.t(), timeout()) :: {integer() | nil, integer() | nil} - def fetch_token_counters(address_hash, timeout) do - total_token_transfers_task = - Task.async(fn -> - TokenTransfersCount.fetch(address_hash) - end) - - total_token_holders_task = - Task.async(fn -> - TokenHoldersCount.fetch(address_hash) - end) + @spec fetch_token_counters(t()) :: {non_neg_integer(), non_neg_integer()} + def fetch_token_counters(%__MODULE__{} = token) do + transfers_count = TokenCounters.fetch(token) + holders_count = token.holder_count || 0 - [total_token_transfers_task, total_token_holders_task] - |> Task.yield_many(timeout) - |> Enum.map(fn {task, res} -> - case res do - {:ok, result} -> - result - - {:exit, reason} -> - Logger.warning("Query fetching token counters terminated: #{inspect(reason)}") - - fallback_cached_value_based_on_async_task_pid( - task.pid, - total_token_transfers_task.pid, - total_token_holders_task.pid, - address_hash - ) - - nil -> - Logger.warning("Query fetching token counters timed out.") - - fallback_cached_value_based_on_async_task_pid( - task.pid, - total_token_transfers_task.pid, - total_token_holders_task.pid, - address_hash - ) - end - end) - |> List.to_tuple() - end - - defp fallback_cached_value_based_on_async_task_pid( - task_pid, - total_token_transfers_task_pid, - total_token_holders_task_pid, - address_hash - ) do - case task_pid do - ^total_token_transfers_task_pid -> - TokenTransfersCount.fetch_count_from_cache(address_hash) - - ^total_token_holders_task_pid -> - TokenHoldersCount.fetch_count_from_cache(address_hash) + if is_nil(token.counters_updated_at) or is_nil(token.holder_count) do + TokenCounters.mark_dirty_at_max_block(token.contract_address_hash) end + + {transfers_count, holders_count} end @doc """ diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index d29f6f394f8..fa1d785f00d 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -411,6 +411,28 @@ defmodule Explorer.Chain.TokenTransfer do Repo.one(query, timeout: :infinity) end + @doc """ + Builds a query for the token transfers of the given token, optionally + bounded by a `(from_block_number, to_block_number]` block range (either + bound may be `nil` to leave that side open). Used by the incremental token + counters consolidation. + """ + @spec count_token_transfers_from_token_hash_query( + Hash.t(), + Explorer.Chain.Block.block_number() | nil, + Explorer.Chain.Block.block_number() | nil + ) :: Ecto.Query.t() + def count_token_transfers_from_token_hash_query(token_address_hash, from_block_number \\ nil, to_block_number \\ nil) do + TokenTransfer + |> where([tt], tt.token_contract_address_hash == ^token_address_hash) + |> then(fn query -> + if is_nil(from_block_number), do: query, else: where(query, [tt], tt.block_number > ^from_block_number) + end) + |> then(fn query -> + if is_nil(to_block_number), do: query, else: where(query, [tt], tt.block_number <= ^to_block_number) + end) + end + @spec count_token_transfers_from_token_hash_and_token_id(Hash.t(), non_neg_integer(), [api?]) :: non_neg_integer() def count_token_transfers_from_token_hash_and_token_id(token_address_hash, token_id, options) do query = diff --git a/apps/explorer/lib/explorer/migrator/backfill_token_counters.ex b/apps/explorer/lib/explorer/migrator/backfill_token_counters.ex new file mode 100644 index 00000000000..31fb738418e --- /dev/null +++ b/apps/explorer/lib/explorer/migrator/backfill_token_counters.ex @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.BackfillTokenCounters do + @moduledoc """ + Backfills the incremental token counters: recalculates + `tokens.transfer_count` (full count bounded by the consolidation safe block) + and `tokens.holder_count` (snapshot-relative full recount — normalizing the + drift accumulated by the legacy delta semantics) for every token that was + never consolidated (`counters_updated_at IS NULL`) and stamps its + consolidation watermark. + + The migration walks the `tokens` primary key with a keyset cursor stored in + the migration `meta`, so it resumes where it left off after a restart. + Tokens consolidated concurrently (via the dirty-marker path of + `Explorer.Chain.Cache.Counters.TokenCountersConsolidator`) are skipped by + the `counters_updated_at IS NULL` guard. + """ + + use Explorer.Migrator.FillingMigration + + import Ecto.Query + + require Logger + + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Chain.Cache.Counters.{Consolidation, TokenCountersConsolidator} + alias Explorer.Chain.Token + alias Explorer.Migrator.FillingMigration + alias Explorer.Repo + + @migration_name "backfill_token_counters" + + # extra cursor sweeps picking up tokens whose recalculation failed on a + # previous pass (they keep a NULL watermark but the cursor already passed + # them) + @max_retry_passes 3 + + @impl FillingMigration + def migration_name, do: @migration_name + + @impl FillingMigration + def last_unprocessed_identifiers(state) do + limit = batch_size() * concurrency() + + query = + from(token in Token, + where: is_nil(token.counters_updated_at), + order_by: [asc: token.contract_address_hash], + limit: ^limit, + select: token.contract_address_hash + ) + + hashes = + state + |> Map.get("max_processed_hash") + |> case do + nil -> query + cursor -> where(query, [token], token.contract_address_hash > ^cursor) + end + |> Repo.all(timeout: :infinity) + + case List.last(hashes) do + nil -> maybe_retry_pass(state) + last_hash -> {hashes, Map.put(state, "max_processed_hash", to_string(last_hash))} + end + end + + defp maybe_retry_pass(state) do + retry_passes = Map.get(state, "retry_passes", 0) + + if retry_passes < @max_retry_passes and Map.has_key?(state, "max_processed_hash") and + Repo.exists?(from(token in Token, where: is_nil(token.counters_updated_at))) do + last_unprocessed_identifiers(%{"retry_passes" => retry_passes + 1}) + else + {[], state} + end + end + + @impl FillingMigration + def unprocessed_data_query, do: nil + + @impl FillingMigration + def update_batch(contract_address_hashes) do + TokenCountersConsolidator.consolidate_tokens(contract_address_hashes, await_safe_block()) + end + + @impl FillingMigration + def update_cache do + BackgroundMigrations.set_backfill_token_counters_finished(true) + end + + # Consolidation is not possible while the chain head is unknown or blocks + # below it are still missing or pending a re-fetch counter correction. Wait + # instead of skipping: the cursor must not advance past unprocessed tokens. + defp await_safe_block do + case Consolidation.safe_block() do + nil -> + Logger.info("#{@migration_name} migration is waiting for a safe block to consolidate up to") + Process.sleep(:timer.seconds(30)) + await_safe_block() + + safe_block -> + safe_block + end + end +end diff --git a/apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex b/apps/explorer/lib/explorer/utility/counters_refetch_block.ex similarity index 55% rename from apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex rename to apps/explorer/lib/explorer/utility/counters_refetch_block.ex index 893cc10cbdc..45be2aebb96 100644 --- a/apps/explorer/lib/explorer/utility/address_counters_refetch_block.ex +++ b/apps/explorer/lib/explorer/utility/counters_refetch_block.ex @@ -1,17 +1,18 @@ # SPDX-License-Identifier: LicenseRef-Blockscout -defmodule Explorer.Utility.AddressCountersRefetchBlock do +defmodule Explorer.Utility.CountersRefetchBlock do @moduledoc """ Keeps block numbers whose old content was already subtracted from the - incremental address counters (`addresses.transactions_count`, - `addresses.token_transfers_count`, `addresses.gas_used`) because the blocks - were queued for re-fetch (see `Explorer.Chain.Block.full_refetch/1`). - - A row means "the negative counters delta for this block was applied, the - positive delta from its re-imported content is still pending". Rows are - consumed by `Explorer.Chain.Cache.Counters.AddressCountersConsolidator` once - the corresponding block no longer requires a re-fetch. While a row exists, - the consolidator does not advance any address consolidation watermark past - this block number. + incremental counters (the address counters `addresses.transactions_count`, + `addresses.token_transfers_count`, `addresses.gas_used` and the token + counter `tokens.transfer_count`) because the blocks were queued for re-fetch + (see `Explorer.Chain.Block.full_refetch/1`). + + A row means "the negative counters deltas for this block were applied, the + positive deltas from its re-imported content are still pending". Rows are + consumed by `Explorer.Chain.Cache.Counters.Consolidation.settle_pending_refetch_blocks/0` + once the corresponding block no longer requires a re-fetch. While a row + exists, no consolidation watermark (address or token) advances past this + block number. """ use Explorer.Schema @@ -19,7 +20,7 @@ defmodule Explorer.Utility.AddressCountersRefetchBlock do alias Explorer.Repo @primary_key false - typed_schema "address_counters_refetch_blocks" do + typed_schema "counters_refetch_blocks" do field(:block_number, :integer, primary_key: true) timestamps() diff --git a/apps/explorer/priv/repo/migrations/20260831000002_add_counters_updated_at_to_tokens.exs b/apps/explorer/priv/repo/migrations/20260831000002_add_counters_updated_at_to_tokens.exs new file mode 100644 index 00000000000..00118c6385f --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20260831000002_add_counters_updated_at_to_tokens.exs @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Repo.Migrations.AddCountersUpdatedAtToTokens do + use Ecto.Migration + + def change do + alter table(:tokens) do + add(:counters_updated_at, :bigint, null: true) + end + end +end diff --git a/apps/explorer/priv/repo/migrations/20260831000003_rename_address_counters_refetch_blocks.exs b/apps/explorer/priv/repo/migrations/20260831000003_rename_address_counters_refetch_blocks.exs new file mode 100644 index 00000000000..f065687fc03 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20260831000003_rename_address_counters_refetch_blocks.exs @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Repo.Migrations.RenameAddressCountersRefetchBlocks do + use Ecto.Migration + + # the block re-fetch corrections queue is shared by the address AND token + # counters consolidation, so the address-scoped name is dropped + def change do + rename(table(:address_counters_refetch_blocks), to: table(:counters_refetch_blocks)) + + execute( + "ALTER INDEX address_counters_refetch_blocks_pkey RENAME TO counters_refetch_blocks_pkey", + "ALTER INDEX counters_refetch_blocks_pkey RENAME TO address_counters_refetch_blocks_pkey" + ) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs b/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs index 389e8c35549..acbf1d9cc1a 100644 --- a/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs +++ b/apps/explorer/test/explorer/chain/cache/counters/address_counters_consolidator_test.exs @@ -3,9 +3,9 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do use Explorer.DataCase, async: false alias Explorer.Chain.Address - alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator} + alias Explorer.Chain.Cache.Counters.{AddressCounters, AddressCountersConsolidator, Consolidation} alias Explorer.Repo - alias Explorer.Utility.AddressCountersRefetchBlock + alias Explorer.Utility.CountersRefetchBlock describe "consolidate_addresses/2" do test "incrementally consolidates an address with a watermark" do @@ -95,15 +95,15 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do describe "safe_block/0" do setup do - initial_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + initial_env = Application.get_env(:explorer, Consolidation) || [] Application.put_env( :explorer, - AddressCountersConsolidator, + Consolidation, Keyword.merge(initial_env, safe_block_lag: 0) ) - on_exit(fn -> Application.put_env(:explorer, AddressCountersConsolidator, initial_env) end) + on_exit(fn -> Application.put_env(:explorer, Consolidation, initial_env) end) :ok end @@ -132,7 +132,7 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do test "is capped below the lowest block pending a re-fetch counter correction" do insert(:block, number: 0) insert(:block, number: 100) - Repo.insert!(%AddressCountersRefetchBlock{block_number: 30}) + Repo.insert!(%CountersRefetchBlock{block_number: 30}) assert AddressCountersConsolidator.safe_block() == 29 end @@ -158,15 +158,15 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do describe "consolidate/0" do setup do - initial_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + initial_env = Application.get_env(:explorer, Consolidation) || [] Application.put_env( :explorer, - AddressCountersConsolidator, + Consolidation, Keyword.merge(initial_env, safe_block_lag: 0) ) - on_exit(fn -> Application.put_env(:explorer, AddressCountersConsolidator, initial_env) end) + on_exit(fn -> Application.put_env(:explorer, Consolidation, initial_env) end) start_supervised!(AddressCounters) @@ -214,7 +214,7 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do transaction = :transaction |> insert(from_address: address) |> with_block(reimported_block) insert(:block, number: 100) - Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + Repo.insert!(%CountersRefetchBlock{block_number: 50}) AddressCountersConsolidator.consolidate() @@ -224,30 +224,55 @@ defmodule Explorer.Chain.Cache.Counters.AddressCountersConsolidatorTest do assert reloaded.gas_used == Decimal.to_integer(transaction.gas_used) # settling a covered block does not move the watermark assert reloaded.counters_updated_at == 100 - assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 0 + assert Repo.aggregate(CountersRefetchBlock, :count) == 0 end test "does not settle blocks still awaiting their re-fetch" do insert(:block, number: 50, refetch_needed: true) insert(:block, number: 100) - Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + Repo.insert!(%CountersRefetchBlock{block_number: 50}) AddressCountersConsolidator.consolidate() - assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + assert Repo.aggregate(CountersRefetchBlock, :count) == 1 end test "does not settle blocks still covered by a missing range" do # refetch_needed already flipped, but the re-import has not fully - # completed yet (the missing range row is only removed afterwards) + # completed yet (the missing range row is only removed afterwards); + # missing ranges are stored descending: from_number is the high end insert(:block, number: 50, refetch_needed: false) insert(:block, number: 100) - insert(:missing_block_range, from_number: 50, to_number: 50) - Repo.insert!(%AddressCountersRefetchBlock{block_number: 50}) + insert(:missing_block_range, from_number: 60, to_number: 40) + Repo.insert!(%CountersRefetchBlock{block_number: 50}) AddressCountersConsolidator.consolidate() - assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + assert Repo.aggregate(CountersRefetchBlock, :count) == 1 + end + end + + describe "reset_covered_watermarks/2" do + test "resets only addresses whose watermark covers the block and marks them dirty" do + start_supervised!(AddressCounters) + + covered_address = insert(:address, counters_updated_at: 100) + fresh_address = insert(:address, counters_updated_at: 30) + + reset_bytes = + AddressCountersConsolidator.reset_covered_watermarks(%{ + covered_address.hash.bytes => 50, + fresh_address.hash.bytes => 50 + }) + + assert reset_bytes == [covered_address.hash.bytes] + assert is_nil(Repo.get(Address, covered_address.hash).counters_updated_at) + assert Repo.get(Address, fresh_address.hash).counters_updated_at == 30 + + # the recalculation of the reset address is scheduled + :sys.get_state(AddressCounters) + assert {[{marked_bytes, _block_number}], _continuation} = AddressCounters.select_dirty(10) + assert marked_bytes == covered_address.hash.bytes end end diff --git a/apps/explorer/test/explorer/chain/cache/counters/token_counters_consolidator_test.exs b/apps/explorer/test/explorer/chain/cache/counters/token_counters_consolidator_test.exs new file mode 100644 index 00000000000..f80ea289eb5 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/counters/token_counters_consolidator_test.exs @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.TokenCountersConsolidatorTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.Counters.{Consolidation, TokenCounters, TokenCountersConsolidator} + alias Explorer.Chain.Token + alias Explorer.Repo + alias Explorer.Utility.CountersRefetchBlock + + describe "consolidate_tokens/2" do + test "incrementally consolidates a token with a watermark" do + token = insert(:token, transfer_count: 7, holder_count: 3, counters_updated_at: 100) + + in_range_block = insert(:block, number: 150) + above_range_block = insert(:block, number: 300) + + insert_transfer(token, in_range_block) + insert_transfer(token, above_range_block) + + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], 200) + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 8 + # holders are delta-maintained, the incremental path leaves them alone + assert reloaded.holder_count == 3 + assert reloaded.counters_updated_at == 200 + end + + test "recalculates a token without a watermark from scratch, replacing stale values" do + token = insert(:token, transfer_count: 1000, holder_count: 1000) + + block = insert(:block, number: 50) + insert_transfer(token, block) + + insert(:address_current_token_balance, token_contract_address_hash: token.contract_address_hash, value: 10) + insert(:address_current_token_balance, token_contract_address_hash: token.contract_address_hash, value: 20) + + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], 200) + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 1 + assert reloaded.holder_count == 2 + assert reloaded.counters_updated_at == 200 + end + + test "the holder recount excludes the burn address, counts any ERC-7984 row and distinct holders once" do + token = insert(:token, holder_count: nil) + + burn_address = insert(:address, hash: "0x0000000000000000000000000000000000000000") + + insert(:address_current_token_balance, + address: burn_address, + token_contract_address_hash: token.contract_address_hash, + value: 100 + ) + + confidential_holder = insert(:address) + + insert(:address_current_token_balance, + address: confidential_holder, + token_contract_address_hash: token.contract_address_hash, + value: 0, + token_type: "ERC-7984" + ) + + multi_row_holder = insert(:address) + + insert(:address_current_token_balance, + address: multi_row_holder, + token_contract_address_hash: token.contract_address_hash, + token_id: 1, + token_type: "ERC-1155", + value: 5 + ) + + insert(:address_current_token_balance, + address: multi_row_holder, + token_contract_address_hash: token.contract_address_hash, + token_id: 2, + token_type: "ERC-1155", + value: 7 + ) + + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], 200) + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + # confidential holder + the multi-row holder counted once; burn excluded + assert reloaded.holder_count == 2 + end + + test "a drifted holder_count is normalized to the recount" do + token = insert(:token, transfer_count: 0, holder_count: 100) + + insert(:address_current_token_balance, token_contract_address_hash: token.contract_address_hash, value: 10) + + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], 200) + + assert Repo.get_by(Token, contract_address_hash: token.contract_address_hash).holder_count == 1 + end + + test "skips tokens already consolidated at or above the safe block" do + token = insert(:token, transfer_count: 7, counters_updated_at: 300) + + block = insert(:block, number: 150) + insert_transfer(token, block) + + TokenCountersConsolidator.consolidate_tokens([token.contract_address_hash], 200) + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 7 + assert reloaded.counters_updated_at == 300 + end + end + + describe "consolidation cycle process" do + test "the process stays responsive while a cycle runs and reports it in its state" do + pid = start_supervised!(TokenCountersConsolidator) + + send(pid, :consolidate) + + # the cycle runs in a task, so the process answers immediately + assert %{cycle_task: _, cycle_started_at: _, last_cycle_finished_at: _} = :sys.get_state(pid, 1_000) + + wait_for_results(fn -> + case :sys.get_state(pid, 1_000) do + %{cycle_task: nil, last_cycle_finished_at: %DateTime{}} -> :ok + _ -> raise Ecto.NoResultsError, queryable: "cycle completion" + end + end) + end + end + + describe "consolidate/0" do + setup do + initial_env = Application.get_env(:explorer, Consolidation) || [] + + Application.put_env( + :explorer, + Consolidation, + Keyword.merge(initial_env, safe_block_lag: 0) + ) + + on_exit(fn -> Application.put_env(:explorer, Consolidation, initial_env) end) + + start_supervised!(TokenCounters) + + :ok + end + + test "consolidates dirty tokens and clears their markers" do + token = insert(:token, transfer_count: 1, counters_updated_at: 10) + + insert(:block, number: 0) + block = insert(:block, number: 100) + insert_transfer(token, block) + + TokenCounters.mark_dirty([{token.contract_address_hash.bytes, 100}]) + :sys.get_state(TokenCounters) + + TokenCountersConsolidator.consolidate() + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 2 + assert reloaded.counters_updated_at == 100 + assert TokenCounters.dirty_empty?() + end + + test "keeps markers above the safe block" do + token = insert(:token, transfer_count: 1, counters_updated_at: 10) + + insert(:block, number: 0) + insert(:block, number: 100) + + TokenCounters.mark_dirty([{token.contract_address_hash.bytes, 500}]) + :sys.get_state(TokenCounters) + + TokenCountersConsolidator.consolidate() + + assert Repo.get_by(Token, contract_address_hash: token.contract_address_hash).counters_updated_at == 100 + refute TokenCounters.dirty_empty?() + end + + test "settles re-fetched blocks: adds the re-imported transfers below the watermark and drops the queue rows" do + token = insert(:token, transfer_count: 5, counters_updated_at: 100) + + reimported_block = insert(:block, number: 50, refetch_needed: false) + insert_transfer(token, reimported_block) + + insert(:block, number: 100) + Repo.insert!(%CountersRefetchBlock{block_number: 50}) + + TokenCountersConsolidator.consolidate() + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 6 + # settling a covered block does not move the watermark + assert reloaded.counters_updated_at == 100 + assert Repo.aggregate(CountersRefetchBlock, :count) == 0 + end + end + + describe "apply_covered_transfer_deltas/3" do + test "applies negative deltas only for blocks covered by the token watermark" do + token = insert(:token, transfer_count: 5, counters_updated_at: 100) + + token_transfers = [ + %{token_contract_address_hash: token.contract_address_hash, block_number: 50}, + # above the watermark: never counted, must not be subtracted + %{token_contract_address_hash: token.contract_address_hash, block_number: 150} + ] + + [updated_bytes] = TokenCountersConsolidator.apply_covered_transfer_deltas(token_transfers, :negative) + + assert updated_bytes == token.contract_address_hash.bytes + + reloaded = Repo.get_by(Token, contract_address_hash: token.contract_address_hash) + + assert reloaded.transfer_count == 4 + assert reloaded.counters_updated_at == 100 + end + + test "skips tokens without a watermark" do + token = insert(:token, transfer_count: 5) + + token_transfers = [%{token_contract_address_hash: token.contract_address_hash, block_number: 50}] + + assert TokenCountersConsolidator.apply_covered_transfer_deltas(token_transfers, :positive) == [] + assert Repo.get_by(Token, contract_address_hash: token.contract_address_hash).transfer_count == 5 + end + end + + describe "reset_covered_watermarks/2" do + test "resets only tokens whose watermark covers the block and marks them dirty" do + start_supervised!(TokenCounters) + + covered_token = insert(:token, counters_updated_at: 100) + fresh_token = insert(:token, counters_updated_at: 30) + + reset_bytes = + TokenCountersConsolidator.reset_covered_watermarks(%{ + covered_token.contract_address_hash.bytes => 50, + fresh_token.contract_address_hash.bytes => 50 + }) + + assert reset_bytes == [covered_token.contract_address_hash.bytes] + assert is_nil(Repo.get_by(Token, contract_address_hash: covered_token.contract_address_hash).counters_updated_at) + assert Repo.get_by(Token, contract_address_hash: fresh_token.contract_address_hash).counters_updated_at == 30 + + # the recalculation of the reset token is scheduled + :sys.get_state(TokenCounters) + assert {[{marked_bytes, _block_number}], _continuation} = TokenCounters.select_dirty(10) + assert marked_bytes == covered_token.contract_address_hash.bytes + end + end + + defp insert_transfer(token, block) do + transaction = :transaction |> insert() |> with_block(block) + + insert(:token_transfer, + transaction: transaction, + block: block, + block_number: block.number, + token_contract_address: token.contract_address + ) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/counters/token_counters_test.exs b/apps/explorer/test/explorer/chain/cache/counters/token_counters_test.exs new file mode 100644 index 00000000000..eb3d6f68bdd --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/counters/token_counters_test.exs @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Counters.TokenCountersTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.Counters.TokenCounters + + setup do + start_supervised!(TokenCounters) + + :ok + end + + describe "fetch/1" do + test "returns the transfer_count column value and seeds the cache" do + token = insert(:token, transfer_count: 5, counters_updated_at: 100) + + assert TokenCounters.fetch(token) == 5 + + # the second read is served from ETS: a live bump is visible on top + TokenCounters.bump_live(token.contract_address_hash.bytes, %{transfer_count: 2, max_block_number: 101}) + + assert TokenCounters.fetch(token) == 7 + end + + test "returns zero and marks the token dirty when the counter was never calculated" do + token = insert(:token, transfer_count: nil) + + assert TokenCounters.fetch(token) == 0 + + :sys.get_state(TokenCounters) + + assert {[{bytes, _block_number}], _continuation} = TokenCounters.select_dirty(10) + assert bytes == token.contract_address_hash.bytes + end + end + + describe "compute_transfer_deltas/2" do + test "computes per-token deltas skipping rows without a block number" do + token_a = insert(:token) + token_b = insert(:token) + + token_transfers = [ + %{token_contract_address_hash: token_a.contract_address_hash, block_number: 100}, + %{token_contract_address_hash: token_a.contract_address_hash, block_number: 105}, + %{token_contract_address_hash: token_b.contract_address_hash, block_number: 101}, + %{token_contract_address_hash: token_a.contract_address_hash, block_number: nil} + ] + + deltas = TokenCounters.compute_transfer_deltas(token_transfers) + + assert %{transfer_count: 2, max_block_number: 105} = deltas[token_a.contract_address_hash.bytes] + assert %{transfer_count: 1, max_block_number: 101} = deltas[token_b.contract_address_hash.bytes] + end + + test "the include? predicate excludes single contributions" do + token = insert(:token) + + token_transfers = [ + %{token_contract_address_hash: token.contract_address_hash, block_number: 10}, + %{token_contract_address_hash: token.contract_address_hash, block_number: 20} + ] + + deltas = TokenCounters.compute_transfer_deltas(token_transfers, fn _bytes, block_number -> block_number <= 15 end) + + assert %{transfer_count: 1, max_block_number: 10} = deltas[token.contract_address_hash.bytes] + end + end + + describe "bump_live/2" do + test "never creates entries for tokens that are not cached" do + token = insert(:token, transfer_count: 1, counters_updated_at: 100) + + TokenCounters.bump_live(token.contract_address_hash.bytes, %{transfer_count: 5, max_block_number: 101}) + + # the first read seeds from the DB column: the bump was not applied anywhere + assert TokenCounters.fetch(token) == 1 + end + end + + describe "handle_new_data/2" do + test "bumps cached entries for live data and marks tokens dirty" do + token = insert(:token, transfer_count: 10, counters_updated_at: 100) + TokenCounters.fetch(token) + + token_transfers = [%{token_contract_address_hash: token.contract_address_hash, block_number: 300}] + + TokenCounters.handle_new_data(token_transfers, true) + :sys.get_state(TokenCounters) + + assert TokenCounters.fetch(token) == 11 + assert {[{bytes, 300}], _continuation} = TokenCounters.select_dirty(10) + assert bytes == token.contract_address_hash.bytes + end + + test "only marks tokens dirty for catchup data" do + token = insert(:token, transfer_count: 10, counters_updated_at: 100) + TokenCounters.fetch(token) + + token_transfers = [%{token_contract_address_hash: token.contract_address_hash, block_number: 300}] + + TokenCounters.handle_new_data(token_transfers, false) + :sys.get_state(TokenCounters) + + assert TokenCounters.fetch(token) == 10 + assert {[{_bytes, 300}], _continuation} = TokenCounters.select_dirty(10) + end + end + + describe "mark_dirty/1" do + test "is a no-op on api-only nodes" do + initial_mode = Application.get_env(:explorer, :mode) + Application.put_env(:explorer, :mode, :api) + on_exit(fn -> Application.put_env(:explorer, :mode, initial_mode) end) + + TokenCounters.mark_dirty([{insert(:token).contract_address_hash.bytes, 10}]) + :sys.get_state(TokenCounters) + + assert TokenCounters.dirty_empty?() + end + + test "does not create new markers above the configured cap" do + initial_env = Application.get_env(:explorer, TokenCounters) || [] + Application.put_env(:explorer, TokenCounters, Keyword.merge(initial_env, max_dirty_markers: 1)) + on_exit(fn -> Application.put_env(:explorer, TokenCounters, initial_env) end) + + token_a = insert(:token) + token_b = insert(:token) + + TokenCounters.mark_dirty([{token_a.contract_address_hash.bytes, 10}]) + TokenCounters.mark_dirty([{token_b.contract_address_hash.bytes, 20}]) + # already-marked tokens can still be bumped + TokenCounters.mark_dirty([{token_a.contract_address_hash.bytes, 30}]) + :sys.get_state(TokenCounters) + + assert {[{bytes, 30}], _continuation} = TokenCounters.select_dirty(10) + assert bytes == token_a.contract_address_hash.bytes + end + end + + describe "stats/0" do + test "reports cached tokens and dirty markers" do + token = insert(:token, transfer_count: 5, counters_updated_at: 100) + TokenCounters.fetch(token) + + TokenCounters.mark_dirty([{insert(:token).contract_address_hash.bytes, 10}]) + :sys.get_state(TokenCounters) + + assert %{cached_tokens: 1, dirty_markers: 1, memory_bytes: memory_bytes} = TokenCounters.stats() + assert memory_bytes > 0 + end + end + + describe "delete_dirty_markers/2" do + test "keeps markers above the safe block" do + token_a = insert(:token) + token_b = insert(:token) + + TokenCounters.mark_dirty([{token_a.contract_address_hash.bytes, 100}, {token_b.contract_address_hash.bytes, 200}]) + :sys.get_state(TokenCounters) + + TokenCounters.delete_dirty_markers( + [token_a.contract_address_hash.bytes, token_b.contract_address_hash.bytes], + 150 + ) + + assert {[{bytes, 200}], _continuation} = TokenCounters.select_dirty(10) + assert bytes == token_b.contract_address_hash.bytes + end + end + + describe "eviction" do + test "drops entries not touched within the TTL" do + initial_env = Application.get_env(:explorer, TokenCounters) || [] + Application.put_env(:explorer, TokenCounters, Keyword.merge(initial_env, ttl: 0)) + on_exit(fn -> Application.put_env(:explorer, TokenCounters, initial_env) end) + + token = insert(:token, transfer_count: 5, counters_updated_at: 100) + TokenCounters.fetch(token) + + Process.sleep(1) + send(Process.whereis(TokenCounters), :evict) + :sys.get_state(TokenCounters) + + assert is_nil(TokenCounters.snapshot(token.contract_address_hash.bytes)) + end + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/address/current_token_balances_test.exs b/apps/explorer/test/explorer/chain/import/runner/address/current_token_balances_test.exs index fc544223519..4e213346163 100644 --- a/apps/explorer/test/explorer/chain/import/runner/address/current_token_balances_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/address/current_token_balances_test.exs @@ -145,10 +145,12 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalancesTest do token_id: ^token_id_6 } ], + # one address holding multiple token_ids of the same token is + # a single holder address_current_token_balances_update_token_holder_counts: [ %{ contract_address_hash: ^token_contract_address_hash, - holder_count: 2 + holder_count: 1 }, %{ contract_address_hash: ^token_erc_20_contract_address_hash, @@ -160,7 +162,7 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalancesTest do }, %{ contract_address_hash: ^token_erc_404_contract_address_hash, - holder_count: 2 + holder_count: 1 } ] }} = @@ -573,6 +575,92 @@ defmodule Explorer.Chain.Import.Runner.Address.CurrentTokenBalancesTest do options ) end + + test "burn address balances never affect the holder_count", %{ + token: %Token{contract_address_hash: token_contract_address_hash}, + options: options + } do + %Address{hash: burn_address_hash} = insert(:address, hash: "0x0000000000000000000000000000000000000000") + + assert {:ok, %{address_current_token_balances_update_token_holder_counts: []}} = + run_changes( + %{ + address_hash: burn_address_hash, + token_contract_address_hash: token_contract_address_hash, + block_number: 1, + value: Decimal.new(100), + value_fetched_at: DateTime.utc_now() + }, + options + ) + end + + test "an inserted ERC-7984 balance counts as a holder regardless of value, updates do not double count", %{ + address: %Address{hash: address_hash}, + options: options + } do + %Token{contract_address_hash: token_contract_address_hash} = insert(:token, type: "ERC-7984", holder_count: 0) + + changes = %{ + address_hash: address_hash, + token_contract_address_hash: token_contract_address_hash, + block_number: 1, + value: Decimal.new(0), + value_fetched_at: DateTime.utc_now(), + token_type: "ERC-7984" + } + + assert {:ok, + %{ + address_current_token_balances_update_token_holder_counts: [ + %{contract_address_hash: ^token_contract_address_hash, holder_count: 1} + ] + }} = run_changes(changes, options) + + # updating an existing confidential balance is a 1 -> 1 transition + assert {:ok, %{address_current_token_balances_update_token_holder_counts: []}} = + run_changes(%{changes | block_number: 2, value_fetched_at: DateTime.utc_now()}, options) + end + + test "zeroing one token_id of a multi-token_id holder does not change the holder_count", %{ + address: %Address{hash: address_hash} = address, + options: options + } do + %Token{contract_address_hash: token_contract_address_hash} = insert(:token, type: "ERC-1155", holder_count: 1) + + insert(:address_current_token_balance, + address: address, + token_contract_address_hash: token_contract_address_hash, + token_id: 1, + token_type: "ERC-1155", + value: 5, + block_number: 1 + ) + + insert(:address_current_token_balance, + address: address, + token_contract_address_hash: token_contract_address_hash, + token_id: 2, + token_type: "ERC-1155", + value: 7, + block_number: 1 + ) + + # the address still holds token_id 2, so its holder-ness did not change + assert {:ok, %{address_current_token_balances_update_token_holder_counts: []}} = + run_changes( + %{ + address_hash: address_hash, + token_contract_address_hash: token_contract_address_hash, + block_number: 2, + value: Decimal.new(0), + value_fetched_at: DateTime.utc_now(), + token_id: Decimal.new(1), + token_type: "ERC-1155" + }, + options + ) + end end defp run_changes(changes, options) when is_map(changes) do diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index dd03ca2ce8e..2e26a1b51f9 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -15,7 +15,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do alias Explorer.Chain.{Address, Block, Transaction, PendingBlockOperation} alias Explorer.Chain.Cache.BlockNumber alias Explorer.{Chain, Repo} - alias Explorer.Utility.{AddressCountersRefetchBlock, MissingBlockRange} + alias Explorer.Utility.{CountersRefetchBlock, MissingBlockRange} describe "run/1" do setup do @@ -118,7 +118,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, block_params) - assert {:ok, %{address_counters_refetched_block_numbers: [^block_number]}} = + assert {:ok, %{counters_refetched_block_numbers: [^block_number]}} = Multi.new() |> Blocks.run([block_changes], options) |> Repo.transaction() @@ -130,11 +130,11 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert reloaded.gas_used == 100 assert reloaded.counters_updated_at == block_number + 10 - assert Repo.aggregate(AddressCountersRefetchBlock, :count) == 1 + assert Repo.aggregate(CountersRefetchBlock, :count) == 1 # a second import of the same block applies no further deltas: the # re-fetch was already settled into the queue - assert {:ok, %{address_counters_refetched_block_numbers: []}} = + assert {:ok, %{counters_refetched_block_numbers: []}} = Multi.new() |> Blocks.run([block_changes], options) |> Repo.transaction() @@ -170,7 +170,7 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, block_params) - assert {:ok, %{address_counters_corrections: %{reset: reset_bytes}}} = + assert {:ok, %{counters_corrections: %{reset: reset_bytes}}} = Multi.new() |> Blocks.run([block_changes], options) |> Repo.transaction() diff --git a/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs b/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs index 5f0a3e7f8f5..a04629742ec 100644 --- a/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs +++ b/apps/explorer/test/explorer/migrator/backfill_address_counters_test.exs @@ -5,17 +5,17 @@ defmodule Explorer.Migrator.BackfillAddressCountersTest do import Ecto.Query alias Explorer.Chain.Address - alias Explorer.Chain.Cache.Counters.AddressCountersConsolidator + alias Explorer.Chain.Cache.Counters.{AddressCountersConsolidator, Consolidation} alias Explorer.Migrator.{BackfillAddressCounters, MigrationStatus} alias Explorer.Repo setup do - initial_consolidator_env = Application.get_env(:explorer, AddressCountersConsolidator) || [] + initial_consolidation_env = Application.get_env(:explorer, Consolidation) || [] Application.put_env( :explorer, - AddressCountersConsolidator, - Keyword.merge(initial_consolidator_env, safe_block_lag: 0) + Consolidation, + Keyword.merge(initial_consolidation_env, safe_block_lag: 0) ) initial_migrator_env = Application.get_env(:explorer, BackfillAddressCounters) || [] @@ -27,7 +27,7 @@ defmodule Explorer.Migrator.BackfillAddressCountersTest do ) on_exit(fn -> - Application.put_env(:explorer, AddressCountersConsolidator, initial_consolidator_env) + Application.put_env(:explorer, Consolidation, initial_consolidation_env) Application.put_env(:explorer, BackfillAddressCounters, initial_migrator_env) end) diff --git a/apps/explorer/test/explorer/migrator/backfill_token_counters_test.exs b/apps/explorer/test/explorer/migrator/backfill_token_counters_test.exs new file mode 100644 index 00000000000..d50e8167667 --- /dev/null +++ b/apps/explorer/test/explorer/migrator/backfill_token_counters_test.exs @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Migrator.BackfillTokenCountersTest do + use Explorer.DataCase, async: false + + import Ecto.Query + + alias Explorer.Chain.Cache.Counters.Consolidation + alias Explorer.Chain.Token + alias Explorer.Migrator.{BackfillTokenCounters, MigrationStatus} + alias Explorer.Repo + + setup do + initial_consolidation_env = Application.get_env(:explorer, Consolidation) || [] + + Application.put_env( + :explorer, + Consolidation, + Keyword.merge(initial_consolidation_env, safe_block_lag: 0) + ) + + initial_migrator_env = Application.get_env(:explorer, BackfillTokenCounters) || [] + + Application.put_env( + :explorer, + BackfillTokenCounters, + Keyword.merge(initial_migrator_env, batch_size: 2, concurrency: 2, timeout: 0) + ) + + on_exit(fn -> + Application.put_env(:explorer, Consolidation, initial_consolidation_env) + Application.put_env(:explorer, BackfillTokenCounters, initial_migrator_env) + end) + + :ok + end + + test "backfills counters for tokens that were never consolidated" do + insert(:block, number: 0) + block = insert(:block, number: 100) + + # legacy values written at unknown heights get recalculated + active_token = insert(:token, transfer_count: 999, holder_count: 999) + + transaction = :transaction |> insert() |> with_block(block) + + insert(:token_transfer, + transaction: transaction, + block: block, + block_number: block.number, + token_contract_address: active_token.contract_address + ) + + insert(:address_current_token_balance, token_contract_address_hash: active_token.contract_address_hash, value: 10) + + dormant_token = insert(:token, transfer_count: nil, holder_count: nil) + + already_consolidated_token = insert(:token, transfer_count: 42, holder_count: 42, counters_updated_at: 77) + + assert MigrationStatus.get_status("backfill_token_counters") == nil + + BackfillTokenCounters.start_link([]) + + wait_for_results(fn -> + Repo.one!( + from(ms in MigrationStatus, + where: ms.migration_name == ^"backfill_token_counters" and ms.status == "completed" + ) + ) + end) + + safe_block = Consolidation.safe_block() + + active_reloaded = Repo.get_by(Token, contract_address_hash: active_token.contract_address_hash) + assert active_reloaded.transfer_count == 1 + assert active_reloaded.holder_count == 1 + assert active_reloaded.counters_updated_at == safe_block + + dormant_reloaded = Repo.get_by(Token, contract_address_hash: dormant_token.contract_address_hash) + assert dormant_reloaded.transfer_count == 0 + assert dormant_reloaded.holder_count == 0 + assert dormant_reloaded.counters_updated_at == safe_block + + # tokens already consolidated are left untouched + consolidated_reloaded = Repo.get_by(Token, contract_address_hash: already_consolidated_token.contract_address_hash) + assert consolidated_reloaded.transfer_count == 42 + assert consolidated_reloaded.holder_count == 42 + assert consolidated_reloaded.counters_updated_at == 77 + + # nothing is left to process + assert Repo.one(from(token in Token, where: is_nil(token.counters_updated_at), select: count())) == 0 + end + + test "resumes from the cursor stored in the migration meta" do + insert(:block, number: 0) + insert(:block, number: 100) + + tokens = Enum.map(1..5, fn _ -> insert(:token) end) + max_hash = tokens |> Enum.map(&to_string(&1.contract_address_hash)) |> Enum.max() + + BackfillTokenCounters.start_link([]) + + wait_for_results(fn -> + Repo.one!( + from(ms in MigrationStatus, + where: ms.migration_name == ^"backfill_token_counters" and ms.status == "completed" + ) + ) + end) + + %{meta: meta} = MigrationStatus.fetch("backfill_token_counters") + + assert meta["max_processed_hash"] >= max_hash + end +end diff --git a/apps/indexer/lib/indexer/fetcher/token_counters_updater.ex b/apps/indexer/lib/indexer/fetcher/token_counters_updater.ex index 91960a0cdfa..1399be01d9b 100644 --- a/apps/indexer/lib/indexer/fetcher/token_counters_updater.ex +++ b/apps/indexer/lib/indexer/fetcher/token_counters_updater.ex @@ -1,7 +1,15 @@ # SPDX-License-Identifier: LicenseRef-Blockscout defmodule Indexer.Fetcher.TokenCountersUpdater do @moduledoc """ - Updates counters for cataloged tokens. + Periodically exports the counters of cataloged tokens to the Multichain + Search service. + + The counter values are read straight from the `tokens.holder_count` and + `tokens.transfer_count` columns — they are maintained by the incremental + counters machinery (import-time holder deltas and + `Explorer.Chain.Cache.Counters.TokenCountersConsolidator`), so no aggregates + run here. Tokens not yet consolidated are skipped until their first + consolidation. """ use Indexer.Fetcher, restart: :permanent @@ -10,7 +18,6 @@ defmodule Indexer.Fetcher.TokenCountersUpdater do alias Explorer.Chain.Token alias Explorer.MicroserviceInterfaces.MultichainSearch alias Indexer.BufferedTask - alias Timex.Duration @behaviour BufferedTask @@ -44,28 +51,33 @@ defmodule Indexer.Fetcher.TokenCountersUpdater do @impl BufferedTask def init(initial, reducer, _) do - counters_updater_milliseconds_interval = Application.get_env(:indexer, __MODULE__)[:milliseconds_interval] + # BufferedTask re-runs init whenever the queue drains; throttle the full + # export sweep to the configured interval (the legacy implementation was + # implicitly throttled by the metadata staleness predicate it reused) + interval = Application.get_env(:indexer, __MODULE__)[:milliseconds_interval] + last_stream_at = :persistent_term.get({__MODULE__, :last_stream_at}, 0) + now = System.system_time(:millisecond) - interval_in_minutes = - counters_updater_milliseconds_interval - |> Duration.from_milliseconds() - |> Duration.to_minutes() - |> trunc() + if now - last_stream_at >= interval do + :persistent_term.put({__MODULE__, :last_stream_at}, now) - {:ok, tokens} = Token.stream_cataloged_tokens(initial, reducer, interval_in_minutes, true) + {:ok, tokens} = Token.stream_cataloged_tokens_for_counters(initial, reducer, true) - tokens + tokens + else + initial + end end @impl BufferedTask def run(entries, _json_rpc_named_arguments) do - Logger.debug("updating token counters") + Logger.debug("exporting token counters") entries |> Enum.reduce(%{}, fn token, acc -> - {transfers_count, holders_count} = Token.fetch_token_counters(token.contract_address_hash, :infinity) + data_for_multichain = + MultichainSearch.prepare_token_counters_for_queue(token.transfer_count || 0, token.holder_count || 0) - data_for_multichain = MultichainSearch.prepare_token_counters_for_queue(transfers_count, holders_count) Map.put(acc, token.contract_address_hash.bytes, data_for_multichain) end) |> MultichainSearch.send_token_info_to_queue(:counters) diff --git a/config/runtime.exs b/config/runtime.exs index 31c37f6b987..2cde9a38271 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -439,14 +439,21 @@ config :explorer, Explorer.Chain.Cache.Counters.AddressCountersConsolidator, interval: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_INTERVAL", "10m"), batch_size: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_BATCH_SIZE", 100, min: 1), concurrency: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_CONCURRENCY", 4, min: 1), - safe_block_lag: ConfigHelper.parse_integer_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG", 12, min: 0), query_timeout: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT", "5m") -config :explorer, Explorer.Chain.Cache.Counters.TokenHoldersCount, - cache_period: ConfigHelper.parse_time_env_var("CACHE_TOKEN_HOLDERS_COUNTER_PERIOD", "1h") +config :explorer, Explorer.Chain.Cache.Counters.Consolidation, + safe_block_lag: ConfigHelper.parse_integer_env_var("CACHE_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG", 12, min: 0) -config :explorer, Explorer.Chain.Cache.Counters.TokenTransfersCount, - cache_period: ConfigHelper.parse_time_env_var("CACHE_TOKEN_TRANSFERS_COUNTER_PERIOD", "1h") +config :explorer, Explorer.Chain.Cache.Counters.TokenCounters, + ttl: ConfigHelper.parse_time_env_var("CACHE_TOKEN_COUNTERS_TTL", "2h"), + max_dirty_markers: ConfigHelper.parse_integer_env_var("CACHE_TOKEN_COUNTERS_MAX_DIRTY_MARKERS", 200_000, min: 1) + +config :explorer, Explorer.Chain.Cache.Counters.TokenCountersConsolidator, + enabled: !ConfigHelper.parse_bool_env_var("CACHE_TOKEN_COUNTERS_CONSOLIDATION_DISABLED"), + interval: ConfigHelper.parse_time_env_var("CACHE_TOKEN_COUNTERS_CONSOLIDATION_INTERVAL", "10m"), + batch_size: ConfigHelper.parse_integer_env_var("CACHE_TOKEN_COUNTERS_CONSOLIDATION_BATCH_SIZE", 100, min: 1), + concurrency: ConfigHelper.parse_integer_env_var("CACHE_TOKEN_COUNTERS_CONSOLIDATION_CONCURRENCY", 2, min: 1), + query_timeout: ConfigHelper.parse_time_env_var("CACHE_TOKEN_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT", "5m") config :explorer, Explorer.Chain.Cache.Counters.AverageBlockTime, enabled: true, @@ -957,6 +964,12 @@ config :explorer, Explorer.Migrator.BackfillAddressCounters, concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY", 2), timeout: ConfigHelper.parse_time_env_var("MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT", "500ms") +config :explorer, Explorer.Migrator.BackfillTokenCounters, + enabled: !ConfigHelper.parse_bool_env_var("MIGRATION_BACKFILL_TOKEN_COUNTERS_DISABLED"), + batch_size: ConfigHelper.parse_integer_env_var("MIGRATION_BACKFILL_TOKEN_COUNTERS_BATCH_SIZE", 50), + concurrency: ConfigHelper.parse_integer_env_var("MIGRATION_BACKFILL_TOKEN_COUNTERS_CONCURRENCY", 4), + timeout: ConfigHelper.parse_time_env_var("MIGRATION_BACKFILL_TOKEN_COUNTERS_TIMEOUT", "100ms") + config :explorer, Explorer.Chain.BridgedToken, eth_omni_bridge_mediator: System.get_env("BRIDGED_TOKENS_ETH_OMNI_BRIDGE_MEDIATOR"), bsc_omni_bridge_mediator: System.get_env("BRIDGED_TOKENS_BSC_OMNI_BRIDGE_MEDIATOR"), diff --git a/cspell.json b/cspell.json index 008c6164dd9..46f2fbd29d4 100644 --- a/cspell.json +++ b/cspell.json @@ -794,6 +794,7 @@ "wighawag", "WITHDRAWABILITY", "wobserver", + "wordsize", "wysdvjkizxonu", "x-ratelimit-limit", "x-ratelimit-remaining", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 9608cc3522b..043126b6269 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -121,8 +121,6 @@ RELEASE_LINK= # CACHE_BLOCK_COUNT_PERIOD=7200 # CACHE_TXS_COUNT_PERIOD=7200 # CACHE_ADDRESS_SUM_PERIOD=3600 -# CACHE_TOKEN_HOLDERS_COUNTER_PERIOD=3600 -# CACHE_TOKEN_TRANSFERS_COUNTER_PERIOD=3600 # CACHE_ADDRESS_COUNT_PERIOD=1800 # CACHE_AVERAGE_BLOCK_PERIOD=1800 # CACHE_MARKET_HISTORY_PERIOD=21600 @@ -133,12 +131,23 @@ RELEASE_LINK= # CACHE_ADDRESS_COUNTERS_CONSOLIDATION_INTERVAL=10m # CACHE_ADDRESS_COUNTERS_CONSOLIDATION_BATCH_SIZE=100 # CACHE_ADDRESS_COUNTERS_CONSOLIDATION_CONCURRENCY=4 -# CACHE_ADDRESS_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG=12 # CACHE_ADDRESS_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT=5m +# CACHE_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG=12 +# CACHE_TOKEN_COUNTERS_TTL=2h +# CACHE_TOKEN_COUNTERS_MAX_DIRTY_MARKERS=200000 +# CACHE_TOKEN_COUNTERS_CONSOLIDATION_DISABLED=false +# CACHE_TOKEN_COUNTERS_CONSOLIDATION_INTERVAL=10m +# CACHE_TOKEN_COUNTERS_CONSOLIDATION_BATCH_SIZE=100 +# CACHE_TOKEN_COUNTERS_CONSOLIDATION_CONCURRENCY=2 +# CACHE_TOKEN_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT=5m # MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED=false # MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE=10 # MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY=2 # MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT=500ms +# MIGRATION_BACKFILL_TOKEN_COUNTERS_DISABLED=false +# MIGRATION_BACKFILL_TOKEN_COUNTERS_BATCH_SIZE=50 +# MIGRATION_BACKFILL_TOKEN_COUNTERS_CONCURRENCY=4 +# MIGRATION_BACKFILL_TOKEN_COUNTERS_TIMEOUT=100ms # CACHE_HOT_SMART_CONTRACTS_5M_PERIOD=30s # CACHE_HOT_SMART_CONTRACTS_1H_PERIOD=6m # CACHE_HOT_SMART_CONTRACTS_3H_PERIOD=18m From fbf8d898a26d1dab97e493bcf307765c2f97b0ed Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:46:54 +0400 Subject: [PATCH 16/52] fix: Disable old broadcast for token transfers without subscribers (#14777) --- .../lib/block_scout_web/notifier.ex | 99 ++++++++++++------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 4851f8fbccc..2a6e2496349 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -321,15 +321,10 @@ defmodule BlockScoutWeb.Notifier do ) |> Instance.preload_nft(@api_true) - transfers_by_token = - Enum.group_by(all_token_transfers_full, fn tt -> to_string(tt.token_contract_address_hash) end) + broadcast_token_transfers_websocket_v2(all_token_transfers_full) - broadcast_token_transfers_websocket_v2(all_token_transfers_full, transfers_by_token) - - for {_token_contract_address_hash, token_transfers} <- transfers_by_token do - token_transfers - |> Enum.each(&broadcast_token_transfer/1) - end + # TODO: delete duplicated event when old UI becomes deprecated + broadcast_token_transfers_websocket_v1(all_token_transfers_full) end end @@ -865,26 +860,40 @@ defmodule BlockScoutWeb.Notifier do end end - defp broadcast_token_transfers_websocket_v2(tokens_transfers, transfers_by_token) do - for {token_contract_address_hash, token_transfers} <- transfers_by_token do - Endpoint.broadcast("tokens:#{token_contract_address_hash}", "token_transfer", %{ - token_transfer: Enum.count(token_transfers) - }) - end + defp broadcast_token_transfers_websocket_v2(tokens_transfers) do + case Enum.filter(tokens_transfers, &token_transfer_has_v2_subscribers?/1) do + [] -> + :ok - prepared_token_transfers = - TransactionView.render("token_transfers.json", %{ - token_transfers: tokens_transfers, - conn: nil - }) + relevant_token_transfers -> + for {token_contract_address_hash, token_transfers} <- + Enum.group_by(relevant_token_transfers, fn tt -> to_string(tt.token_contract_address_hash) end) do + Endpoint.broadcast("tokens:#{token_contract_address_hash}", "token_transfer", %{ + token_transfer: Enum.count(token_transfers) + }) + end + + prepared_token_transfers = + TransactionView.render("token_transfers.json", %{ + token_transfers: relevant_token_transfers, + conn: nil + }) + + relevant_token_transfers + |> Enum.zip(prepared_token_transfers) + |> group_by_address_hashes_and_broadcast( + "token_transfer", + :token_transfers, + &{&1["transaction_hash"], &1["block_hash"], &1["log_index"]} + ) + end + end + # TODO: delete this function when old UI becomes deprecated + defp broadcast_token_transfers_websocket_v1(tokens_transfers) do tokens_transfers - |> Enum.zip(prepared_token_transfers) - |> group_by_address_hashes_and_broadcast( - "token_transfer", - :token_transfers, - &{&1["transaction_hash"], &1["block_hash"], &1["log_index"]} - ) + |> Enum.filter(&token_transfer_has_old_subscribers?/1) + |> Enum.each(&broadcast_token_transfer/1) end defp broadcast_token_transfer(token_transfer) do @@ -948,10 +957,16 @@ defmodule BlockScoutWeb.Notifier do defp address_has_subscribers?(nil), do: false defp address_has_subscribers?(address_hash) do - has_subscribers?("addresses:" <> to_string(address_hash)) or + address_has_v2_subscribers?(address_hash) or address_has_old_subscribers?(address_hash) end + defp address_has_v2_subscribers?(nil), do: false + + defp address_has_v2_subscribers?(address_hash) do + has_subscribers?("addresses:" <> to_string(address_hash)) + end + # TODO: delete this function when old UI becomes deprecated defp address_has_old_subscribers?(nil), do: false @@ -960,18 +975,34 @@ defmodule BlockScoutWeb.Notifier do end defp token_transfer_has_subscribers?(tt) do - address_has_subscribers?(tt.from_address_hash) or - address_has_subscribers?(tt.to_address_hash) or - token_has_subscribers?(tt.token_contract_address_hash) + token_transfer_has_v2_subscribers?(tt) or + token_transfer_has_old_subscribers?(tt) end - defp token_has_subscribers?(nil), do: false + defp token_transfer_has_v2_subscribers?(tt) do + address_has_v2_subscribers?(tt.from_address_hash) or + address_has_v2_subscribers?(tt.to_address_hash) or + token_has_v2_subscribers?(tt.token_contract_address_hash) + end - defp token_has_subscribers?(token_address_hash) do - hash_string = to_string(token_address_hash) + # TODO: delete this function when old UI becomes deprecated + defp token_transfer_has_old_subscribers?(tt) do + address_has_old_subscribers?(tt.from_address_hash) or + address_has_old_subscribers?(tt.to_address_hash) or + token_has_old_subscribers?(tt.token_contract_address_hash) + end + + defp token_has_v2_subscribers?(nil), do: false + + defp token_has_v2_subscribers?(token_address_hash) do + has_subscribers?("tokens:" <> to_string(token_address_hash)) + end + + # TODO: delete this function when old UI becomes deprecated + defp token_has_old_subscribers?(nil), do: false - has_subscribers?("tokens:" <> hash_string) or - has_subscribers?("tokens_old:" <> hash_string) + defp token_has_old_subscribers?(token_address_hash) do + has_subscribers?("tokens_old:" <> to_string(token_address_hash)) end defp transaction_has_subscribers?(transaction) do From de8ed34c5e5c1fb4b9b4fef99fe50bd96dc7dd47 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 7 Sep 2026 17:30:46 +0300 Subject: [PATCH 17/52] perf: Preload only rendered transaction fields in v1 tokentx endpoints (#14782) --- apps/explorer/lib/explorer/etherscan.ex | 28 ++++++++++++++--- .../explorer/test/explorer/etherscan_test.exs | 31 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan.ex b/apps/explorer/lib/explorer/etherscan.ex index ea38e4cec84..6704e551270 100644 --- a/apps/explorer/lib/explorer/etherscan.ex +++ b/apps/explorer/lib/explorer/etherscan.ex @@ -776,13 +776,33 @@ defmodule Explorer.Etherscan do |> maybe_preload_entities() end + # Transaction fields needed to render the `tokentx`-family RPC responses (see + # `BlockScoutWeb.API.RPC.AddressView.prepare_common_token_transfer/3`) and to decode the method call. + # Token transfers are fetched in pages of up to `@default_options.page_size` items, so preloading full + # transaction rows for each of them was one of the top consumers of DB time on the API read replica. + # `hash` must stay in the list: Ecto uses it to match preloaded transactions to token transfers. + @token_transfer_transaction_fields [ + :hash, + :block_timestamp, + :nonce, + :index, + :gas, + :gas_price, + :gas_used, + :cumulative_gas_used, + :input, + :to_address_hash, + :created_contract_address_hash + ] + defp maybe_preload_entities(query) do + transaction_query = + from(transaction in Transaction, select: struct(transaction, ^@token_transfer_transaction_fields)) + if DenormalizationHelper.tt_denormalization_finished?() do - query - |> preload([:transaction, :token]) + preload(query, [:token, transaction: ^transaction_query]) else - query - |> preload([:block, :token, :transaction]) + preload(query, [:block, :token, transaction: ^transaction_query]) end end diff --git a/apps/explorer/test/explorer/etherscan_test.exs b/apps/explorer/test/explorer/etherscan_test.exs index 0c9763bf259..04e7ab4a6e0 100644 --- a/apps/explorer/test/explorer/etherscan_test.exs +++ b/apps/explorer/test/explorer/etherscan_test.exs @@ -1277,6 +1277,37 @@ defmodule Explorer.EtherscanTest do assert token_transfer.to_address_hash == found_token_transfer.to_address_hash end + test "preloads only the transaction fields used by the RPC view" do + transaction = + :transaction + |> insert() + |> with_block() + + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) + + [found_token_transfer] = Etherscan.list_token_transfers(:erc20, token_transfer.from_address_hash, nil, %{}) + + assert %Transaction{} = found_transaction = found_token_transfer.transaction + assert found_transaction.hash == transaction.hash + assert found_transaction.input == transaction.input + assert found_transaction.nonce == transaction.nonce + assert found_transaction.index == transaction.index + assert found_transaction.gas == transaction.gas + assert found_transaction.gas_price == transaction.gas_price + assert found_transaction.gas_used == transaction.gas_used + assert found_transaction.cumulative_gas_used == transaction.cumulative_gas_used + + # signature fields are not rendered by the RPC view, so they must not be fetched + assert is_nil(found_transaction.r) + assert is_nil(found_transaction.s) + assert is_nil(found_transaction.v) + end + test "with address with 0 token transfers" do address = insert(:address) From 8eca95261d35272e5613f5bb075eae246d86a7f8 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:26:23 +0400 Subject: [PATCH 18/52] perf: Scam addresses ETS cache (#14781) Co-authored-by: Victor Baranov --- apps/explorer/lib/explorer/application.ex | 1 + .../lib/explorer/chain/address/reputation.ex | 19 +- .../chain/address/scam_badge_to_address.ex | 13 +- .../explorer/chain/cache/scam_addresses.ex | 196 ++++++++++++++++++ .../chain/cache/scam_addresses_test.exs | 125 +++++++++++ config/runtime.exs | 5 + docker-compose/envs/common-blockscout.env | 18 +- 7 files changed, 356 insertions(+), 21 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/scam_addresses.ex create mode 100644 apps/explorer/test/explorer/chain/cache/scam_addresses_test.exs diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 528b161cc3f..aa74c875cf5 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -448,6 +448,7 @@ defmodule Explorer.Application do |> configure_mode_dependent_process(:indexer) |> configure_chain_type_dependent_process(:zksync), configure_mode_dependent_process(Explorer.Chain.Fetcher.AddressesBlacklist, :api), + configure_mode_dependent_process(Explorer.Chain.Cache.ScamAddresses, :api), only_in_mode(Explorer.Migrator.SwitchPendingOperations, :indexer), configure_mode_dependent_process(Explorer.Utility.RateLimiter, :api), Hammer.child_for_supervisor() |> configure_mode_dependent_process(:api), diff --git a/apps/explorer/lib/explorer/chain/address/reputation.ex b/apps/explorer/lib/explorer/chain/address/reputation.ex index 1e74e5b1495..75717cf375b 100644 --- a/apps/explorer/lib/explorer/chain/address/reputation.ex +++ b/apps/explorer/lib/explorer/chain/address/reputation.ex @@ -5,9 +5,8 @@ defmodule Explorer.Chain.Address.Reputation do """ use Explorer.Schema - alias Explorer.Chain.Address.ScamBadgeToAddress + alias Explorer.Chain.Cache.ScamAddresses alias Explorer.Chain.Hash - alias Explorer.Repo @enum_values [:ok, :scam] def enum_values, do: @enum_values @@ -19,20 +18,18 @@ defmodule Explorer.Chain.Address.Reputation do end def preload_reputation(address_hashes) do - scam_badges = + scam_hashes = if Application.get_env(:block_scout_web, :hide_scam_addresses) do - ScamBadgeToAddress - |> where([sb], sb.address_hash in ^address_hashes) - |> Repo.replica().all() - |> Map.new(&{&1.address_hash, &1}) + ScamAddresses.scam_hashes(address_hashes) else - %{} + MapSet.new() end Enum.map(address_hashes, fn address_hash -> - case Map.get(scam_badges, address_hash) do - nil -> {address_hash, %__MODULE__{reputation: "ok"}} - _badge -> {address_hash, %__MODULE__{reputation: "scam"}} + if MapSet.member?(scam_hashes, address_hash) do + {address_hash, %__MODULE__{reputation: "scam"}} + else + {address_hash, %__MODULE__{reputation: "ok"}} end end) end diff --git a/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex b/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex index 41d86b36e0f..5a9187d01d9 100644 --- a/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex +++ b/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex @@ -10,6 +10,7 @@ defmodule Explorer.Chain.Address.ScamBadgeToAddress do alias Explorer.{Chain, Repo} alias Explorer.Chain.{Address, Hash} + alias Explorer.Chain.Cache.ScamAddresses import Ecto.Query, only: [from: 2] @@ -51,7 +52,11 @@ defmodule Explorer.Chain.Address.ScamBadgeToAddress do end) |> Enum.filter(&(!is_nil(&1))) - safe_add(insert_params) + result = safe_add(insert_params) + + ScamAddresses.reload() + + result end defp safe_add(insert_params) do @@ -88,7 +93,11 @@ defmodule Explorer.Chain.Address.ScamBadgeToAddress do select: bta ) - Repo.delete_all(query) + result = Repo.delete_all(query) + + ScamAddresses.reload() + + result end @doc """ diff --git a/apps/explorer/lib/explorer/chain/cache/scam_addresses.ex b/apps/explorer/lib/explorer/chain/cache/scam_addresses.ex new file mode 100644 index 00000000000..02a194a5424 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/scam_addresses.ex @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.ScamAddresses do + @moduledoc """ + Keeps the set of addresses carrying a scam badge in an ETS table. + + `Explorer.Chain.Address.Reputation` needs nothing but a membership test, and + `scam_address_badge_mappings` changes only through the admin badge endpoints, + so the whole set fits in memory and the per-preload replica query becomes an + ETS lookup. + + The table is reloaded on an interval and whenever a badge is assigned or + revoked. Both happen in this process, one at a time, so a reload can never + install a set that predates the change that triggered it. + + That holds within an instance. A badge change reloads the cache of the instance + that served the admin request and of no other, so `:update_interval` is also the + window in which instances can disagree about a badge — lower it where that + matters. Nothing propagates the change sooner: libcluster runs only in the + separate `:api` and `:indexer` modes, and the chain event sender is in-process + in `:all` mode, so neither transport reaches every deployment topology. + + The cache is authoritative only once it has been loaded. Until the first + reload succeeds — and whenever the process is not running at all — reads fall + through to the database, so a disabled or not yet warmed up cache behaves + exactly as if it did not exist. That fallback is also what bounds the cache: + holding the whole table in memory pays off only while the table is small, and + past `:max_size` rows the reload — a full scan on every instance, every + interval — costs more than the queries it saves, so an oversized table is not + cached at all. + """ + + use GenServer + + require Logger + + import Ecto.Query, only: [select: 3, where: 3] + + alias Explorer.Chain.Address.ScamBadgeToAddress + alias Explorer.Chain.Cache.Counters.Helper, as: CountersHelper + alias Explorer.Chain.Hash + alias Explorer.Repo + + @table_name "scam_address_badge_mappings" + + @cache_name :scam_addresses + + # Tells a loaded table apart from an empty one. Cached keys are `Hash.Address` + # structs, so an atom can never collide with one. + @loaded_key :__loaded__ + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @impl true + @spec init(any()) :: {:ok, nil} + def init(_) do + :ets.new(@cache_name, [ + :set, + :named_table, + :public, + read_concurrency: true + ]) + + send(self(), :reload) + + {:ok, nil} + end + + @impl true + def handle_info(:reload, state) do + load() + Process.send_after(self(), :reload, update_interval()) + + {:noreply, state} + end + + @impl true + def handle_call(:reload, _from, state) do + load() + + {:reply, :ok, state} + end + + @doc """ + Returns the subset of `address_hashes` that carry a scam badge. + """ + @spec scam_hashes([Hash.Address.t()]) :: MapSet.t(Hash.Address.t()) + def scam_hashes([]), do: MapSet.new() + + def scam_hashes(address_hashes) do + if loaded?() do + for address_hash <- address_hashes, + :ets.member(@cache_name, address_hash), + into: MapSet.new(), + do: address_hash + else + db_scam_hashes(address_hashes) + end + end + + @doc """ + Checks whether the given address carries a scam badge. + """ + @spec scam?(Hash.Address.t()) :: boolean() + def scam?(address_hash) do + [address_hash] |> scam_hashes() |> MapSet.member?(address_hash) + end + + @doc """ + Reloads the cache, so that a badge just assigned or revoked shows up without + waiting for the next interval. + """ + @spec reload() :: :ok + def reload do + GenServer.call(__MODULE__, :reload) + catch + # The cache is not running — disabled, or on its way down — and the reads it + # would have served already fall through to the database. + :exit, _ -> :ok + end + + defp load do + entries = if oversized?(), do: :oversized, else: fetch_entries() + + :ets.delete_all_objects(@cache_name) + + # An oversized table is left without the marker, which is what sends reads to + # the database rather than letting an empty table report every address clean. + case entries do + :oversized -> :ok + entries -> :ets.insert(@cache_name, [{@loaded_key} | entries]) + end + + :ok + rescue + error -> + # Reads fall back to the database until the next interval. Letting this + # crash would take the table down with the process and restart it in a loop + # for as long as the database is unreachable. + Logger.error("Failed to load the scam addresses cache: #{inspect(error)}") + + :ok + end + + defp fetch_entries do + ScamBadgeToAddress + |> select([badge], {badge.address_hash}) + |> Repo.replica().all() + end + + # A `reltuples` estimate rather than a `count/0`: this guard exists to keep the + # reload cheap, so it cannot start with a full scan of its own. A table + # PostgreSQL has no statistics for reports `nil` and is let through — a table + # that new is not the one the guard is here for. + defp oversized? do + max_size = max_size() + estimated_size = CountersHelper.estimated_count_from(@table_name, api?: true) + + if estimated_size && estimated_size > max_size do + Logger.warning( + "Scam addresses cache is not used: #{@table_name} is estimated at #{estimated_size} rows, over the limit of #{max_size}. " <> + "Scam badge lookups fall back to the database. Raise CACHE_SCAM_ADDRESSES_MAX_SIZE to cache the table anyway." + ) + + true + else + false + end + end + + defp loaded? do + :ets.whereis(@cache_name) != :undefined and :ets.member(@cache_name, @loaded_key) + end + + defp db_scam_hashes(address_hashes) do + ScamBadgeToAddress + |> where([badge], badge.address_hash in ^address_hashes) + |> select([badge], badge.address_hash) + |> Repo.replica().all() + |> MapSet.new() + end + + defp config do + Application.get_env(:explorer, __MODULE__) + end + + defp update_interval do + config()[:update_interval] + end + + defp max_size do + config()[:max_size] + end +end diff --git a/apps/explorer/test/explorer/chain/cache/scam_addresses_test.exs b/apps/explorer/test/explorer/chain/cache/scam_addresses_test.exs new file mode 100644 index 00000000000..8269b160579 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/scam_addresses_test.exs @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.ScamAddressesTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Address.{Reputation, ScamBadgeToAddress} + alias Explorer.Chain.Cache.ScamAddresses + + describe "scam_hashes/1" do + test "falls back to the database while the cache is not running" do + %{address_hash: badged_hash} = insert(:scam_badge_to_address) + clean_hash = insert(:address).hash + + assert ScamAddresses.scam_hashes([badged_hash, clean_hash]) == MapSet.new([badged_hash]) + end + + test "answers from the cache once it is loaded" do + %{address_hash: badged_hash} = insert(:scam_badge_to_address) + clean_hash = insert(:address).hash + + start_cache() + + # only a cached answer can still see a badge that is no longer in the database + Repo.delete_all(ScamBadgeToAddress) + + assert ScamAddresses.scam_hashes([badged_hash, clean_hash]) == MapSet.new([badged_hash]) + end + + test "returns an empty set for no address hashes" do + assert ScamAddresses.scam_hashes([]) == MapSet.new() + end + + test "goes back to the database when the table outgrows the size limit" do + %{address_hash: badged_hash} = insert(:scam_badge_to_address) + + start_cache() + assert ScamAddresses.scam?(badged_hash) + + analyze_badges() + put_max_size(0) + ScamAddresses.reload() + + # nothing is cached any more, so the row going away is visible at once + Repo.delete_all(ScamBadgeToAddress) + + refute ScamAddresses.scam?(badged_hash) + end + end + + describe "badge changes" do + test "assigning a badge fills the cache" do + address_hash = insert(:address).hash + + start_cache() + refute ScamAddresses.scam?(address_hash) + + ScamBadgeToAddress.add([to_string(address_hash)]) + + assert ScamAddresses.scam?(address_hash) + end + + test "revoking a badge drops it from the cache" do + %{address_hash: address_hash} = insert(:scam_badge_to_address) + + start_cache() + assert ScamAddresses.scam?(address_hash) + + ScamBadgeToAddress.delete([address_hash]) + + refute ScamAddresses.scam?(address_hash) + end + end + + describe "preload_reputation/1" do + setup do + put_test_env(:block_scout_web, :hide_scam_addresses, true) + end + + test "marks cached addresses as scam" do + %{address_hash: badged_hash} = insert(:scam_badge_to_address) + clean_hash = insert(:address).hash + + start_cache() + Repo.delete_all(ScamBadgeToAddress) + + assert [{^badged_hash, %Reputation{reputation: "scam"}}, {^clean_hash, %Reputation{reputation: "ok"}}] = + Reputation.preload_reputation([badged_hash, clean_hash]) + end + end + + # The cache loads itself on start, and `reload/0` is synchronous, so waiting on + # it is enough to know the table is filled. + defp start_cache do + start_supervised!(ScamAddresses) + + ScamAddresses.reload() + end + + defp put_max_size(max_size) do + config = Application.get_env(:explorer, ScamAddresses, []) + + put_test_env(:explorer, ScamAddresses, Keyword.merge(config, max_size: max_size)) + end + + # A setting that was not configured has to be restored as not configured — + # putting back the `nil` that `get_env/2` reports would shadow the defaults + # every later `get_env/3` passes. + defp put_test_env(app, key, value) do + initial_config = Application.fetch_env(app, key) + + Application.put_env(app, key, value) + + on_exit(fn -> + case initial_config do + {:ok, initial_value} -> Application.put_env(app, key, initial_value) + :error -> Application.delete_env(app, key) + end + end) + end + + # The size guard reads `pg_class` statistics, which a table this fresh has none + # of — without an `ANALYZE` the estimate is `nil` and the guard lets it through. + defp analyze_badges do + Repo.query!("ANALYZE #{ScamBadgeToAddress.__schema__(:source)}") + end +end diff --git a/config/runtime.exs b/config/runtime.exs index 2cde9a38271..4c6e9ccb675 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1002,6 +1002,11 @@ config :explorer, Explorer.Chain.Fetcher.AddressesBlacklist, retry_interval: ConfigHelper.parse_time_env_var("ADDRESSES_BLACKLIST_RETRY_INTERVAL", "5s"), provider: ConfigHelper.parse_catalog_value("ADDRESSES_BLACKLIST_PROVIDER", ["blockaid"], false, "blockaid") +config :explorer, Explorer.Chain.Cache.ScamAddresses, + enabled: ConfigHelper.parse_bool_env_var("HIDE_SCAM_ADDRESSES"), + update_interval: ConfigHelper.parse_time_env_var("CACHE_SCAM_ADDRESSES_UPDATE_INTERVAL", "5m"), + max_size: ConfigHelper.parse_integer_env_var("CACHE_SCAM_ADDRESSES_MAX_SIZE", 200_000, min: 0) + rate_limiter_redis_url = ConfigHelper.parse_url_env_var("RATE_LIMITER_REDIS_URL") rate_limiter_redis_sentinel_urls = ConfigHelper.safe_get_env("RATE_LIMITER_REDIS_SENTINEL_URLS", "") diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 043126b6269..8f32a6d2852 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -140,20 +140,14 @@ RELEASE_LINK= # CACHE_TOKEN_COUNTERS_CONSOLIDATION_BATCH_SIZE=100 # CACHE_TOKEN_COUNTERS_CONSOLIDATION_CONCURRENCY=2 # CACHE_TOKEN_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT=5m -# MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED=false -# MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE=10 -# MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY=2 -# MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT=500ms -# MIGRATION_BACKFILL_TOKEN_COUNTERS_DISABLED=false -# MIGRATION_BACKFILL_TOKEN_COUNTERS_BATCH_SIZE=50 -# MIGRATION_BACKFILL_TOKEN_COUNTERS_CONCURRENCY=4 -# MIGRATION_BACKFILL_TOKEN_COUNTERS_TIMEOUT=100ms # CACHE_HOT_SMART_CONTRACTS_5M_PERIOD=30s # CACHE_HOT_SMART_CONTRACTS_1H_PERIOD=6m # CACHE_HOT_SMART_CONTRACTS_3H_PERIOD=18m # CACHE_TRANSACTIONS_24H_STATS_PERIOD= # CACHE_FRESH_PENDING_TRANSACTIONS_COUNTER_PERIOD= # CACHE_PENDING_OPERATIONS_COUNT_PERIOD= +# CACHE_SCAM_ADDRESSES_UPDATE_INTERVAL= +# CACHE_SCAM_ADDRESSES_MAX_SIZE= # PENDING_OPERATIONS_HELPER_TRANSACTIONS_BATCH_SIZE=1000 # PENDING_OPERATIONS_HELPER_BLOCKS_BATCH_SIZE=10 # TOKEN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD= @@ -636,6 +630,14 @@ ACCOUNT_REDIS_URL=redis://redis-db:6379 # MIGRATION_EMPTY_INTERNAL_TRANSACTIONS_DATA_BATCH_SIZE= # MIGRATION_EMPTY_INTERNAL_TRANSACTIONS_DATA_CONCURRENCY= # MIGRATION_EMPTY_INTERNAL_TRANSACTIONS_DATA_TIMEOUT= +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED=false +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE=10 +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY=2 +# MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT=500ms +# MIGRATION_BACKFILL_TOKEN_COUNTERS_DISABLED=false +# MIGRATION_BACKFILL_TOKEN_COUNTERS_BATCH_SIZE=50 +# MIGRATION_BACKFILL_TOKEN_COUNTERS_CONCURRENCY=4 +# MIGRATION_BACKFILL_TOKEN_COUNTERS_TIMEOUT=100ms # HEALTH_MONITOR_CHECK_INTERVAL=1m # HEALTH_MONITOR_BLOCKS_PERIOD=5m # HEALTH_MONITOR_BATCHES_PERIOD=4h From 9aeca252702613e304b94d6467c7d5849ff4ba6d Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 8 Sep 2026 12:35:04 +0300 Subject: [PATCH 19/52] perf: Limit logs before joining transactions in topic-only getLogs (#14780) --- apps/explorer/lib/explorer/etherscan/logs.ex | 212 +++++++-------- .../test/explorer/etherscan/logs_test.exs | 257 +++++++++++++++++- 2 files changed, 348 insertions(+), 121 deletions(-) diff --git a/apps/explorer/lib/explorer/etherscan/logs.ex b/apps/explorer/lib/explorer/etherscan/logs.ex index c6c3e575aaf..1393807b080 100644 --- a/apps/explorer/lib/explorer/etherscan/logs.ex +++ b/apps/explorer/lib/explorer/etherscan/logs.ex @@ -6,7 +6,8 @@ defmodule Explorer.Etherscan.Logs do """ - import Ecto.Query, only: [dynamic: 2, from: 2, where: 2, where: 3, subquery: 1, order_by: 3, union_all: 2] + import Ecto.Query, + only: [dynamic: 2, from: 2, join: 5, limit: 2, where: 2, where: 3, subquery: 1, order_by: 3, union_all: 2] alias Explorer.{Chain, Repo} alias Explorer.Chain.{DenormalizationHelper, Log, Transaction} @@ -87,62 +88,24 @@ defmodule Explorer.Etherscan.Logs do |> page_logs(paging_options) |> where_topic_match(prepared_filter, union_multiple_values: true) - if DenormalizationHelper.transactions_denormalization_finished?() do - all_transaction_logs_query = - from(log in subquery(logs_query), - join: transaction in Transaction, - on: log.transaction_hash == transaction.hash and log.block_hash == transaction.block_hash, - where: transaction.block_consensus == true, - select: map(log, ^@log_fields), - select_merge: %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: transaction.block_number, - block_timestamp: transaction.block_timestamp, - block_consensus: transaction.block_consensus - }, - order_by: [asc: log.block_number, asc: log.index], - limit: 1000 - ) - - all_transaction_logs_query - |> Chain.wrapped_union_subquery() - |> order_by([log], asc: log.block_number, asc: log.index) - |> Repo.replica().all() - else - all_transaction_logs_query = - from(log in subquery(logs_query), - join: transaction in Transaction, - on: log.transaction_hash == transaction.hash and log.block_hash == transaction.block_hash, - inner_join: block in assoc(transaction, :block), - where: block.consensus == true, - select: map(log, ^@log_fields), - select_merge: %{ - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: transaction.block_number, - block_timestamp: block.timestamp, - block_consensus: block.consensus - }, - order_by: [asc: log.block_number, asc: log.index], - limit: 1000 - ) - - all_transaction_logs_query - |> Chain.wrapped_union_subquery() - |> order_by([log], asc: log.block_number, asc: log.index) - |> Repo.replica().all() - end + logs_query + |> join_transaction_data() + |> limit(1000) + |> fetch_ordered() end - # Since address_hash was not present, we know that a - # topic filter has been applied, so we use a different - # query that is optimized for a logs filter over an - # address_hash + # Since address_hash was not present, we know that a topic filter has been + # applied. Ordering, paging and the LIMIT are applied inside a subquery over + # `logs` alone, so the planner has to produce at most 1000 logs before + # joining `transactions`. Joining first and limiting afterwards lets the + # planner scan the whole `transactions` block range up front, which is + # prohibitively slow for wide ranges. + # + # Logs of non-consensus blocks are never deleted, so consensus has to be + # checked before the LIMIT: otherwise a page could come back short, or empty + # with no cursor to continue from, while later consensus logs still exist. + # The check uses the same predicate as `join_transaction_data/1`, so no row + # that makes it into the page is dropped by the outer join afterwards. def list_logs(filter, paging_options) do paging_options = if is_nil(paging_options), do: @default_paging_options, else: paging_options prepared_filter = Map.merge(@base_filter, filter) @@ -152,74 +115,83 @@ defmodule Explorer.Etherscan.Logs do |> where_topic_match(prepared_filter) |> where([log], log.block_number >= ^prepared_filter.from_block) |> where([log], log.block_number <= ^prepared_filter.to_block) + |> where_consensus() + |> page_logs(paging_options) + |> order_by([log], asc: log.block_number, asc: log.index) + |> limit(1000) + logs_query + |> join_transaction_data() + |> fetch_ordered() + end + + # Keeps only logs whose consensus predicate matches the one applied by + # `join_transaction_data/1` in the current denormalization state. Each check + # is a primary-key lookup per candidate log. `transactions.block_consensus` + # can diverge from `blocks.consensus` (see + # `Explorer.Migrator.TransactionBlockConsensus`), which is why the predicate + # is not simply `blocks.consensus` in both states. + defp where_consensus(logs_query) do if DenormalizationHelper.transactions_denormalization_finished?() do - block_transaction_query = - from(transaction in Transaction, - where: transaction.block_number >= ^prepared_filter.from_block, - where: transaction.block_number <= ^prepared_filter.to_block, - where: transaction.block_consensus == true, - select: %{ - transaction_hash: transaction.hash, - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: transaction.block_hash, - block_number: transaction.block_number, - block_timestamp: transaction.block_timestamp, - block_consensus: transaction.block_consensus - } - ) - - query_with_block_transaction_data = - from(log in logs_query, - join: block_transaction_data in subquery(block_transaction_query), - on: - block_transaction_data.transaction_hash == log.transaction_hash and - block_transaction_data.block_hash == log.block_hash, - order_by: log.block_number, - limit: 1000, - select: block_transaction_data, - select_merge: map(log, ^@log_fields) - ) - - query_with_block_transaction_data - |> order_by([log], asc: log.index) - |> page_logs(paging_options) - |> Repo.replica().all() + join(logs_query, :inner, [log], transaction in Transaction, + on: + log.transaction_hash == transaction.hash and log.block_hash == transaction.block_hash and + transaction.block_consensus == true + ) else - block_transaction_query = - from(transaction in Transaction, - join: block in assoc(transaction, :block), - where: block.number >= ^prepared_filter.from_block, - where: block.number <= ^prepared_filter.to_block, - where: block.consensus == true, - select: %{ - transaction_hash: transaction.hash, - gas_price: transaction.gas_price, - gas_used: transaction.gas_used, - transaction_index: transaction.index, - block_hash: block.hash, - block_number: block.number, - block_timestamp: block.timestamp, - block_consensus: block.consensus - } - ) - - query_with_block_transaction_data = - from(log in logs_query, - join: block_transaction_data in subquery(block_transaction_query), - on: block_transaction_data.transaction_hash == log.transaction_hash, - order_by: log.block_number, - limit: 1000, - select: block_transaction_data, - select_merge: map(log, ^@log_fields) - ) - - query_with_block_transaction_data - |> order_by([log], asc: log.index) - |> page_logs(paging_options) - |> Repo.replica().all() + join(logs_query, :inner, [log], block in assoc(log, :block), on: block.consensus == true) + end + end + + # Re-selects the joined query through an outer subquery so that fields + # taken from the `logs` subquery are loaded with their schema types (`Hash` + # structs instead of raw binaries), then applies the final ordering. + defp fetch_ordered(query) do + query + |> Chain.wrapped_union_subquery() + |> order_by([log], asc: log.block_number, asc: log.index) + |> Repo.replica().all() + end + + # Wraps `logs_query` in a subquery and joins each log to its consensus + # transaction by `(hash, block_hash)`, a primary-key lookup per log. The + # selected shape is identical in both denormalization states. + defp join_transaction_data(logs_query) do + if DenormalizationHelper.transactions_denormalization_finished?() do + from(log in subquery(logs_query), + join: transaction in Transaction, + on: log.transaction_hash == transaction.hash and log.block_hash == transaction.block_hash, + where: transaction.block_consensus == true, + select: map(log, ^@log_fields), + select_merge: %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: transaction.block_timestamp, + block_consensus: transaction.block_consensus + }, + order_by: [asc: log.block_number, asc: log.index] + ) + else + from(log in subquery(logs_query), + join: transaction in Transaction, + on: log.transaction_hash == transaction.hash and log.block_hash == transaction.block_hash, + inner_join: block in assoc(transaction, :block), + where: block.consensus == true, + select: map(log, ^@log_fields), + select_merge: %{ + gas_price: transaction.gas_price, + gas_used: transaction.gas_used, + transaction_index: transaction.index, + block_hash: transaction.block_hash, + block_number: transaction.block_number, + block_timestamp: block.timestamp, + block_consensus: block.consensus + }, + order_by: [asc: log.block_number, asc: log.index] + ) end end diff --git a/apps/explorer/test/explorer/etherscan/logs_test.exs b/apps/explorer/test/explorer/etherscan/logs_test.exs index dc23b3c634e..07822172423 100644 --- a/apps/explorer/test/explorer/etherscan/logs_test.exs +++ b/apps/explorer/test/explorer/etherscan/logs_test.exs @@ -2,10 +2,13 @@ defmodule Explorer.Etherscan.LogsTest do use Explorer.DataCase + import Ecto.Query, only: [from: 2] import Explorer.Factory alias Explorer.Etherscan.Logs - alias Explorer.Chain.Transaction + alias Explorer.Chain.{Block, Transaction} + alias Explorer.Chain.Cache.BackgroundMigrations + alias Explorer.Repo @first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65" @first_topic_hex_string_2 "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" @@ -860,5 +863,257 @@ defmodule Explorer.Etherscan.LogsTest do assert block_number_order == Enum.sort(block_number_order) end + + test "paginates topic-only logs" do + contract_address = insert(:contract_address) + first_topic = topic(@first_topic_hex_string_1) + + transaction_a = + :transaction + |> insert(to_address: contract_address) + |> with_block() + + transaction_b = + :transaction + |> insert(to_address: contract_address) + |> with_block() + + transaction_c = + :transaction + |> insert(to_address: contract_address) + |> with_block() + + inserted_records = + for {transaction, count} <- [{transaction_a, 700}, {transaction_b, 700}, {transaction_c, 600}], + i <- 1..count do + insert(:log, + address: contract_address, + transaction: transaction, + block_number: transaction.block.number, + block: transaction.block, + index: i, + first_topic: first_topic + ) + end + + # A log with a different topic in the same range must not be returned. + insert(:log, + address: contract_address, + transaction: transaction_a, + block_number: transaction_a.block.number, + block: transaction_a.block, + index: 701, + first_topic: topic(@first_topic_hex_string_2) + ) + + filter = %{ + from_block: transaction_a.block.number, + to_block: transaction_c.block.number, + first_topic: first_topic + } + + first_found_logs = Logs.list_logs(filter) + + assert Enum.count(first_found_logs) == 1_000 + + last_record = List.last(first_found_logs) + + next_page_params = %{ + log_index: last_record.index, + block_number: last_record.block_number + } + + second_found_logs = Logs.list_logs(filter, next_page_params) + + assert Enum.count(second_found_logs) == 1_000 + + all_found_logs = first_found_logs ++ second_found_logs + + assert Enum.all?(all_found_logs, &(&1.first_topic == first_topic)) + + found_keys = MapSet.new(all_found_logs, &{&1.block_number, &1.index}) + inserted_keys = MapSet.new(inserted_records, &{&1.block_number, &1.index}) + + assert MapSet.equal?(found_keys, inserted_keys) + end + + test "topic-only logs are sorted by block and index" do + first_block = insert(:block) + second_block = insert(:block) + third_block = insert(:block) + + contract_address = insert(:contract_address) + first_topic = topic(@first_topic_hex_string_1) + + transaction_block1 = + %Transaction{} = + :transaction + |> insert(to_address: contract_address) + |> with_block(first_block) + + transaction_block2 = + %Transaction{} = + :transaction + |> insert(to_address: contract_address) + |> with_block(second_block) + + transaction_block3 = + %Transaction{} = + :transaction + |> insert(to_address: contract_address) + |> with_block(third_block) + + for {transaction, block, index} <- [ + {transaction_block3, third_block, 1}, + {transaction_block1, first_block, 2}, + {transaction_block2, second_block, 1}, + {transaction_block1, first_block, 1} + ] do + insert(:log, + address: contract_address, + transaction: transaction, + block: block, + block_number: block.number, + index: index, + first_topic: first_topic + ) + end + + filter = %{ + from_block: first_block.number, + to_block: third_block.number, + first_topic: first_topic + } + + found_logs = Logs.list_logs(filter) + + assert length(found_logs) == 4 + + order = Enum.map(found_logs, &{&1.block_number, &1.index}) + + assert order == Enum.sort(order) + end + + test "topic-only logs skip a full page of non-consensus logs and still return later consensus logs" do + contract_address = insert(:contract_address) + first_topic = topic(@first_topic_hex_string_1) + + non_consensus_block = insert(:block) + consensus_block = insert(:block, number: non_consensus_block.number + 1) + + non_consensus_transaction = + :transaction + |> insert(to_address: contract_address) + |> with_block(non_consensus_block) + + consensus_transaction = + :transaction + |> insert(to_address: contract_address) + |> with_block(consensus_block) + + for i <- 1..1_000 do + insert(:log, + address: contract_address, + transaction: non_consensus_transaction, + block: non_consensus_block, + block_number: non_consensus_block.number, + index: i, + first_topic: first_topic + ) + end + + consensus_log = + insert(:log, + address: contract_address, + transaction: consensus_transaction, + block: consensus_block, + block_number: consensus_block.number, + index: 1, + first_topic: first_topic + ) + + # The factory only attaches transactions to consensus blocks, so the + # reorg is simulated after the fact. + Repo.update_all(from(b in Block, where: b.hash == ^non_consensus_block.hash), set: [consensus: false]) + + Repo.update_all(from(t in Transaction, where: t.hash == ^non_consensus_transaction.hash), + set: [block_consensus: false] + ) + + filter = %{ + from_block: non_consensus_block.number, + to_block: consensus_block.number, + first_topic: first_topic + } + + [found_log] = Logs.list_logs(filter) + + assert found_log.block_number == consensus_log.block_number + assert found_log.index == consensus_log.index + assert found_log.block_consensus == true + end + + test "topic-only logs skip a full page of logs whose transactions lost consensus while their block did not" do + old_denormalization_finished = BackgroundMigrations.get_transactions_denormalization_finished() + BackgroundMigrations.set_transactions_denormalization_finished(true) + + on_exit(fn -> + BackgroundMigrations.set_transactions_denormalization_finished(old_denormalization_finished) + end) + + contract_address = insert(:contract_address) + first_topic = topic(@first_topic_hex_string_1) + + stale_block = insert(:block) + consensus_block = insert(:block, number: stale_block.number + 1) + + stale_transaction = + :transaction + |> insert(to_address: contract_address) + |> with_block(stale_block) + + consensus_transaction = + :transaction + |> insert(to_address: contract_address) + |> with_block(consensus_block) + + for i <- 1..1_000 do + insert(:log, + address: contract_address, + transaction: stale_transaction, + block: stale_block, + block_number: stale_block.number, + index: i, + first_topic: first_topic + ) + end + + consensus_log = + insert(:log, + address: contract_address, + transaction: consensus_transaction, + block: consensus_block, + block_number: consensus_block.number, + index: 1, + first_topic: first_topic + ) + + # `transactions.block_consensus` and `blocks.consensus` can diverge; the + # block keeps consensus here while the denormalized flag says otherwise. + Repo.update_all(from(t in Transaction, where: t.hash == ^stale_transaction.hash), + set: [block_consensus: false] + ) + + filter = %{ + from_block: stale_block.number, + to_block: consensus_block.number, + first_topic: first_topic + } + + [found_log] = Logs.list_logs(filter) + + assert found_log.block_number == consensus_log.block_number + assert found_log.index == consensus_log.index + end end end From ad84255422a11f684380198a6716817a1dd599a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:47:27 +0300 Subject: [PATCH 20/52] chore(deps): bump open_api_spex from 3.22.3 to 3.22.4 (#14790) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 4ade0fedc17..1576f540db6 100644 --- a/mix.lock +++ b/mix.lock @@ -127,7 +127,7 @@ "nx": {:hex, :nx, "0.12.1", "6e9fee43a77646d04faad2ba4e449b9e270c6b23e413f203cb4d81e71c0a617f", [:mix], [{:complex, "~> 0.7", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee80f6ae898f68bbfe7f30216b06aab10096231ec99ded1208a827256b23d0fb"}, "oauth2": {:hex, :oauth2, "2.1.1", "3bec9bf49e88e1b0c0ddf9f44c393d71fd0f88e905766f2e2cc5d57e6bcc5352", [:mix], [{:tesla, "~> 1.18", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "1d5997cb1ff1643dac17076b6c00c91e4381d8389f3c49a8c390984606dad439"}, "oban": {:hex, :oban, "2.24.0", "cd877a089ead66658c40864fc616e90c75ddba95559fffae3fccaf05da58a4e5", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab497b2be51191a2d9d7a1b267714463891be711b20487b7b65b5c2269e4b312"}, - "open_api_spex": {:hex, :open_api_spex, "3.22.3", "0e383bf23cc3a060bffaebbcd09fc06bfc908d948c00e518aed36bbf8a2fe473", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "5f74f1878fdc38f8e961b0b943ac7af88dcf3a82a0c0ef6680ddfd3d161aecbd"}, + "open_api_spex": {:hex, :open_api_spex, "3.22.4", "00d8f32676b459080a860b81d777a8010e61dfa333f54175dcf51dd6f35a0fac", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f658d73396e2277e584085a6146481eb085d815a716089defe4f94b26cf8b05f"}, "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, From abd9e7b923bcd337fa06056957ac35269aae05ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:48:02 +0300 Subject: [PATCH 21/52] chore(deps): bump joken from 2.6.2 to 2.7.0 (#14792) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 1576f540db6..5d7718b7108 100644 --- a/mix.lock +++ b/mix.lock @@ -93,9 +93,9 @@ "inet_cidr": {:hex, :inet_cidr, "1.0.9", "e0ef72a2942529da78c8e4147d53f2ef5f6f5293335c3637b0fdf83c012cc816", [:mix], [], "hexpm", "172da15ff7cf635b1feaf14f5818be28c811b37cc5fb7c5f7c01058c1c1066cc"}, "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, - "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, + "joken": {:hex, :joken, "2.7.0", "a9fd87805b1b58313435c04b950857d7557f019b157d503dc4783006e4e80ed1", [:mix], [{:jose, "~> 1.11.12", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "ffd0d92e12dbf497311386b75f9b5027d370e32c74f6e5576804d3a9eed668ab"}, "joken_jwks": {:hex, :joken_jwks, "1.7.0", "5db750732dddbfb51068c16da6e43439ad8a3932020bdd505e944dd918ec0771", [:mix], [{:hackney, "~> 1.18", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.6", [hex: :joken, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:tesla, "~> 1.4", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "c1dd2d5161f079f660bda773c338d3ff2b90bb79673a0852becc0433416797d3"}, - "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, + "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"}, "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"}, "junit_formatter": {:hex, :junit_formatter, "3.4.0", "d0e8db6c34dab6d3c4154c3b46b21540db1109ae709d6cf99ba7e7a2ce4b1ac2", [:mix], [], "hexpm", "bb36e2ae83f1ced6ab931c4ce51dd3dbef1ef61bb4932412e173b0cfa259dacd"}, From c62038fac076899f725e70e4d5faa078e5b27db7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:48:28 +0300 Subject: [PATCH 22/52] chore(deps): bump redix from 1.8.0 to 1.8.2 (#14786) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 5d7718b7108..cebf9aaf6c8 100644 --- a/mix.lock +++ b/mix.lock @@ -156,7 +156,7 @@ "que": {:hex, :que, "0.12.0", "4629c156036b8a3bf9ffdd79aa4069ab414822144d3ef031b5cc61fc805942d3", [:mix], [{:ex_utils, "~> 0.1", [hex: :ex_utils, repo: "hexpm", optional: false]}, {:memento, "~> 0.6", [hex: :memento, repo: "hexpm", optional: false]}], "hexpm", "906651d60b1036840f1afacc0036541583eda9ba4b0b03f322530a1c84100507"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"}, - "redix": {:hex, :redix, "1.8.0", "f0a2d1dc047a4a1b3004525d27e5df7a4e5b42768246bf161e44bb5e744f2a34", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "24288583e0b08cde42aa781053cd14d992a02976fe06986fb664b8ebfcf0f853"}, + "redix": {:hex, :redix, "1.8.2", "7f0bb263c7c74b62ae5db3922146cbaa5d953993f8d678fce28ab9a24b12cd79", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c6912716ae2f9aeb4f867383fc9b8522317528c6d183daefdc12c9cb7696dbd9"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"}, From b04e7e7f5be4b41c58ce5a3b011e633cf68f3531 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:49:05 +0300 Subject: [PATCH 23/52] chore(deps): bump mint from 1.9.3 to 1.10.0 (#14789) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 40a79271c0d..e96986bb480 100644 --- a/mix.exs +++ b/mix.exs @@ -107,7 +107,7 @@ defmodule BlockScout.Mixfile do {:prometheus_ex, "~> 5.1.0", override: true}, {:absinthe_plug, git: "https://github.com/blockscout/absinthe_plug.git", tag: "1.5.8", override: true}, {:tesla, "~> 1.21.0"}, - {:mint, "~> 1.9.0"}, + {:mint, "~> 1.10.0"}, # Documentation {:ex_doc, "~> 0.40.1", only: :dev, runtime: false}, {:number, "~> 1.0.3"} diff --git a/mix.lock b/mix.lock index cebf9aaf6c8..da440477f75 100644 --- a/mix.lock +++ b/mix.lock @@ -112,7 +112,7 @@ "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mimerl": {:hex, :mimerl, "1.5.0", "f35aca6f23242339b3666e0ac0702379e362b469d0aea167f6cc713547e777ed", [:rebar3], [], "hexpm", "db648ce065bae14ea84ca8b5dd123f42f49417cef693541110bf6f9e9be9ecc4"}, "mimetype_parser": {:hex, :mimetype_parser, "0.1.3", "628ac9fe56aa7edcedb534d68397dd66674ab82493c8ebe39acb9a19b666099d", [:mix], [], "hexpm", "7d8f80c567807ce78cd93c938e7f4b0a20b1aaaaab914bf286f68457d9f7a852"}, - "mint": {:hex, :mint, "1.9.3", "3337184d69179695c7a9f1714d92c11e629d36c8c037a21cf490131d3d150554", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5f7c9342480c069dbbc4eeac3490303c9e01870ff01a7f1d29b6107054fc1e74"}, + "mint": {:hex, :mint, "1.10.0", "85af3353bfc504f5bdfe494bd92b8490f87a306dc659ee1ad0af435107e898dc", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "8b16fb72aaa7531d206a1f05e4cc85509ba531ccec7a17a22736c9c95cbb24d1"}, "mnemoniac": {:hex, :mnemoniac, "0.1.5", "d10bf14abb94adc1a98f6cee4d81d197e1b3986cad8ec43b2e21d2cd9f4d99b6", [:mix], [], "hexpm", "518ae4b6ddda5285a5c02bf83e1eb6ce1af7e4d65d3f7f5bff3270689a00bcc3"}, "mock": {:hex, :mock, "0.3.9", "10e44ad1f5962480c5c9b9fa779c6c63de9bd31997c8e04a853ec990a9d841af", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "9e1b244c4ca2551bb17bb8415eed89e40ee1308e0fbaed0a4fdfe3ec8a4adbd3"}, "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, From 607539648fd592bd2beb27f23f3e0161a30123d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:49:29 +0300 Subject: [PATCH 24/52] chore(deps): bump absinthe from 1.11.0 to 1.12.0 (#14784) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index da440477f75..387c3138a78 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,5 @@ %{ - "absinthe": {:hex, :absinthe, "1.11.0", "dee6dfa57f86d52b25d8b0f925585b6b047d4921ac703c1075b6e7b8bc2fc83c", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "39b3b4b6e3eb405fa98b449feef0dacff81d89bf01c2866cfa513616c5530ba6"}, + "absinthe": {:hex, :absinthe, "1.12.0", "fe69ee9940e47a6a80ab506f2c1def96964ab52e7f52fe2cd642d0f574967931", [:mix], [{:dataloader, "~> 1.0.0 or ~> 2.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b0672939fa75c24b6286c784edc01bec1bedfbe58885acdda5d097c7053b03c6"}, "absinthe_phoenix": {:hex, :absinthe_phoenix, "2.0.5", "1fdcfc59db241b1d5e45e4c8c9db162277e64ec5728e0fd6958a3fa491a05b82", [:mix], [{:absinthe, "~> 1.5", [hex: :absinthe, repo: "hexpm", optional: false]}, {:absinthe_plug, "~> 1.5", [hex: :absinthe_plug, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.5", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.13 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}], "hexpm", "086c6d4a1c32f7444713130d204c87b1b006169f5159026b73f02f7d38ccd05c"}, "absinthe_plug": {:git, "https://github.com/blockscout/absinthe_plug.git", "90a8188e94e2650f13259fb16462075a87f98e18", [tag: "1.5.8"]}, "absinthe_relay": {:hex, :absinthe_relay, "1.6.0", "73590bdb0dcc192622f39fa11da6fca9df4e339b603c45ec7a93493eb94f1829", [:mix], [{:absinthe, ">= 1.7.10", [hex: :absinthe, repo: "hexpm", optional: false]}, {:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "32d6397a7af3fd02678ef9bc8e2f574487f14593cb3e4f9110fb1c695d4d2ac0"}, From 2cbbe2d251041ac3bca4a9b1f38e7026b685ad5b Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 8 Sep 2026 16:07:35 +0300 Subject: [PATCH 25/52] perf: Make Postgrex prepared statements mode configurable per repo (#14799) --- config/runtime/prod.exs | 16 ++++++++++++++-- docker-compose/envs/common-blockscout.env | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/runtime/prod.exs b/config/runtime/prod.exs index 41b59300895..3ce0685efce 100644 --- a/config/runtime/prod.exs +++ b/config/runtime/prod.exs @@ -42,13 +42,24 @@ pool_size = ConfigHelper.parse_integer_env_var("POOL_SIZE", 50) queue_target = ConfigHelper.parse_integer_env_var("DATABASE_QUEUE_TARGET", 50) database_url = ConfigHelper.parse_url_env_var("DATABASE_URL") +# Postgrex prepared statements mode. With `named`, a query is a single round trip to the +# database (the parsed statement is cached per connection); `unnamed` needs two round trips +# per query (parse/describe, then bind/execute) but is required when connections go through a +# transaction-mode pooler (e.g. PgBouncer without `max_prepared_statements`). +prepare_modes = ["named", "unnamed"] +prepare_mode = ConfigHelper.parse_catalog_value("DATABASE_PREPARE_MODE", prepare_modes, true, "unnamed") + +api_prepare_mode = + ConfigHelper.parse_catalog_value("DATABASE_API_PREPARE_MODE", prepare_modes, true, Atom.to_string(prepare_mode)) + # Configures the database config :explorer, Explorer.Repo, [ url: database_url, pool_size: pool_size, - queue_target: queue_target + queue_target: queue_target, + prepare: prepare_mode ] |> Keyword.merge(ExplorerConfigHelper.ssl_options(database_url)) @@ -60,7 +71,8 @@ config :explorer, [ url: api_db_url, pool_size: ConfigHelper.parse_integer_env_var("POOL_SIZE_API", 50), - queue_target: queue_target + queue_target: queue_target, + prepare: api_prepare_mode ] |> Keyword.merge(ExplorerConfigHelper.ssl_options(api_db_url)) diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 8f32a6d2852..68498355654 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -6,6 +6,8 @@ DATABASE_URL=postgresql://blockscout:ceWb1MeLBEeOIfk65gU8EjF8@db:5432/blockscout # DATABASE_EVENT_POOL_SIZE # DATABASE_EVENT_URL= # DATABASE_QUEUE_TARGET +# DATABASE_PREPARE_MODE= +# DATABASE_API_PREPARE_MODE= # TEST_DATABASE_URL= # TEST_DATABASE_READ_ONLY_API_URL= From 1d0325b0776798bff72ffc2817c8d16a61befd24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:32:40 +0300 Subject: [PATCH 26/52] chore(deps-dev): bump ex_doc from 0.40.3 to 0.40.4 (#14787) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.lock b/mix.lock index 387c3138a78..eb4a11bd8b0 100644 --- a/mix.lock +++ b/mix.lock @@ -43,7 +43,7 @@ "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "digital_token": {:hex, :digital_token, "1.0.0", "454a4444061943f7349a51ef74b7fb1ebd19e6a94f43ef711f7dae88c09347df", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "8ed6f5a8c2fa7b07147b9963db506a1b4c7475d9afca6492136535b064c9e9e6"}, "dns": {:hex, :dns, "2.4.0", "44790a0375b28bdc7b59fc894460bfcb03ffeec4c5984e2c3e8b0797b1518327", [:mix], [], "hexpm", "e178e353c469820d02ba889d6a80d01c8c27b47dfcda4016a9cbc6218e3eed64"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "ecto": {:hex, :ecto, "3.13.6", "352135b474f91d1ab99a1b502171d207e9db60421c9e3d0ecab4c7ab96b24d14", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8afa059bc16cd2c94739ec0a11e3e5df69d828125119109bef35f20a21a76af2"}, "ecto_sql": {:hex, :ecto_sql, "3.13.5", "2f8282b2ad97bf0f0d3217ea0a6fff320ead9e2f8770f810141189d182dc304e", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aa36751f4e6a2b56ae79efb0e088042e010ff4935fc8684e74c23b1f49e25fdc"}, "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, @@ -60,7 +60,7 @@ "ex_cldr_lists": {:hex, :ex_cldr_lists, "2.12.2", "731532a3c0a9bfd062acbe79fd39affa4d6e3e5aa5b08a7405524166ce1cd47a", [:mix], [{:ex_cldr_numbers, "~> 2.25", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.18", [hex: :ex_doc, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "16e0b6dc63091c71a0c9a2c50e13db64798f261c3e121d7e3fb7d354872180e8"}, "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.38.3", "64a99531df09397174b7f477785c43016f223d01856725b541a19d36b8149523", [:mix], [{:decimal, "~> 1.6 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 1.0 or ~> 2.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.45", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.17", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "3a0d87ef2747c66d78ae8967023d415d3d76aa310981047ad601e3287e8fe73c"}, "ex_cldr_units": {:hex, :ex_cldr_units, "3.20.5", "5f12206c41b76eb2baadfbc555826fc8e04c400ef6f42d8b1bf0959ab65dc25a", [:mix], [{:decimal, "~> 1.6 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 1.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr_lists, "~> 2.10", [hex: :ex_cldr_lists, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.36", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_doc, "~> 0.18", [hex: :ex_doc, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "51e49f21d2c51127de7358367cc4fbc95eff04593b93ab07b20f12a076db6995"}, - "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, + "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, "ex_eth_bls": {:hex, :ex_eth_bls, "0.1.0", "33c2bf424b360e4b64d7630dd72ec028dac63df56802d0a14ade54a23ad1c743", [:mix], [{:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.8", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "25f6ffc36de4952e55adff1c712f0b9680850773678550f1da970d4d18329365"}, "ex_hash_ring": {:hex, :ex_hash_ring, "6.0.4", "bef9d2d796afbbe25ab5b5a7ed746e06b99c76604f558113c273466d52fa6d6b", [:mix], [], "hexpm", "89adabf31f7d3dfaa36802ce598ce918e9b5b33bae8909ac1a4d052e1e567d18"}, "ex_json_schema": {:hex, :ex_json_schema, "0.11.3", "0f9128fde2a46976c21d1ed9506bf8462776ac9e333e7b42ececcdbf7433a1da", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "0e8e16866bc14339b1bf1e441f1d9231031c8fe2cf45fa5a988a8b685de8c002"}, @@ -102,7 +102,7 @@ "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"}, "logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"}, "logger_json": {:hex, :logger_json, "7.0.4", "e315f2b9a755504658a745f3eab90d88d2cd7ac2ecfd08c8da94d8893965ab5c", [:mix], [{:decimal, ">= 0.0.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d1369f8094e372db45d50672c3b91e8888bcd695fdc444a37a0734e96717c45c"}, - "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "math": {:hex, :math, "0.7.0", "12af548c3892abf939a2e242216c3e7cbfb65b9b2fe0d872d05c6fb609f8127b", [:mix], [], "hexpm", "7987af97a0c6b58ad9db43eb5252a49fc1dfe1f6d98f17da9282e297f594ebc2"}, From 3cdffdf75671568b41ce6e836ec34f08d6252a98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:32:55 +0300 Subject: [PATCH 27/52] chore(deps): bump hammer from 7.4.0 to 7.5.0 (#14785) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index eb4a11bd8b0..0d45756cf7f 100644 --- a/mix.lock +++ b/mix.lock @@ -83,7 +83,7 @@ "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, - "hammer": {:hex, :hammer, "7.4.0", "7ec06643280583b73245d360c6c8797c080ad6cc45788206abc4358eadd70414", [:mix], [], "hexpm", "ae50e0cadd17c68e2379eb8bf06b63bc882a2f9bd6350f8a2c2727c56d082b3d"}, + "hammer": {:hex, :hammer, "7.5.0", "7f9621ebb137f94d1bce60175cf2dd1be26e6e5c40f561124307fdd053fe8693", [:mix], [{:igniter, "~> 0.8", [hex: :igniter, repo: "hexpm", optional: true]}], "hexpm", "cb6fb12e5b5a7f1631d5753e9c45493b225bc11f4f09f11ec7f28140c9f0905f"}, "hammer_backend_redis": {:hex, :hammer_backend_redis, "7.1.1", "979362db9e6f30b9b71f671b28550b5192cd5c2614e60da6ee9347d8085b962b", [:mix], [{:hammer, "~> 7.0", [hex: :hammer, repo: "hexpm", optional: false]}, {:redix, "~> 1.5", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "717bb15f14e709dcae67ba67f90229d2aaee2b6a9bfcc2ca96212f414cd474dc"}, "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, From abeeda3ba0eac636907c96f855a3be33d969bf47 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:43:22 +0400 Subject: [PATCH 28/52] perf: Optimize join_associations (#14775) --- .../api/v2/block_controller_test.exs | 61 +++++ apps/explorer/lib/explorer/chain.ex | 141 +++++++++--- apps/explorer/lib/explorer/query_helper.ex | 70 ++++++ apps/explorer/test/explorer/chain_test.exs | 210 +++++++++++++++++- cspell.json | 3 + 5 files changed, 450 insertions(+), 35 deletions(-) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs index efd87ff59eb..38abc7f5ce7 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/block_controller_test.exs @@ -7,6 +7,8 @@ defmodule BlockScoutWeb.API.V2.BlockControllerTest do alias Explorer.Chain.Beacon.Deposit, as: BeaconDeposit alias Explorer.Chain.Cache.{BlockNumber, Counters.AverageBlockTime} + @page_size 50 + setup do Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id()) @@ -224,6 +226,55 @@ defmodule BlockScoutWeb.API.V2.BlockControllerTest do ] } = json_response(request, 422) end + + # Regression: `select_block_type/1` marks `:nephews` as `:required`, and + # `nephews` is a `has_many through`. Joining a to-many association repeats the + # parent row per child, so the `limit/2` in `Explorer.Chain.fetch_blocks/4` + # used to count joined rows rather than blocks and returned a short page. + test "type=uncle returns a full page when every uncle has a single nephew", %{conn: conn} do + for _ <- 1..(@page_size + 10), do: insert_uncle_with_nephews(1) + + response = json_response(get(conn, "/api/v2/blocks", %{"type" => "uncle"}), 200) + + assert length(response["items"]) == @page_size + refute is_nil(response["next_page_params"]) + end + + test "type=uncle returns a full page when uncles have several nephews", %{conn: conn} do + for _ <- 1..(@page_size + 10), do: insert_uncle_with_nephews(2) + + response = json_response(get(conn, "/api/v2/blocks", %{"type" => "uncle"}), 200) + + assert length(response["items"]) == @page_size + refute is_nil(response["next_page_params"]) + end + + test "type=uncle returns every uncle once regardless of nephew count", %{conn: conn} do + for _ <- 1..10, do: insert_uncle_with_nephews(3) + + response = json_response(get(conn, "/api/v2/blocks", %{"type" => "uncle"}), 200) + + hashes = Enum.map(response["items"], & &1["hash"]) + + assert length(hashes) == 10 + assert hashes == Enum.uniq(hashes) + end + + # The default listing marks the to-many `:transactions` and `:rewards` as + # `:optional`. They must stay preloaded rather than joined, otherwise the + # `limit/2` counts joined rows and the page comes back short — the same + # failure the `type=uncle` cases above cover for `:required`. + test "a full page is returned when blocks carry several transactions", %{conn: conn} do + for _ <- 1..(@page_size + 10) do + block = insert(:block) + for _ <- 1..3, do: :transaction |> insert() |> with_block(block) + end + + response = json_response(get(conn, "/api/v2/blocks"), 200) + + assert length(response["items"]) == @page_size + refute is_nil(response["next_page_params"]) + end end describe "/blocks/{block_number}/countdown" do @@ -1042,6 +1093,16 @@ defmodule BlockScoutWeb.API.V2.BlockControllerTest do end end + defp insert_uncle_with_nephews(nephew_count) do + uncle = insert(:block, consensus: false) + + for index <- 0..(nephew_count - 1) do + insert(:block_second_degree_relation, uncle_hash: uncle.hash, nephew: insert(:block), index: index) + end + + uncle + end + defp compare_item(%Block{} = block, json) do assert to_string(block.hash) == json["hash"] assert block.number == json["height"] diff --git a/apps/explorer/lib/explorer/chain.ex b/apps/explorer/lib/explorer/chain.ex index f52edb9e5f6..48aa757f64c 100644 --- a/apps/explorer/lib/explorer/chain.ex +++ b/apps/explorer/lib/explorer/chain.ex @@ -78,7 +78,7 @@ defmodule Explorer.Chain do alias Explorer.Market.MarketHistoryCache alias Explorer.MicroserviceInterfaces.MultichainSearch - alias Explorer.{PagingOptions, Repo} + alias Explorer.{PagingOptions, QueryHelper, Repo} alias Dataloader.Ecto, as: DataloaderEcto @@ -1506,7 +1506,7 @@ defmodule Explorer.Chain do elements blocks -> - blocks + blocks |> select_repo(options).preload(Map.keys(necessity_by_association)) end end @@ -2267,47 +2267,123 @@ defmodule Explorer.Chain do @doc """ Dynamically joins and preloads associations in a query based on necessity. - This function adjusts the provided Ecto query to include joins for associations. It supports - both optional and required joins. Optional joins use the `preload` function to fetch associations - without enforcing their presence. Required joins ensure the association exists. + `:optional` returns the same rows as a plain preload; `:required` additionally + keeps only the entities that have the association. + + A to-one association is fetched through a join — `LEFT` for `:optional`, + `INNER` for `:required` — and preloaded from it, saving a round trip. A + to-many association cannot be fetched that way: the join repeats the parent + row once per child, and while Ecto collapses the duplicates when assembling + structs, a `limit/2` applied elsewhere counts joined rows rather than + entities, so the page silently comes back short. Those keep the preload path, + with `:required` adding an `INNER JOIN` purely to filter, made row-preserving + by `distinct/2`. + + The preload path is also taken whenever the join would be unsafe or + impossible: the source schema cannot be resolved, the association is a + `through` one or its related schema has no primary key (Ecto needs it to map + joined rows back onto parents), the query already binds the association, or + the spec is not a plain association name — `{name, query}` and `{name, fun}` + specs carry their own loading strategy that a join would drop. + + None of that weakens `:required`: whenever the spec names an association at + all, the filtering `INNER JOIN` is applied on the preload path too, unaliased + so that it cannot collide with a binding the query already has. ## Parameters - `query`: The initial Ecto query. - - `associations`: A single association or a tuple with nested association preloads. + - `preload_spec`: An association name, or a preload spec naming one. - `necessity`: Specifies if the association is `:optional` or `:required`. ## Returns - The modified query with the specified associations joined according to the defined necessity. """ - @spec join_association(atom() | Ecto.Query.t(), [{atom(), atom()}], :optional | :required) :: Ecto.Query.t() - def join_association(query, [{association, nested_preload}], necessity) - when is_atom(association) and is_atom(nested_preload) do - case necessity do - :optional -> - preload(query, [{^association, ^nested_preload}]) - - :required -> - from(q in query, - inner_join: a in assoc(q, ^association), - as: ^association, - left_join: b in assoc(a, ^nested_preload), - as: ^nested_preload, - preload: [{^association, {a, [{^nested_preload, b}]}}] - ) + @spec join_association(atom() | Ecto.Query.t(), term(), :optional | :required) :: Ecto.Query.t() + def join_association(query, preload_spec, necessity) do + with {association, nested_preloads} <- join_target(preload_spec), + true <- joinable?(query, association) do + join_and_preload(query, association, nested_preloads, necessity) + else + _ -> preload_association(query, preload_spec, necessity) end end - @spec join_association(atom() | Ecto.Query.t(), atom(), :optional | :required) :: Ecto.Query.t() - def join_association(query, association, necessity) do - case necessity do - :optional -> - preload(query, ^association) + # `:required` keeps its filtering even when the spec itself cannot be joined. + # The `INNER JOIN` is what drops the entities that do not have the association, + # and the caller depends on that regardless of how the data ends up loaded, so + # it is applied off the association's name alone. The join is left unaliased, + # which is also what makes this path safe for an association the query already + # binds. + defp preload_association(query, preload_spec, :required) do + case spec_association(preload_spec) do + nil -> + preload(query, ^List.wrap(preload_spec)) - :required -> - from(q in query, inner_join: a in assoc(q, ^association), as: ^association, preload: [{^association, a}]) + association -> + query + |> filter_by_association(association) + |> preload(^List.wrap(preload_spec)) end end + defp preload_association(query, preload_spec, :optional), do: preload(query, ^List.wrap(preload_spec)) + + defp join_and_preload(query, association, nested_preloads, :optional) do + from(q in query, + left_join: a in assoc(q, ^association), + as: ^association, + preload: [{^association, {a, ^nested_preloads}}] + ) + end + + defp join_and_preload(query, association, nested_preloads, :required) do + from(q in query, + inner_join: a in assoc(q, ^association), + as: ^association, + preload: [{^association, {a, ^nested_preloads}}] + ) + end + + # An `INNER JOIN` is the only way to express "entities having this association" + # without knowing its keys, but on a to-many association it duplicates the + # parent row; `distinct/2` restores one row per entity so a `limit/2` applied + # elsewhere still counts entities. + defp filter_by_association(query, association) do + if QueryHelper.association_cardinality(query, association) == :one do + from(q in query, inner_join: assoc(q, ^association)) + else + from(q in query, inner_join: assoc(q, ^association), distinct: true) + end + end + + # The association and nested preloads a join could supply, or `nil` when the + # spec has to be loaded as written: `{name, query}` and `{name, fun}` carry + # their own loading strategy that a join would silently drop. + defp join_target(association) when is_atom(association) and not is_nil(association), do: {association, []} + + defp join_target([{association, nested_preloads}]) + when is_atom(association) and (is_atom(nested_preloads) or is_list(nested_preloads)), + do: {association, List.wrap(nested_preloads)} + + defp join_target(_preload_spec), do: nil + + # The association a spec names, whatever shape it takes — including the shapes + # `join_target/1` refuses, since `:required` still has to filter on them. + defp spec_association(association) when is_atom(association) and not is_nil(association), do: association + defp spec_association([{association, _nested_preloads}]) when is_atom(association), do: association + defp spec_association({association, _custom_preload}) when is_atom(association), do: association + defp spec_association(_preload_spec), do: nil + + # Callers compose `join_associations/2` onto hand-written queries that may + # already bind or preload the very same association, and joining it again + # collides on the `as:` alias. Such associations fall back to the preload path, + # where `:required` still gets its filtering from an unaliased join. + defp joinable?(query, association) do + QueryHelper.join_preloadable?(query, association) and + QueryHelper.association_cardinality(query, association) == :one and + not QueryHelper.association_bound?(query, association) + end + @doc """ Applies dynamic joins to a query based on provided association necessities. @@ -2334,12 +2410,9 @@ defmodule Explorer.Chain do join_association(acc_query, association, :required) end) - optional_preloads = Enum.map(optional_associations, fn {association, _join} -> association end) - - case optional_preloads do - [] -> query_with_required_joins - _ -> preload(query_with_required_joins, ^optional_preloads) - end + Enum.reduce(optional_associations, query_with_required_joins, fn {association, _join}, acc_query -> + join_association(acc_query, association, :optional) + end) end def page_blocks(query, %PagingOptions{key: nil}), do: query diff --git a/apps/explorer/lib/explorer/query_helper.ex b/apps/explorer/lib/explorer/query_helper.ex index 8a687e05cf8..01971c6bf27 100644 --- a/apps/explorer/lib/explorer/query_helper.ex +++ b/apps/explorer/lib/explorer/query_helper.ex @@ -98,4 +98,74 @@ defmodule Explorer.QueryHelper do fragment(~s(?."ctid" = ?."ctid"), unquote(first_table_binding), unquote(second_table_binding)) end end + + @doc """ + Returns the cardinality of `association` on the schema `queryable` selects from. + + `:unknown` when the schema cannot be resolved — a query built on a subquery or + a raw source — or when it declares no such association. + """ + @spec association_cardinality(Ecto.Queryable.t(), atom()) :: :one | :many | :unknown + def association_cardinality(queryable, association) do + case association_reflection(queryable, association) do + %{cardinality: cardinality} -> cardinality + _ -> :unknown + end + end + + @doc """ + Whether `association` can be fetched through a join and preloaded from it. + + `Ecto.Repo.Assoc` maps joined rows back onto their parents by the related + schema's primary key and raises `Ecto.NoPrimaryKeyFieldError` without one, and + a `through` association's reflection does not even name a related schema. + Neither can come from a join. + """ + @spec join_preloadable?(Ecto.Queryable.t(), atom()) :: boolean() + def join_preloadable?(queryable, association) do + with %{related: related} <- association_reflection(queryable, association), + schema when not is_nil(schema) <- ecto_schema(related) do + schema.__schema__(:primary_key) != [] + else + _ -> false + end + end + + @doc """ + Whether `query` already carries `association` as a named binding or a + join-preload, in which case joining it again collides on the `as:` alias. + """ + @spec association_bound?(Ecto.Queryable.t(), atom()) :: boolean() + def association_bound?(%Ecto.Query{assocs: assocs} = query, association), + do: has_named_binding?(query, association) or List.keymember?(assocs, association, 0) + + def association_bound?(_queryable, _association), do: false + + @doc """ + Returns the association reflection `association` resolves to on `queryable`, + or `nil` when either the schema or the association cannot be resolved. + """ + @spec association_reflection(Ecto.Queryable.t(), atom()) :: struct() | nil + def association_reflection(queryable, association) do + with schema when not is_nil(schema) <- query_schema(queryable), + %{} = reflection <- schema.__schema__(:association, association) do + reflection + else + _ -> nil + end + end + + @doc """ + Returns the Ecto schema `queryable` selects from, or `nil` when it selects from + a subquery, a raw source, or anything that is not a schema. + """ + @spec query_schema(Ecto.Queryable.t()) :: module() | nil + def query_schema(%Ecto.Query{from: %{source: {_source, schema}}}), do: ecto_schema(schema) + def query_schema(queryable), do: ecto_schema(queryable) + + defp ecto_schema(module) when is_atom(module) and not is_nil(module) do + if Code.ensure_loaded?(module) and function_exported?(module, :__schema__, 2), do: module + end + + defp ecto_schema(_module), do: nil end diff --git a/apps/explorer/test/explorer/chain_test.exs b/apps/explorer/test/explorer/chain_test.exs index 2d8a318b975..425c742b235 100644 --- a/apps/explorer/test/explorer/chain_test.exs +++ b/apps/explorer/test/explorer/chain_test.exs @@ -24,7 +24,7 @@ defmodule Explorer.ChainTest do } alias Explorer.{Chain, Etherscan} - alias Explorer.Chain.Cache.ChainId + alias Explorer.Chain.Cache.{ChainId, Uncles} alias Explorer.Chain.Cache.Counters.{ BlocksCount, @@ -2567,4 +2567,212 @@ defmodule Explorer.ChainTest do Application.put_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth, init_config) end end + + # The uncles cache is warmed by the indexer with its own fixed preload list + # (`:transactions`, `[miner: :names]`, `:rewards`, `:nephews`), which is + # narrower than what a listing asks for. Serving those elements as they are + # would answer a request with associations left unloaded. + describe "list_blocks/1 served from a warm uncles cache" do + setup do + Supervisor.terminate_child(Explorer.Supervisor, Uncles.child_id()) + Supervisor.restart_child(Explorer.Supervisor, Uncles.child_id()) + + :ok + end + + test "loads the requested associations the cache does not preload itself" do + relations = + for _ <- 1..3 do + miner = insert(:address) + insert(:smart_contract, address_hash: miner.hash) + uncle = insert(:block, consensus: false, miner: miner) + insert(:block_second_degree_relation, uncle_hash: uncle.hash, nephew: insert(:block), index: 0) + end + + Uncles.update_from_second_degree_relations(relations) + + blocks = + Chain.list_blocks( + block_type: "Uncle", + necessity_by_association: %{[miner: [:names, :smart_contract]] => :optional}, + paging_options: %PagingOptions{page_size: 2, key: nil} + ) + + assert length(blocks) == 2 + + assert Enum.all?(blocks, fn block -> + not match?(%Ecto.Association.NotLoaded{}, block.miner.smart_contract) and + block.miner.smart_contract.address_hash == block.miner_hash + end) + end + end + + # `join_associations/2` fetches a to-one association through a join and + # everything else through a preload. A join against a to-many association + # repeats the parent row once per child, which makes a `limit/2` count joined + # rows rather than entities, so these cover the shapes where the distinction + # decides whether a page comes back whole. + describe "join_associations/2" do + test "a required to-one association loads its to-many nested preload" do + address = insert(:address) + insert(:address_name, address: address, primary: true, name: "first") + insert(:address_name, address: address, name: "second") + + block = insert(:block) + transaction = :transaction |> insert(from_address: address) |> with_block(block) + + assert {:ok, loaded} = + Chain.hash_to_transaction(transaction.hash, + necessity_by_association: %{[from_address: :names] => :required} + ) + + assert loaded.hash == transaction.hash + assert loaded.from_address.hash == address.hash + assert length(loaded.from_address.names) == 2 + end + + test "a required to-one association returns each entity once and fills a full page" do + block = insert(:block) + + transactions = + for _ <- 1..10 do + address = insert(:address) + insert(:address_name, address: address, primary: true, name: "first") + insert(:address_name, address: address, name: "second") + :transaction |> insert(from_address: address) |> with_block(block) + end + + loaded = + Chain.block_to_transactions(block.hash, + necessity_by_association: %{[from_address: :names] => :required}, + paging_options: %PagingOptions{page_size: 10, key: nil} + ) + + hashes = Enum.map(loaded, & &1.hash) + + assert length(hashes) == 10 + assert hashes == Enum.uniq(hashes) + assert MapSet.new(hashes) == MapSet.new(Enum.map(transactions, & &1.hash)) + assert Enum.all?(loaded, &(length(&1.from_address.names) == 2)) + end + + test "a required association drops the entities that do not have it" do + block = insert(:block) + with_from_address = :transaction |> insert(from_address: insert(:address)) |> with_block(block) + + loaded = + Chain.block_to_transactions(block.hash, + necessity_by_association: %{[from_address: :names] => :required}, + paging_options: %PagingOptions{page_size: 10, key: nil} + ) + + assert Enum.map(loaded, & &1.hash) == [with_from_address.hash] + end + + test "a required to-many association keeps one row per entity" do + uncle = insert(:block, consensus: false) + + for index <- 0..2 do + insert(:block_second_degree_relation, uncle_hash: uncle.hash, nephew: insert(:block), index: index) + end + + loaded = + Chain.list_blocks( + block_type: "Uncle", + necessity_by_association: %{:nephews => :required}, + paging_options: %PagingOptions{page_size: 10, key: nil} + ) + + assert Enum.map(loaded, & &1.hash) == [uncle.hash] + assert length(hd(loaded).nephews) == 3 + end + + test "an optional to-many association fills a full page" do + for _ <- 1..12 do + block = insert(:block) + for _ <- 1..3, do: :transaction |> insert() |> with_block(block) + end + + loaded = + Chain.list_blocks( + necessity_by_association: %{:transactions => :optional, :rewards => :optional}, + paging_options: %PagingOptions{page_size: 10, key: nil} + ) + + hashes = Enum.map(loaded, & &1.hash) + + assert length(hashes) == 10 + assert hashes == Enum.uniq(hashes) + assert Enum.all?(loaded, &(length(&1.transactions) == 3)) + end + + test "a has_one through and a primary-key-less schema still load" do + address = insert(:address) + insert(:address_name, address: address, primary: true, name: "named") + + token_transfer = insert(:token_transfer, from_address: address) + + [loaded] = + Chain.address_hash_to_token_transfers_new(address.hash, + necessity_by_association: %{ + :token => :optional, + [from_address: [:scam_badge, :names]] => :optional + } + ) + + assert loaded.transaction_hash == token_transfer.transaction_hash + assert loaded.token.contract_address_hash == token_transfer.token_contract_address_hash + assert length(loaded.from_address.names) == 1 + assert loaded.from_address.scam_badge == nil + end + + test "a required spec carrying its own query still filters" do + %Transaction{hash: collated_hash} = :transaction |> insert() |> with_block() + %Transaction{hash: pending_hash} = insert(:transaction) + + spec = [block: from(block in Block, select: struct(block, [:hash, :number]))] + + assert {:ok, %Transaction{hash: ^collated_hash} = loaded} = + Chain.hash_to_transaction(collated_hash, necessity_by_association: %{spec => :required}) + + assert loaded.block.number + + assert {:error, :not_found} = + Chain.hash_to_transaction(pending_hash, necessity_by_association: %{spec => :required}) + + assert {:ok, %Transaction{hash: ^pending_hash}} = + Chain.hash_to_transaction(pending_hash, necessity_by_association: %{spec => :optional}) + end + + test "a required association the query already binds filters without an alias collision" do + %Transaction{hash: collated_hash} = :transaction |> insert() |> with_block() + %Transaction{hash: pending_hash} = insert(:transaction) + + loaded = + from(transaction in Transaction, left_join: block in assoc(transaction, :block), as: :block) + |> Chain.join_associations(%{:block => :required}) + |> Repo.all() + + assert Enum.map(loaded, & &1.hash) == [collated_hash] + refute pending_hash in Enum.map(loaded, & &1.hash) + assert hd(loaded).block.number + end + + test "a required through association the query already binds stays on the preload path" do + transaction = :transaction |> insert() |> with_block() + + token_transfer = + insert(:token_transfer, + transaction: transaction, + block: transaction.block, + block_number: transaction.block_number + ) + + [loaded] = + Chain.transaction_to_token_transfers(transaction.hash, necessity_by_association: %{:token => :required}) + + assert loaded.log_index == token_transfer.log_index + assert loaded.token.contract_address_hash == token_transfer.token_contract_address_hash + end + end end diff --git a/cspell.json b/cspell.json index 46f2fbd29d4..6fdccc2ddc9 100644 --- a/cspell.json +++ b/cspell.json @@ -383,6 +383,7 @@ "keccak", "Keepalive", "keyfind", + "keymember", "keyout", "kittencream", "KnxbUejwY", @@ -549,6 +550,7 @@ "predeploy", "prederive", "prederived", + "preloadable", "prewalk", "progressbar", "proxiable", @@ -732,6 +734,7 @@ "tzdata", "ueberauth", "ufixed", + "unaliased", "uncatalog", "unclosable", "unconfigured", From bbba374bef0ece5f9830c906c39e8af37f987801 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 9 Sep 2026 13:59:23 +0300 Subject: [PATCH 29/52] fix: support hexadecimal block number in eth_getBalance (#14804) --- .../api/rpc/eth_controller_test.exs | 26 +++++++++++++++++++ apps/explorer/lib/explorer/eth_rpc.ex | 7 +++++ 2 files changed, 33 insertions(+) diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs index 42e62fd6a5a..9a727307a9a 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/rpc/eth_controller_test.exs @@ -710,6 +710,32 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do assert response["result"] == "0x2" end + test "with a hexadecimal block provided", %{conn: conn, api_params: api_params} do + address = insert(:address) + + insert(:fetched_balance, block_number: 1, address_hash: address.hash, value: 1) + insert(:fetched_balance, block_number: 2, address_hash: address.hash, value: 2) + insert(:fetched_balance, block_number: 3, address_hash: address.hash, value: 3) + + assert response = + conn + |> post("/api/eth-rpc", params(api_params, [to_string(address.hash), "0x2"])) + |> json_response(200) + + assert response["result"] == "0x2" + end + + test "with an invalid hexadecimal block provided", %{conn: conn, api_params: api_params} do + address = insert(:address) + + assert response = + conn + |> post("/api/eth-rpc", params(api_params, [to_string(address.hash), "0xnonsense"])) + |> json_response(200) + + assert response["error"] == "Query parameter 'block' is invalid" + end + test "with a block provided and no balance", %{conn: conn, api_params: api_params} do address = insert(:address) diff --git a/apps/explorer/lib/explorer/eth_rpc.ex b/apps/explorer/lib/explorer/eth_rpc.ex index 55b22253df0..1f5da9b0cb7 100644 --- a/apps/explorer/lib/explorer/eth_rpc.ex +++ b/apps/explorer/lib/explorer/eth_rpc.ex @@ -1334,6 +1334,13 @@ defmodule Explorer.EthRPC do defp block_param("earliest"), do: {:ok, :earliest} defp block_param("pending"), do: {:ok, :pending} + defp block_param("0x" <> hexadecimal_digits) do + case Integer.parse(hexadecimal_digits, 16) do + {integer, ""} -> {:ok, integer} + _ -> :error + end + end + defp block_param(string_integer) when is_bitstring(string_integer) do case Integer.parse(string_integer) do {integer, ""} -> {:ok, integer} From e6788fff21e2c13ee90567608b6b70b22f1c127a Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 9 Sep 2026 14:04:54 +0300 Subject: [PATCH 30/52] fix: Decouple cache propagation to API nodes from block import (#14778) --- apps/explorer/lib/explorer/application.ex | 1 + .../lib/explorer/chain/cache/propagator.ex | 253 ++++++++++++++++++ apps/explorer/lib/explorer/chain/map_cache.ex | 75 ++++-- .../lib/explorer/chain/ordered_cache.ex | 92 ++++--- .../explorer/chain/cache/propagator_test.exs | 167 ++++++++++++ .../chain/cache/transactions_test.exs | 17 ++ apps/indexer/lib/indexer/block/fetcher.ex | 42 ++- config/runtime.exs | 4 + docker-compose/envs/common-blockscout.env | 2 + 9 files changed, 585 insertions(+), 68 deletions(-) create mode 100644 apps/explorer/lib/explorer/chain/cache/propagator.ex create mode 100644 apps/explorer/test/explorer/chain/cache/propagator_test.exs diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index aa74c875cf5..9cc8c1f37ca 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -82,6 +82,7 @@ defmodule Explorer.Application do ), Supervisor.child_spec({Task.Supervisor, name: Explorer.WETHMigratorSupervisor}, id: WETHMigratorSupervisor), {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, + Explorer.Chain.Cache.Propagator, Accounts, AddressesCoinBalanceSum, AddressesCoinBalanceSumMinusBurnt, diff --git a/apps/explorer/lib/explorer/chain/cache/propagator.ex b/apps/explorer/lib/explorer/chain/cache/propagator.ex new file mode 100644 index 00000000000..a18093931be --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/propagator.ex @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.Propagator do + @moduledoc """ + Propagates cache writes from indexer nodes to the other nodes of the cluster. + + `Explorer.Chain.OrderedCache` and `Explorer.Chain.MapCache` write to their local + `ConCache` synchronously and then hand the written data to this process. The + process coalesces pending writes per cache and ships them to `Node.list/0` with + `:erpc.multicast/4` from a short-lived sender process, so that: + + - the local write never depends on the state of a remote node or of the + distribution link. Before this process existed the multicast ran inside the + block import task and, when the distribution buffer to an API node was full, + the VM suspended the task: tens of thousands of tasks piled up holding + preloaded blocks and transactions, and the indexer's own caches stalled; + - at most one batch is in flight at any time and the pending data is bounded. + Ordered caches keep at most `max_size` newest elements, map caches keep only + the set of dirty keys (the current local value is read when the batch is + sent), so a slow cluster only makes API nodes skip intermediate states; + - a sender that stays blocked (e.g. suspended on a busy distribution port) is + killed after `send_timeout` and its batch is dropped. + + Elements are sent exactly as they were written locally, preloads included, so + the receiving nodes never touch the database to apply a propagated write. A + warning `Cache propagation to [...] did not complete` in the logs means the + distribution link cannot keep up with the payload. + + ## Configuration + + - `:flush_interval` - how long (ms) writes are accumulated before a batch is + sent, defaults to 100. A batch is never sent while another one is in flight. + - `:send_timeout` - how long (ms) a sender may take before it is killed and its + batch dropped, defaults to 30 seconds. + """ + + use GenServer + + require Logger + + @default_flush_interval 100 + @default_send_timeout :timer.seconds(30) + + @typep pending_key :: {:ordered, module()} | {:map, module()} + @type pending :: %{optional(pending_key) => [{term(), struct()}] | MapSet.t(atom())} + + ## Client + + @doc """ + Starts the propagator. + + Besides `:name`, `:flush_interval` and `:send_timeout`, the following options + exist for tests: `:nodes_fun` (a zero-arity function returning the nodes to + propagate to, defaults to `Node.list/0`) and `:multicast_fun` (a 4-arity + function with the signature of `:erpc.multicast/4`). + """ + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts \\ []) do + {name, opts} = Keyword.pop(opts, :name, __MODULE__) + gen_server_opts = if name, do: [name: name], else: [] + + GenServer.start_link(__MODULE__, opts, gen_server_opts) + end + + @doc """ + Queues prepared `{id, element}` pairs of an `Explorer.Chain.OrderedCache` for + propagation. Only the `max_size` most prevailing pending elements are kept. + """ + @spec enqueue_ordered(module(), [{term(), struct()}], GenServer.server()) :: :ok + def enqueue_ordered(cache_module, prepared_elements, server \\ __MODULE__) do + GenServer.cast(server, {:ordered, cache_module, prepared_elements}) + end + + @doc """ + Marks keys of an `Explorer.Chain.MapCache` as dirty. Their current local values + are read and propagated when the next batch is sent. + """ + @spec enqueue_map(module(), atom() | [atom()], GenServer.server()) :: :ok + def enqueue_map(cache_module, keys, server \\ __MODULE__) do + GenServer.cast(server, {:map, cache_module, List.wrap(keys)}) + end + + @doc """ + Returns the data queued but not yet sent, for introspection and tests. + """ + @spec pending(GenServer.server()) :: pending() + def pending(server \\ __MODULE__), do: GenServer.call(server, :pending) + + @doc """ + Sends the pending data right away instead of waiting for the flush interval. + Does nothing while a batch is already in flight. + """ + @spec flush(GenServer.server()) :: :ok + def flush(server \\ __MODULE__), do: GenServer.call(server, :flush) + + @doc false + # Merges new prepared elements into the pending ones: newest version of an id + # wins, the result is ordered by `prevails?/2` and capped to `max_size/0`. + @spec coalesce_ordered(module(), [{term(), struct()}], [{term(), struct()}]) :: [{term(), struct()}] + def coalesce_ordered(cache_module, existing, new) do + (new ++ existing) + |> Enum.uniq_by(&elem(&1, 0)) + |> Enum.sort_by(&elem(&1, 0), &cache_module.prevails?/2) + |> Enum.take(cache_module.max_size()) + end + + ## Server + + @impl GenServer + def init(opts) do + config = Application.get_env(:explorer, __MODULE__, []) + + state = %{ + pending: %{}, + in_flight: nil, + flush_timer: nil, + flush_interval: opts[:flush_interval] || config[:flush_interval] || @default_flush_interval, + send_timeout: opts[:send_timeout] || config[:send_timeout] || @default_send_timeout, + nodes_fun: opts[:nodes_fun] || (&Node.list/0), + multicast_fun: opts[:multicast_fun] || (&:erpc.multicast/4) + } + + {:ok, state} + end + + @impl GenServer + def handle_cast({:ordered, cache_module, prepared_elements}, state) do + pending = + Map.update( + state.pending, + {:ordered, cache_module}, + coalesce_ordered(cache_module, [], prepared_elements), + &coalesce_ordered(cache_module, &1, prepared_elements) + ) + + {:noreply, schedule_flush(%{state | pending: pending})} + end + + def handle_cast({:map, cache_module, keys}, state) do + new_keys = MapSet.new(keys) + pending = Map.update(state.pending, {:map, cache_module}, new_keys, &MapSet.union(&1, new_keys)) + + {:noreply, schedule_flush(%{state | pending: pending})} + end + + @impl GenServer + def handle_call(:pending, _from, state), do: {:reply, state.pending, state} + + def handle_call(:flush, _from, state) do + {:reply, :ok, state |> cancel_flush_timer() |> do_flush()} + end + + @impl GenServer + def handle_info(:flush, state) do + {:noreply, do_flush(%{state | flush_timer: nil})} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, %{in_flight: %{ref: ref} = in_flight} = state) do + Process.cancel_timer(in_flight.timer) + log_sender_exit(reason, in_flight) + + {:noreply, schedule_flush(%{state | in_flight: nil})} + end + + def handle_info({:sender_timeout, ref}, %{in_flight: %{ref: ref} = in_flight} = state) do + Logger.warning(fn -> + [ + "Cache propagation to ", + inspect(in_flight.nodes), + " did not complete in ", + to_string(state.send_timeout), + "ms, dropping the batch: ", + inspect(in_flight.summary) + ] + end) + + Process.exit(in_flight.pid, :kill) + + {:noreply, state} + end + + def handle_info(_message, state), do: {:noreply, state} + + ## Internals + + defp schedule_flush(%{pending: pending} = state) when map_size(pending) == 0, do: state + + defp schedule_flush(%{flush_timer: nil} = state) do + %{state | flush_timer: Process.send_after(self(), :flush, state.flush_interval)} + end + + defp schedule_flush(state), do: state + + defp cancel_flush_timer(%{flush_timer: nil} = state), do: state + + defp cancel_flush_timer(%{flush_timer: timer} = state) do + Process.cancel_timer(timer) + %{state | flush_timer: nil} + end + + defp do_flush(%{pending: pending} = state) when map_size(pending) == 0, do: state + + # A batch is being sent: pending data keeps coalescing and the sender's exit + # schedules the next flush. + defp do_flush(%{in_flight: %{}} = state), do: state + + defp do_flush(state) do + case state.nodes_fun.() do + [] -> + # Nothing to propagate to. Nodes joining later start from the database + # fallbacks of their caches and catch up with the following writes. + %{state | pending: %{}} + + nodes -> + batch = state.pending + multicast_fun = state.multicast_fun + + {pid, ref} = spawn_monitor(fn -> send_batch(batch, nodes, multicast_fun) end) + timer = Process.send_after(self(), {:sender_timeout, ref}, state.send_timeout) + + in_flight = %{pid: pid, ref: ref, timer: timer, nodes: nodes, summary: summarize(batch)} + + %{state | pending: %{}, in_flight: in_flight} + end + end + + defp send_batch(batch, nodes, multicast_fun) do + Enum.each(batch, fn + {{:ordered, cache_module}, prepared_elements} -> + multicast_fun.(nodes, cache_module, :do_raw_update, [prepared_elements, false]) + + {{:map, cache_module}, keys} -> + values = cache_module.current_values(MapSet.to_list(keys)) + multicast_fun.(nodes, cache_module, :apply_propagated, [values]) + end) + end + + defp summarize(batch) do + Map.new(batch, fn + {{:ordered, cache_module}, prepared_elements} -> {cache_module, length(prepared_elements)} + {{:map, cache_module}, keys} -> {cache_module, MapSet.to_list(keys)} + end) + end + + defp log_sender_exit(:normal, _in_flight), do: :ok + # already reported by the timeout handler + defp log_sender_exit(:killed, _in_flight), do: :ok + + defp log_sender_exit(reason, in_flight) do + Logger.error(fn -> + ["Cache propagation to ", inspect(in_flight.nodes), " failed: ", inspect(reason)] + end) + end +end diff --git a/apps/explorer/lib/explorer/chain/map_cache.ex b/apps/explorer/lib/explorer/chain/map_cache.ex index 7aa96a894ff..38ab88308c5 100644 --- a/apps/explorer/lib/explorer/chain/map_cache.ex +++ b/apps/explorer/lib/explorer/chain/map_cache.ex @@ -48,9 +48,12 @@ defmodule Explorer.Chain.MapCache do ## Distributed writes - In split API/indexer deployments, `set/2` and `update/2` use `do_raw/2`: the write runs on - the local `ConCache` first, then indexer nodes multicast the same operation to other cluster - nodes via `:erpc` (`propagate: false` on receivers). Reads are not distributed. + In split API/indexer deployments, `set/2` and `update/2` write to the local `ConCache` + first, then indexer nodes mark the key as dirty in `Explorer.Chain.Cache.Propagator`. The + propagator later reads the current local value with `current_values/1` and asynchronously + multicasts it to other cluster nodes via `:erpc`, which store it with `apply_propagated/1`. + The local write never waits for a remote node. Reads are not distributed, and neither is + the `:async_task` key since pids are node-local. """ @type key :: atom() @@ -163,7 +166,8 @@ defmodule Explorer.Chain.MapCache do @impl MapCache def set(key, value) do - do_raw(fn -> ConCache.put(cache_name(), key, value) end) + ConCache.put(cache_name(), key, value) + propagate(key) end @impl MapCache @@ -176,30 +180,63 @@ defmodule Explorer.Chain.MapCache do @impl MapCache def update(key, value) do - do_raw(fn -> ConCache.update(cache_name(), key, fn old_val -> handle_update(key, old_val, value) end) end) + ConCache.update(cache_name(), key, fn old_val -> handle_update(key, old_val, value) end) + propagate(key) end @doc """ - Applies a ConCache write locally, then propagates it from indexer nodes. + Applies a ConCache write locally. - With the default `propagate: true`, the given zero-arity function runs on this node first. - When `Explorer.mode/0` is `:indexer`, the same function is multicast to `Node.list/0` with - `propagate: false` so API nodes apply the write without re-propagating. + With the default `propagate: true`, every key of the cache is then marked dirty in + `Explorer.Chain.Cache.Propagator` on indexer nodes. `propagate: false` is the entry point + used by indexer nodes running a version that still multicasts closures; new versions + propagate values through `apply_propagated/1`. """ def do_raw(function, propagate \\ true) do function.() - case Explorer.mode() do - :indexer -> - if propagate do - Node.list() |> :erpc.multicast(__MODULE__, :do_raw, [function, false]) - else - Logger.error("[#{__MODULE__}] Indexer got unexpected propagation call to do_raw/2") - :ok - end + if propagate do + Enum.each(cache_keys(), &propagate/1) + else + log_unexpected_propagation("do_raw/2") + end + + :ok + end + + @doc """ + Receiving side of the propagation: stores the given `%{key => value}` map as is. + Values come from the indexer node, which is the source of truth for the cache. + """ + def apply_propagated(values) when is_map(values) do + log_unexpected_propagation("apply_propagated/1") + + Enum.each(values, fn {key, value} -> ConCache.put(cache_name(), key, value) end) + end + + @doc """ + Current local values of the given keys, read without triggering `c:handle_fallback/1`. + Used by `Explorer.Chain.Cache.Propagator` when a batch is sent. + """ + def current_values(keys) do + Map.new(keys, fn key -> {key, ConCache.get(cache_name(), key)} end) + end + + # pids are node-local, propagating them makes no sense + defp propagate(:async_task), do: :ok + + defp propagate(key) do + if Explorer.mode() == :indexer do + # credo:disable-for-next-line Credo.Check.Design.AliasUsage + Explorer.Chain.Cache.Propagator.enqueue_map(__MODULE__, key) + end + + :ok + end - _ -> - :ok + defp log_unexpected_propagation(function) do + if Explorer.mode() == :indexer do + Logger.error("[#{__MODULE__}] Indexer got unexpected propagation call to #{function}") end end diff --git a/apps/explorer/lib/explorer/chain/ordered_cache.ex b/apps/explorer/lib/explorer/chain/ordered_cache.ex index ba37da1a2cb..ea3dbd62f9a 100644 --- a/apps/explorer/lib/explorer/chain/ordered_cache.ex +++ b/apps/explorer/lib/explorer/chain/ordered_cache.ex @@ -40,8 +40,11 @@ defmodule Explorer.Chain.OrderedCache do ## Distributed writes In split API/indexer deployments, `update/1` uses `do_raw_update/2`: the ids list and - elements are written to the local `ConCache` first, then indexer nodes multicast the prepared - update to other cluster nodes via `:erpc` (`propagate: false` on receivers). + elements are written to the local `ConCache` first, then indexer nodes hand the prepared + update to `Explorer.Chain.Cache.Propagator`. The propagator coalesces updates and + asynchronously multicasts them to other cluster nodes via `:erpc`, which call + `do_raw_update/2` with `propagate: false` and write locally without any database access. + The local write never waits for a remote node. """ @type element :: struct() @@ -265,27 +268,10 @@ defmodule Explorer.Chain.OrderedCache do def update(elements) when is_list(elements) do case Explorer.mode() do mode when mode in [:all, :api, :indexer] -> - elements_for_preload = - elements - |> Enum.sort_by(&element_to_id(&1), &prevails?(&1, &2)) - |> Enum.take(max_size()) - - preloaded_elements = - try do - do_preloads(elements_for_preload) - rescue - postgrex_error in Postgrex.Error -> - Logger.error(fn -> - [ - "Error while preloading elements for ordered cache: ", - Exception.format(:error, postgrex_error, __STACKTRACE__) - ] - end) - - elements_for_preload - end - - preloaded_elements + elements + |> Enum.sort_by(&element_to_id(&1), &prevails?(&1, &2)) + |> Enum.take(max_size()) + |> do_preloads() |> Enum.map(&{element_to_id(&1), sanitize_before_update(&1)}) |> do_raw_update(true) @@ -297,13 +283,36 @@ defmodule Explorer.Chain.OrderedCache do def update(element), do: update([element]) @doc """ - Merges prepared elements into the local ordered cache, then propagates from indexer nodes. + Merges prepared `{id, element}` pairs into the local ordered cache. + + With `propagate: true` (the writing side) the elements are written locally and, when + `Explorer.mode/0` is `:indexer`, handed to `Explorer.Chain.Cache.Propagator`, which + multicasts them to the other cluster nodes asynchronously. The local write never waits + for a remote node. - Always updates the local ids list and element entries first. When `Explorer.mode/0` is - `:indexer` and `propagate` is `true`, multicasts the same prepared elements to `Node.list/0` - with `propagate: false` so API nodes apply the write without re-propagating. + With `propagate: false` (the receiving side) the elements, already preloaded by the + sender, are written locally without any database access. """ - def do_raw_update(prepared_elements, propagate) do + def do_raw_update(prepared_elements, true) do + write_locally(prepared_elements) + + if Explorer.mode() == :indexer do + # credo:disable-for-next-line Credo.Check.Design.AliasUsage + Explorer.Chain.Cache.Propagator.enqueue_ordered(__MODULE__, prepared_elements) + end + + :ok + end + + def do_raw_update(prepared_elements, false) do + if Explorer.mode() == :indexer do + Logger.error("Indexer got unexpected propagation call to do_raw_update/2") + end + + write_locally(prepared_elements) + end + + defp write_locally(prepared_elements) do ConCache.update(cache_name(), ids_list_key(), fn ids -> updated_list = prepared_elements @@ -312,26 +321,25 @@ defmodule Explorer.Chain.OrderedCache do # ids_list is set to never expire {:ok, %ConCache.Item{value: updated_list, ttl: :infinity}} end) - - case Explorer.mode() do - :indexer -> - if propagate do - Node.list() |> :erpc.multicast(__MODULE__, :do_raw_update, [prepared_elements, false]) - else - Logger.error("Indexer got unexpected propagation call to do_raw_update/2") - :ok - end - - _ -> - :ok - end end defp do_preloads(elements) do if Enum.empty?(preloads()) do elements else - Explorer.Repo.preload(elements, preloads()) + try do + Explorer.Repo.preload(elements, preloads()) + rescue + error in [Postgrex.Error, DBConnection.ConnectionError] -> + Logger.error(fn -> + [ + "Error while preloading elements for ordered cache: ", + Exception.format(:error, error, __STACKTRACE__) + ] + end) + + elements + end end end diff --git a/apps/explorer/test/explorer/chain/cache/propagator_test.exs b/apps/explorer/test/explorer/chain/cache/propagator_test.exs new file mode 100644 index 00000000000..ce7a92d1fb2 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/propagator_test.exs @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.PropagatorTest do + # `Explorer.mode/0` is global state + use ExUnit.Case, async: false + + import ExUnit.CaptureLog + + alias Explorer.Chain.Cache.{Blocks, Propagator, Transactions} + + defmodule TestMapCache do + use Explorer.Chain.MapCache, + name: :propagator_test_map_cache, + keys: [:foo, :bar, :async_task] + + # both return shapes are used so that the type checker does not flag the + # `{:update, _}` branch of `get/1` as unreachable + def handle_fallback(:bar), do: {:update, 0} + def handle_fallback(_key), do: {:return, nil} + end + + @nodes [:api1@test, :api2@test] + + defp start_propagator(opts) do + test_pid = self() + + default_opts = [ + name: nil, + flush_interval: 10, + send_timeout: 5_000, + nodes_fun: fn -> @nodes end, + multicast_fun: fn nodes, module, function, args -> + send(test_pid, {:multicast, self(), nodes, module, function, args}) + :ok + end + ] + + start_supervised!({Propagator, Keyword.merge(default_opts, opts)}) + end + + describe "enqueue_ordered/3" do + test "coalesces pending elements: newest version of an id wins, ordered by prevails?/2, capped to max_size/0" do + pid = start_propagator(flush_interval: :timer.minutes(1)) + + Propagator.enqueue_ordered(Blocks, [{1, :one}, {3, :three}], pid) + Propagator.enqueue_ordered(Blocks, [{2, :two}, {3, :three_v2}], pid) + + assert Propagator.pending(pid) == %{{:ordered, Blocks} => [{3, :three_v2}, {2, :two}, {1, :one}]} + + max_size = Blocks.max_size() + Propagator.enqueue_ordered(Blocks, Enum.map(1..(max_size + 10), &{&1, &1}), pid) + + %{{:ordered, Blocks} => pending} = Propagator.pending(pid) + assert length(pending) == max_size + assert hd(pending) == {max_size + 10, max_size + 10} + end + + test "sends the pending elements to the other nodes with propagate: false" do + pid = start_propagator([]) + + Propagator.enqueue_ordered(Transactions, [{{1, 0}, :transaction}], pid) + + assert_receive {:multicast, _sender, @nodes, Transactions, :do_raw_update, [[{{1, 0}, :transaction}], false]} + assert Propagator.pending(pid) == %{} + end + end + + describe "enqueue_map/3" do + setup do + start_supervised!(TestMapCache) + :ok + end + + test "deduplicates keys and sends their current local values" do + pid = start_propagator([]) + TestMapCache.set(:foo, 1) + + Propagator.enqueue_map(TestMapCache, [:foo, :foo, :bar], pid) + + assert_receive {:multicast, _sender, @nodes, TestMapCache, :apply_propagated, [%{foo: 1, bar: nil}]} + end + + test "apply_propagated/1 stores the received values" do + TestMapCache.apply_propagated(%{foo: 7, bar: 8}) + + assert TestMapCache.get_all() == %{foo: 7, bar: 8, async_task: nil} + end + + test "set/2 and update/2 mark the key dirty on indexer nodes, except for :async_task" do + old_mode = Application.get_env(:explorer, :mode) + Application.put_env(:explorer, :mode, :indexer) + on_exit(fn -> Application.put_env(:explorer, :mode, old_mode) end) + + TestMapCache.set(:foo, 1) + TestMapCache.update(:bar, 2) + TestMapCache.set_async_task(self()) + + assert TestMapCache.get_all() == %{foo: 1, bar: 2, async_task: self()} + assert Propagator.pending()[{:map, TestMapCache}] == MapSet.new([:foo, :bar]) + end + end + + describe "in-flight batches" do + test "at most one batch is in flight, data queued meanwhile is coalesced and sent afterwards" do + test_pid = self() + + blocking_multicast = fn nodes, module, function, args -> + send(test_pid, {:multicast, self(), nodes, module, function, args}) + + receive do + :continue -> :ok + end + end + + pid = start_propagator(multicast_fun: blocking_multicast) + + Propagator.enqueue_ordered(Blocks, [{1, :one}], pid) + assert_receive {:multicast, sender, @nodes, Blocks, :do_raw_update, [[{1, :one}], false]} + + Propagator.enqueue_ordered(Blocks, [{2, :two}], pid) + Propagator.enqueue_ordered(Blocks, [{3, :three}], pid) + Propagator.flush(pid) + + refute_receive {:multicast, _, _, Blocks, _, _}, 50 + assert Propagator.pending(pid) == %{{:ordered, Blocks} => [{3, :three}, {2, :two}]} + + send(sender, :continue) + + assert_receive {:multicast, sender, @nodes, Blocks, :do_raw_update, [[{3, :three}, {2, :two}], false]} + send(sender, :continue) + end + + test "kills a sender exceeding send_timeout, drops its batch and keeps working" do + test_pid = self() + + stuck_multicast = fn nodes, module, function, args -> + send(test_pid, {:multicast, self(), nodes, module, function, args}) + Process.sleep(:infinity) + end + + pid = start_propagator(multicast_fun: stuck_multicast, send_timeout: 50) + + log = + capture_log(fn -> + Propagator.enqueue_ordered(Blocks, [{1, :one}], pid) + + assert_receive {:multicast, sender, @nodes, Blocks, _, _} + ref = Process.monitor(sender) + assert_receive {:DOWN, ^ref, :process, ^sender, :killed}, 1_000 + + Propagator.enqueue_ordered(Blocks, [{2, :two}], pid) + assert_receive {:multicast, _, @nodes, Blocks, :do_raw_update, [[{2, :two}], false]} + end) + + assert log =~ "did not complete" + end + + test "drops the pending data when there are no other nodes" do + pid = start_propagator(nodes_fun: fn -> [] end) + + Propagator.enqueue_ordered(Blocks, [{1, :one}], pid) + Propagator.flush(pid) + + assert Propagator.pending(pid) == %{} + refute_receive {:multicast, _, _, _, _, _}, 50 + end + end +end diff --git a/apps/explorer/test/explorer/chain/cache/transactions_test.exs b/apps/explorer/test/explorer/chain/cache/transactions_test.exs index 0ebd8dfb831..3832e9fc9ec 100644 --- a/apps/explorer/test/explorer/chain/cache/transactions_test.exs +++ b/apps/explorer/test/explorer/chain/cache/transactions_test.exs @@ -112,6 +112,23 @@ defmodule Explorer.Chain.Cache.TransactionsTest do end end + describe "do_raw_update/2 with propagate: false" do + test "writes the received elements as is, without touching the database" do + Application.put_env(:explorer, :mode, :api) + + block = insert(:block) + transaction = insert(:transaction) |> with_block(block) |> preload_all() + prepared = [{Transactions.element_to_id(transaction), transaction}] + + # the rows are gone: the receiving node must not need them + Repo.delete!(transaction) + + Transactions.do_raw_update(prepared, false) + + assert Transactions.take(1) == [transaction] + end + end + defp preload_all(transactions) when is_list(transactions) do Enum.map(transactions, &preload_all(&1)) end diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index 783002787a2..a8326800ec7 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -278,13 +278,19 @@ defmodule Indexer.Block.Fetcher do Prometheus.Instrumenter.set_block_batch_fetch(fetch_time, callback_module) result = {:ok, %{inserted: inserted, errors: blocks_errors}} - Task.Supervisor.start_child(task_supervisor, fn -> - update_block_cache(inserted[:blocks], inserted) - update_transactions_cache(inserted[:transactions], inserted) - update_addresses_cache(inserted[:addresses]) - update_uncles_cache(inserted[:block_second_degree_relations]) - update_withdrawals_cache(inserted[:withdrawals]) - end) + # only what the caches need is captured by the task, not the whole import result + inserted_for_caches = + Map.take(inserted, [ + :blocks, + :transactions, + :block_rewards, + :token_transfers, + :addresses, + :block_second_degree_relations, + :withdrawals + ]) + + Task.Supervisor.start_child(task_supervisor, fn -> update_caches(inserted_for_caches) end) async_match_arbitrum_messages_to_l2(arbitrum_transactions_for_further_handling) @@ -485,6 +491,28 @@ defmodule Indexer.Block.Fetcher do defp enable_partial_async_import?, do: Application.get_env(:indexer, :enable_partial_async_import?) + # Every cache is refreshed independently: a failure while updating one of them + # must not leave the others stale. + defp update_caches(inserted) do + update_cache_safely("blocks", fn -> update_block_cache(inserted[:blocks], inserted) end) + update_cache_safely("transactions", fn -> update_transactions_cache(inserted[:transactions], inserted) end) + update_cache_safely("addresses", fn -> update_addresses_cache(inserted[:addresses]) end) + update_cache_safely("uncles", fn -> update_uncles_cache(inserted[:block_second_degree_relations]) end) + update_cache_safely("withdrawals", fn -> update_withdrawals_cache(inserted[:withdrawals]) end) + end + + defp update_cache_safely(cache_name, fun) do + fun.() + rescue + error -> + Logger.error(fn -> + ["Failed to update ", cache_name, " cache: ", Exception.format(:error, error, __STACKTRACE__)] + end) + catch + :exit, reason -> + Logger.error(fn -> ["Failed to update ", cache_name, " cache: ", inspect(reason)] end) + end + defp update_block_cache([], _), do: :ok defp update_block_cache(blocks, inserted) when is_list(blocks) do diff --git a/config/runtime.exs b/config/runtime.exs index 4c6e9ccb675..0c1217db415 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -679,6 +679,10 @@ config :explorer, Explorer.Chain.Cache.Uncles, ttl_check_interval: false, global_ttl: nil +config :explorer, Explorer.Chain.Cache.Propagator, + flush_interval: ConfigHelper.parse_time_env_var("CACHE_PROPAGATION_FLUSH_INTERVAL", "100ms"), + send_timeout: ConfigHelper.parse_time_env_var("CACHE_PROPAGATION_SEND_TIMEOUT", "30s") + celo_l2_migration_block = ConfigHelper.parse_integer_or_nil_env_var("CELO_L2_MIGRATION_BLOCK") celo_epoch_manager_contract_address = System.get_env("CELO_EPOCH_MANAGER_CONTRACT") diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 68498355654..bed0c609861 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -167,6 +167,8 @@ RELEASE_LINK= # CONTRACT_CERTIFIED_LIST= # CONTRACT_ENABLE_PARTIAL_REVERIFICATION= # CONTRACT_PROXY_EMPTY_IMPLEMENTATION_DATA_CACHE_TTL=1d +# CACHE_PROPAGATION_FLUSH_INTERVAL=100ms +# CACHE_PROPAGATION_SEND_TIMEOUT=30s # UNCLES_IN_AVERAGE_BLOCK_TIME=false # DISABLE_BLOCK_BROADCAST_ENRICHMENT=false # BLOCK_BROADCAST_TYPE=block From cc165474168b943cb30a0abd6eff8afb505c1a89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:44:50 +0300 Subject: [PATCH 31/52] chore(deps): bump oban from 2.24.0 to 2.24.1 (#14791) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 0d45756cf7f..2b0d5c12394 100644 --- a/mix.lock +++ b/mix.lock @@ -126,7 +126,7 @@ "numbers": {:hex, :numbers, "5.2.4", "f123d5bb7f6acc366f8f445e10a32bd403c8469bdbce8ce049e1f0972b607080", [:mix], [{:coerce, "~> 1.0", [hex: :coerce, repo: "hexpm", optional: false]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "eeccf5c61d5f4922198395bf87a465b6f980b8b862dd22d28198c5e6fab38582"}, "nx": {:hex, :nx, "0.12.1", "6e9fee43a77646d04faad2ba4e449b9e270c6b23e413f203cb4d81e71c0a617f", [:mix], [{:complex, "~> 0.7", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee80f6ae898f68bbfe7f30216b06aab10096231ec99ded1208a827256b23d0fb"}, "oauth2": {:hex, :oauth2, "2.1.1", "3bec9bf49e88e1b0c0ddf9f44c393d71fd0f88e905766f2e2cc5d57e6bcc5352", [:mix], [{:tesla, "~> 1.18", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "1d5997cb1ff1643dac17076b6c00c91e4381d8389f3c49a8c390984606dad439"}, - "oban": {:hex, :oban, "2.24.0", "cd877a089ead66658c40864fc616e90c75ddba95559fffae3fccaf05da58a4e5", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab497b2be51191a2d9d7a1b267714463891be711b20487b7b65b5c2269e4b312"}, + "oban": {:hex, :oban, "2.24.1", "2a609c54697ad2c44ba339df30491df2a40eda0758c95b5b879426e4e478bd1f", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef8482472cf198554400b7f8e36a0ffee75c3a64de425d5c7ee625d271925ac7"}, "open_api_spex": {:hex, :open_api_spex, "3.22.4", "00d8f32676b459080a860b81d777a8010e61dfa333f54175dcf51dd6f35a0fac", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "f658d73396e2277e584085a6146481eb085d815a716089defe4f94b26cf8b05f"}, "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, From d2a03c2b29ca19953e4b947a093304c2c13c700b Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:01:08 +0400 Subject: [PATCH 32/52] perf: Deduplicate preloads (#14797) --- .../api/v2/main_page_controller.ex | 43 ++++++++-- .../api/v2/transaction_controller.ex | 84 ++++++++++--------- .../lib/block_scout_web/notifier.ex | 74 ++++++++-------- .../api/v2/transaction_controller_test.exs | 17 ++++ .../chain/address/scam_badge_to_address.ex | 8 +- .../lib/explorer/chain/advanced_filter.ex | 23 ++++- .../lib/explorer/chain/token_transfer.ex | 55 ++++++++---- 7 files changed, 200 insertions(+), 104 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex index b79d0c2f9c3..bde0964e879 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/main_page_controller.ex @@ -27,20 +27,31 @@ defmodule BlockScoutWeb.API.V2.MainPageController do @chain_type_transaction_necessity_by_association %{} end + # Address-info preloads for the `from`/`to`/`created_contract` participants are + # intentionally absent: `preload_transaction_participants/1` loads them for the + # whole page in a single deduplicated pass. @transactions_options [ necessity_by_association: - %{ - :block => :required, - [created_contract_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()]] => - :optional, - [from_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()]] => :optional, - [to_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()]] => :optional - } + %{:block => :required} |> Map.merge(@chain_type_transaction_necessity_by_association), paging_options: %PagingOptions{page_size: 6}, api?: true ] + @transaction_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + + @transaction_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + proxy_implementations_association() => :optional + } + + @api_true [api?: true] + action_fallback(BlockScoutWeb.API.V2.FallbackController) plug(OpenApiSpex.Plug.CastAndValidate, json_render_error_v2: true) @@ -107,10 +118,24 @@ defmodule BlockScoutWeb.API.V2.MainPageController do |> put_status(200) |> put_view(TransactionView) |> render(:transactions, %{ - transactions: recent_transactions |> maybe_preload_ens_and_metadata(:transactions) + transactions: recent_transactions |> preload_transaction_participants() }) end + # Loads the address info of every participant on the page in one pass, + # deduplicating addresses shared between items and between roles of the same + # item. Runs before `maybe_preload_ens_and_metadata/2`, which writes ENS and + # metadata into the very address structs assigned here. + defp preload_transaction_participants(transactions) do + transactions + |> Chain.preload_address_participants( + @transaction_address_fields, + @transaction_participant_necessity_by_association, + @api_true + ) + |> maybe_preload_ens_and_metadata(:transactions) + end + operation :watchlist_transactions, summary: "Last 6 transactions from the current user's watchlist", description: "Retrieves a list of last 6 transactions from the current user's watchlist.", @@ -139,7 +164,7 @@ defmodule BlockScoutWeb.API.V2.MainPageController do |> put_status(200) |> put_view(TransactionView) |> render(:transactions_watchlist, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_participants(), watchlist_names: watchlist_names }) end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index d25fb44e548..b24bfd79559 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -91,8 +91,8 @@ defmodule BlockScoutWeb.API.V2.TransactionController do # Address-info preloads for the transaction participants (from/to/created and # token transfer addresses) are intentionally absent here: the `transaction` # action loads them all in a single deduplicated pass via - # `Chain.preload_transaction_participants/3` using - # `@transaction_participants_necessity_by_association`. + # `Chain.preload_transaction_participants/3`, and the list actions via + # `Chain.preload_address_participants/4`. @transaction_necessity_by_association %{:block => :optional} |> Map.merge(@chain_type_transaction_necessity_by_association) @@ -104,6 +104,23 @@ defmodule BlockScoutWeb.API.V2.TransactionController do proxy_implementations_association() => :optional } + # The same associations for an item of a transaction *list*, where nothing + # renders the contract source, so the smart contract itself is not loaded. + # `:token` stays: `TransactionView.transaction_types/3` reports + # `token_creation` off `created_contract_address.token`. + @transaction_list_participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + :token => :optional, + proxy_implementations_association() => :optional + } + + @transaction_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + @token_transfers_necessity_by_association %{ [token: reputation_association()] => :optional } @@ -255,7 +272,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do full_options = [ - necessity_by_association: transactions_necessity_by_association() + necessity_by_association: @transaction_necessity_by_association ] |> Keyword.merge(paging_options(params, filter_options)) |> Keyword.merge(method_filter_options(params)) @@ -271,38 +288,27 @@ defmodule BlockScoutWeb.API.V2.TransactionController do conn |> put_status(200) |> render(:transactions, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_list_participants(), next_page_params: next_page_params }) end - defp transactions_necessity_by_association do - %{ - :block => :optional, - [ - created_contract_address: [ - :scam_badge, - :names, - :token, - proxy_implementations_association() - ] - ] => :optional, - [ - from_address: [ - :scam_badge, - :names, - proxy_implementations_association() - ] - ] => :optional, - [ - to_address: [ - :scam_badge, - :names, - proxy_implementations_association() - ] - ] => :optional - } - |> Map.merge(@chain_type_transaction_necessity_by_association) + # Loads the address info of every `from`/`to`/`created_contract` participant on + # the page in one pass, deduplicating addresses shared between items and + # between roles of the same item. Preloading it per role through + # `necessity_by_association` instead repeats the `address_names`, scam badge + # and proxy implementation queries once per role. + # + # Runs before `maybe_preload_ens_and_metadata/2`, which writes ENS and metadata + # into the very address structs assigned here. + defp preload_transaction_list_participants(transactions) do + transactions + |> Chain.preload_address_participants( + @transaction_address_fields, + @transaction_list_participant_necessity_by_association, + @api_true + ) + |> maybe_preload_ens_and_metadata(:transactions) end operation :zksync_batch, @@ -504,7 +510,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do end query - |> Chain.join_associations(transactions_necessity_by_association()) + |> Chain.join_associations(@transaction_necessity_by_association) |> preload([{:token_transfers, [:token, :from_address, :to_address]}]) |> Repo.replica().all() end @@ -515,7 +521,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do conn |> put_status(200) |> render(:transactions, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_list_participants(), next_page_params: next_page_params }) end @@ -538,7 +544,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do defp handle_batch_transactions(conn, %{batch_number_param: batch_number} = params, batch_transactions_fun) do full_options = [ - necessity_by_association: transactions_necessity_by_association() + necessity_by_association: @transaction_necessity_by_association ] |> Keyword.merge(paging_options(params)) |> Keyword.merge(@api_true) @@ -559,7 +565,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do conn |> put_status(200) |> render(:transactions, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_list_participants(), next_page_params: next_page_params }) end @@ -591,7 +597,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do def execution_node(conn, %{execution_node_hash_param: execution_node_hash_string} = params) do with {:format, {:ok, execution_node_hash}} <- {:format, Chain.string_to_address_hash(execution_node_hash_string)} do full_options = - [necessity_by_association: transactions_necessity_by_association()] + [necessity_by_association: @transaction_necessity_by_association] |> Keyword.merge(put_key_value_to_paging_options(paging_options(params), :is_index_in_asc_order, true)) |> Keyword.merge(@api_true) @@ -606,7 +612,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do conn |> put_status(200) |> render(:transactions, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_list_participants(), next_page_params: next_page_params }) end @@ -955,7 +961,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do with {:auth, %{watchlist_id: watchlist_id}} <- {:auth, current_user(conn)} do full_options = [ - necessity_by_association: transactions_necessity_by_association() + necessity_by_association: @transaction_necessity_by_association ] |> Keyword.merge(paging_options(params, [:validated])) |> Keyword.merge(@api_true) @@ -970,7 +976,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do conn |> put_status(200) |> render(:transactions_watchlist, %{ - transactions: transactions |> maybe_preload_ens_and_metadata(:transactions), + transactions: transactions |> preload_transaction_list_participants(), next_page_params: next_page_params, watchlist_names: watchlist_names }) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index 2a6e2496349..d6840236b12 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -89,22 +89,26 @@ defmodule BlockScoutWeb.Notifier do @chain_type_transaction_associations [] end - @transaction_associations [ - from_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], - to_address: [ - :scam_badge, - :names, - :smart_contract, - proxy_implementations_association() - ], - created_contract_address: [ - :scam_badge, - :names, - :smart_contract, - proxy_implementations_association() - ] - ] ++ - @chain_type_transaction_associations + # Address-info associations shared by every participant role of a broadcast + # item. Loaded once per broadcast batch by + # `Chain.preload_address_participants/4` rather than per role, which repeats + # each of these queries once per role. + @participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + proxy_implementations_association() => :optional + } + + @transaction_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + + @token_transfer_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] def handle_event({:chain_event, :addresses, type, addresses}) when type in [:realtime, :on_demand] do addresses_count = AddressesCount.fetch() @@ -304,21 +308,10 @@ defmodule BlockScoutWeb.Notifier do |> Repo.preload( DenormalizationHelper.extend_transaction_preload([ [token: Reputation.reputation_association()], - :transaction, - from_address: [ - :scam_badge, - :names, - :smart_contract, - proxy_implementations_association() - ], - to_address: [ - :scam_badge, - :names, - :smart_contract, - proxy_implementations_association() - ] + :transaction ]) ) + |> preload_participants(@token_transfer_address_fields) |> Instance.preload_nft(@api_true) broadcast_token_transfers_websocket_v2(all_token_transfers_full) @@ -796,7 +789,10 @@ defmodule BlockScoutWeb.Notifier do relevant_transactions -> prepared_transactions = TransactionView.render("transactions.json", %{ - transactions: Repo.preload(relevant_transactions, transaction_broadcast_associations()), + transactions: + relevant_transactions + |> Repo.preload(transaction_broadcast_associations()) + |> preload_participants(@transaction_address_fields), conn: nil }) @@ -818,7 +814,8 @@ defmodule BlockScoutWeb.Notifier do if relevant_transactions != [] do relevant_transactions - |> Repo.preload([:block | @transaction_associations]) + |> Repo.preload([:block | @chain_type_transaction_associations]) + |> preload_participants(@transaction_address_fields) |> Enum.map(fn transaction -> Map.put(transaction, :token_transfers, []) end) @@ -828,8 +825,19 @@ defmodule BlockScoutWeb.Notifier do defp transaction_broadcast_associations do if API_V2.enabled?(), - do: [:block, {:token_transfers, [token: Reputation.reputation_association()]} | @transaction_associations], - else: [:block | @transaction_associations] + do: [ + :block, + {:token_transfers, [token: Reputation.reputation_association()]} | @chain_type_transaction_associations + ], + else: [:block | @chain_type_transaction_associations] + end + + # Loads the address info of every participant of the broadcast batch in one + # pass, deduplicating addresses shared between items and between roles of the + # same item. Broadcasts run off the primary repo, as the preloads they replace + # did. + defp preload_participants(items, address_fields) do + Chain.preload_address_participants(items, address_fields, @participant_necessity_by_association, []) end defp broadcast_transaction(%Transaction{block_number: nil} = pending) do diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs index 8c4338e39cf..4710a54773f 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs @@ -44,6 +44,23 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do assert response["next_page_params"] == nil end + # `token_creation` is reported off `created_contract_address.token`, which is + # only there while the participant preload keeps loading `:token`. + test "reports token_creation for a transaction that created a token", %{conn: conn} do + contract_address = insert(:contract_address) + insert(:token, contract_address: contract_address) + + :transaction + |> insert() + |> with_block() + |> with_contract_creation(contract_address) + + request = get(conn, "/api/v2/transactions") + + assert %{"items" => [item]} = json_response(request, 200) + assert "token_creation" in item["transaction_types"] + end + test "transactions with next_page_params", %{conn: conn} do transactions = 51 diff --git a/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex b/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex index 5a9187d01d9..9031be23127 100644 --- a/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex +++ b/apps/explorer/lib/explorer/chain/address/scam_badge_to_address.ex @@ -20,7 +20,13 @@ defmodule Explorer.Chain.Address.ScamBadgeToAddress do """ @primary_key false typed_schema "scam_address_badge_mappings" do - belongs_to(:address, Address, foreign_key: :address_hash, references: :hash, type: Hash.Address, null: false) + belongs_to(:address, Address, + foreign_key: :address_hash, + references: :hash, + type: Hash.Address, + null: false, + primary_key: true + ) timestamps() end diff --git a/apps/explorer/lib/explorer/chain/advanced_filter.ex b/apps/explorer/lib/explorer/chain/advanced_filter.ex index bab9581f309..37a9e9884b5 100644 --- a/apps/explorer/lib/explorer/chain/advanced_filter.ex +++ b/apps/explorer/lib/explorer/chain/advanced_filter.ex @@ -79,6 +79,21 @@ defmodule Explorer.Chain.AdvancedFilter do field(:token_transfer_batch_index, :integer, null: true) end + @address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address}, + {:created_contract_address_hash, :created_contract_address} + ] + + # Address-info associations shared by every participant role of a filter row. + # Loaded once for the whole page by `Chain.preload_address_participants/4` + # rather than per role, which repeats each of these queries three times. + @participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + proxy_implementations_association() => :optional + } + @typep transaction_types :: {:transaction_types, [String.t()] | nil} @typep methods :: {:methods, [String.t()] | nil} @typep age :: {:age, [{:from, DateTime.t() | nil} | {:to, DateTime.t() | nil}] | nil} @@ -153,10 +168,10 @@ defmodule Explorer.Chain.AdvancedFilter do |> Enum.map(&to_advanced_filter/1) |> Enum.sort(&sort_function/2) |> take_page_size(paging_options) - |> Chain.select_repo(options).preload( - from_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], - to_address: [:scam_badge, :names, :smart_contract, proxy_implementations_association()], - created_contract_address: [:names, :smart_contract, proxy_implementations_association()] + |> Chain.preload_address_participants( + @address_fields, + @participant_necessity_by_association, + options ) |> sanitize_fee() |> assign_type() diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index fa1d785f00d..2e34fb99a03 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -158,6 +158,17 @@ defmodule Explorer.Chain.TokenTransfer do @default_paging_options %PagingOptions{page_size: 50} + @participant_address_fields [ + {:from_address_hash, :from_address}, + {:to_address_hash, :to_address} + ] + + @participant_necessity_by_association %{ + :scam_badge => :optional, + :names => :optional, + Implementation.proxy_implementations_association() => :optional + } + @typep paging_options :: {:paging_options, PagingOptions.t()} @typep api? :: {:api?, true | false} @@ -301,23 +312,16 @@ defmodule Explorer.Chain.TokenTransfer do [] _ -> - preloads = - DenormalizationHelper.extend_transaction_preload([ - :transaction, - [token: reputation_association()], - [from_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]], - [to_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]] - ]) - only_consensus_transfers_query() |> where([tt], tt.token_contract_address_hash == ^token_address_hash) |> where([tt], fragment("? @> ARRAY[?::decimal]", tt.token_ids, ^Decimal.new(token_id))) |> where([tt], not is_nil(tt.block_number)) - |> preload(^preloads) + |> preload(^non_address_preloads()) |> order_by([tt], desc: tt.block_number, desc: tt.log_index) |> page_token_transfer(paging_options) |> limit(^paging_options.page_size) |> Chain.select_repo(options).all() + |> preload_participants(options) end end @@ -334,25 +338,40 @@ defmodule Explorer.Chain.TokenTransfer do [] _ -> - preloads = - DenormalizationHelper.extend_transaction_preload([ - :transaction, - [token: reputation_association()], - [from_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]], - [to_address: [:scam_badge, :names, :smart_contract, Implementation.proxy_implementations_association()]] - ]) - only_consensus_transfers_query() - |> preload(^preloads) + |> preload(^non_address_preloads()) |> order_by([tt], desc: tt.block_number, desc: tt.log_index) |> maybe_filter_by_token_type(token_type) |> ExplorerHelper.maybe_hide_scam_addresses_for_token_transfers(options) |> page_token_transfer(paging_options) |> limit(^paging_options.page_size) |> Chain.select_repo(options).all() + |> preload_participants(options) end end + # Everything but the `from`/`to` addresses, which `preload_participants/2` + # loads afterwards. + defp non_address_preloads do + DenormalizationHelper.extend_transaction_preload([ + :transaction, + [token: reputation_association()] + ]) + end + + # Loads the address info of every `from`/`to` participant of the page in one + # pass, deduplicating addresses shared between transfers and between the two + # roles of the same transfer. Preloading it per role instead repeats the + # `addresses` query and every one of these associations twice. + defp preload_participants(token_transfers, options) do + Chain.preload_address_participants( + token_transfers, + @participant_address_fields, + @participant_necessity_by_association, + options + ) + end + @doc """ Conditionally filters token transfers by token type based on denormalization status. From 86724dc6f83f65e2d801b6db9c54a868a50ac60c Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 10 Sep 2026 13:14:37 +0300 Subject: [PATCH 33/52] feat: support OP Stack post-exec transactions (#14734) Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Victor Baranov --- .../views/api/v2/transaction_view.ex | 34 +++++++++- .../block_scout_web/views/transaction_view.ex | 10 +++ apps/block_scout_web/priv/gettext/default.pot | 10 +++ .../priv/gettext/en/LC_MESSAGES/default.po | 10 +++ .../views/api/v2/transaction_view_test.exs | 21 +++++- .../views/transaction_view_test.exs | 18 +++++ .../lib/ethereum_jsonrpc/transaction.ex | 29 ++++++++ .../test/ethereum_jsonrpc/receipt_test.exs | 26 +++++++ .../ethereum_jsonrpc/transaction_test.exs | 68 +++++++++++++++++++ 9 files changed, 224 insertions(+), 2 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex index 14418b66410..f61f9519ef7 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/transaction_view.ex @@ -805,7 +805,39 @@ defmodule BlockScoutWeb.API.V2.TransactionView do | :blob_transaction | :set_code_transaction | :sponsored_transaction - def transaction_types(transaction, types \\ [], stage \\ :sponsored_transaction) + | :op_stack_l1_attributes_transaction + | :op_stack_post_exec_transaction + def transaction_types(transaction, types \\ [], stage \\ :op_stack_l1_attributes_transaction) + + def transaction_types( + %Transaction{type: type, index: index} = transaction, + types, + :op_stack_l1_attributes_transaction + ) do + types = + if chain_type() == :optimism and type == 0x7E and index == 0 do + [:op_stack_l1_attributes_transaction | types] + else + types + end + + transaction_types(transaction, types, :op_stack_post_exec_transaction) + end + + def transaction_types( + %Transaction{type: type} = transaction, + types, + :op_stack_post_exec_transaction + ) do + types = + if chain_type() == :optimism and type == 0x7D do + [:op_stack_post_exec_transaction | types] + else + types + end + + transaction_types(transaction, types, :sponsored_transaction) + end def transaction_types(%Transaction{type: type} = transaction, types, :sponsored_transaction) do # Eden sponsored (batched) transaction type diff --git a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex index 158186b8f00..9d10b638fd4 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/transaction_view.ex @@ -2,6 +2,8 @@ defmodule BlockScoutWeb.TransactionView do use BlockScoutWeb, :view + use Utils.RuntimeEnvHelper, chain_type: [:explorer, :chain_type] + alias BlockScoutWeb.{AccessHelper, AddressView, BlockView, TabHelper} alias BlockScoutWeb.Account.AuthController alias BlockScoutWeb.Cldr.Number @@ -478,6 +480,14 @@ defmodule BlockScoutWeb.TransactionView do def to_address_hash(%Transaction{to_address_hash: address_hash}), do: address_hash def transaction_display_type(%Transaction{} = transaction) do + case {chain_type(), transaction.type, transaction.index} do + {:optimism, 0x7E, 0} -> gettext("L1 attr info tx") + {:optimism, 0x7D, _} -> gettext("Post exec tx") + _ -> default_transaction_display_type(transaction) + end + end + + defp default_transaction_display_type(transaction) do cond do involves_token_transfers?(transaction) -> token_transfer_type = get_transaction_type_from_token_transfers(transaction.token_transfers) diff --git a/apps/block_scout_web/priv/gettext/default.pot b/apps/block_scout_web/priv/gettext/default.pot index 67fde89acea..f7709985ed2 100644 --- a/apps/block_scout_web/priv/gettext/default.pot +++ b/apps/block_scout_web/priv/gettext/default.pot @@ -1635,6 +1635,11 @@ msgstr "" msgid "L1 Gas Used by Transaction" msgstr "" +#: lib/block_scout_web/views/transaction_view.ex:484 +#, elixir-autogen, elixir-format +msgid "L1 attr info tx" +msgstr "" + #: lib/block_scout_web/templates/transaction/overview.html.eex:403 #, elixir-autogen, elixir-format msgid "L2 Gas Limit" @@ -2149,6 +2154,11 @@ msgstr "" msgid "Position %{index}" msgstr "" +#: lib/block_scout_web/views/transaction_view.ex:485 +#, elixir-autogen, elixir-format +msgid "Post exec tx" +msgstr "" + #: lib/block_scout_web/templates/transaction/_decoded_input.html.eex:18 #, elixir-autogen, elixir-format msgid "Potential matches from contract method database:" diff --git a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po index 2221ff8ea4f..aeb47bad4c5 100644 --- a/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po +++ b/apps/block_scout_web/priv/gettext/en/LC_MESSAGES/default.po @@ -1635,6 +1635,11 @@ msgstr "" msgid "L1 Gas Used by Transaction" msgstr "" +#: lib/block_scout_web/views/transaction_view.ex:484 +#, elixir-autogen, elixir-format +msgid "L1 attr info tx" +msgstr "" + #: lib/block_scout_web/templates/transaction/overview.html.eex:403 #, elixir-autogen, elixir-format, fuzzy msgid "L2 Gas Limit" @@ -2149,6 +2154,11 @@ msgstr "" msgid "Position %{index}" msgstr "" +#: lib/block_scout_web/views/transaction_view.ex:485 +#, elixir-autogen, elixir-format +msgid "Post exec tx" +msgstr "" + #: lib/block_scout_web/templates/transaction/_decoded_input.html.eex:18 #, elixir-autogen, elixir-format msgid "Potential matches from contract method database:" diff --git a/apps/block_scout_web/test/block_scout_web/views/api/v2/transaction_view_test.exs b/apps/block_scout_web/test/block_scout_web/views/api/v2/transaction_view_test.exs index 04109c83143..2b60994e1c6 100644 --- a/apps/block_scout_web/test/block_scout_web/views/api/v2/transaction_view_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/api/v2/transaction_view_test.exs @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.API.V2.TransactionViewTest do alias BlockScoutWeb.API.V2.TransactionView alias Explorer.Chain.SmartContract.Proxy.Models.Implementation - alias Explorer.Chain.Transaction + alias Explorer.Chain.{Transaction, Wei} alias Explorer.Market.MarketHistory alias Explorer.Repo @@ -25,6 +25,25 @@ defmodule BlockScoutWeb.API.V2.TransactionViewTest do @tuple_value {"", "", "", "", ""} @tuple_json ["", "", "", "", ""] + describe "OP Stack transaction types" do + test "labels L1 attributes and PostExec transactions only on OP Stack chains" do + value = %Wei{value: Decimal.new(0)} + l1_attributes = build(:transaction, type: 0x7E, index: 0, value: value) + post_exec = build(:transaction, type: 0x7D, index: 1, value: value) + + if Application.get_env(:explorer, :chain_type) == :optimism do + assert :op_stack_l1_attributes_transaction in TransactionView.transaction_types(l1_attributes) + assert :op_stack_post_exec_transaction in TransactionView.transaction_types(post_exec) + else + refute :op_stack_l1_attributes_transaction in TransactionView.transaction_types(l1_attributes) + refute :op_stack_post_exec_transaction in TransactionView.transaction_types(post_exec) + end + + refute :op_stack_l1_attributes_transaction in TransactionView.transaction_types(%{l1_attributes | index: 1}) + refute :op_stack_l1_attributes_transaction in TransactionView.transaction_types(%{l1_attributes | index: nil}) + end + end + test "loads historic exchange rates once for a transaction list", %{conn: conn} do first_date = ~D[2026-07-20] second_date = ~D[2026-07-21] diff --git a/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs b/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs index e1b5a1a10e4..978e31ca392 100644 --- a/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs +++ b/apps/block_scout_web/test/block_scout_web/views/transaction_view_test.exs @@ -8,6 +8,24 @@ defmodule BlockScoutWeb.TransactionViewTest do alias Explorer.Repo alias BlockScoutWeb.{BlockView, TransactionView} + describe "transaction_display_type/1" do + test "labels L1 attributes and PostExec transactions only on OP Stack chains" do + l1_attributes = build(:transaction, type: 0x7E, index: 0) + post_exec = build(:transaction, type: 0x7D, index: 1) + + if Application.get_env(:explorer, :chain_type) == :optimism do + assert TransactionView.transaction_display_type(l1_attributes) == "L1 attr info tx" + assert TransactionView.transaction_display_type(post_exec) == "Post exec tx" + else + refute TransactionView.transaction_display_type(l1_attributes) == "L1 attr info tx" + refute TransactionView.transaction_display_type(post_exec) == "Post exec tx" + end + + refute TransactionView.transaction_display_type(%{l1_attributes | index: 1}) == "L1 attr info tx" + refute TransactionView.transaction_display_type(%{l1_attributes | index: nil}) == "L1 attr info tx" + end + end + describe "block_number/1" do test "returns pending text for pending transaction" do pending = insert(:transaction) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex index 681d801e690..ccd8304b37c 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transaction.ex @@ -30,6 +30,9 @@ defmodule EthereumJSONRPC.Transaction do # EIP-2718 type of the Eden sponsored (batched) transaction: `0x76`. @eden_sponsored_transaction_type 118 + @post_exec_tx_type 0x7D + @zero_address_hash_string "0x0000000000000000000000000000000000000000" + case @chain_type do :ethereum -> @chain_type_fields quote( @@ -571,10 +574,36 @@ defmodule EthereumJSONRPC.Transaction do defp chain_type_normalization(elixir) do case chain_type() do :eden -> eden_compatibility_fields(elixir) + :optimism -> optimism_compatibility_fields(elixir) _ -> elixir end end + # OP Stack post-exec transactions only carry post-execution metadata. Add compatibility values + # for fields required by the standard transaction parser while retaining the block context + # supplied by the execution client. + @spec optimism_compatibility_fields(%{String.t() => any()}) :: %{String.t() => any()} + defp optimism_compatibility_fields(%{"type" => @post_exec_tx_type} = elixir) do + Enum.reduce( + %{ + "from" => @zero_address_hash_string, + "gas" => 0, + "gasPrice" => 0, + "nonce" => 0, + "r" => 0, + "s" => 0, + "v" => 0, + "value" => 0 + }, + elixir, + fn {key, default}, transaction -> + if Map.get(transaction, key), do: transaction, else: Map.put(transaction, key, default) + end + ) + end + + defp optimism_compatibility_fields(elixir), do: elixir + # Eden sponsored transactions (EIP-2718 type `0x76`) carry `gasLimit` and an ordered `calls` # array instead of the standard `gas`, `to`, `input` and `value` fields. The first call provides # the compatibility values for `to` and `input`, while `value` is the sum across all the calls. diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipt_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipt_test.exs index 033f2b4f71a..7ed40b8ddfc 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipt_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/receipt_test.exs @@ -28,4 +28,30 @@ defmodule EthereumJSONRPC.ReceiptTest do "blockNumber" => nil } end + + test "converts an OP Stack PostExec receipt into collated transaction fields" do + receipt = + Receipt.to_elixir(%{ + "blockHash" => "0xab6dfcf5ad132602e939db5f84f56945b1be4e136daab002f9bf9ff0c7f9f7a5", + "blockNumber" => "0x1b", + "contractAddress" => nil, + "cumulativeGasUsed" => "0x5208", + "effectiveGasPrice" => "0x0", + "gasUsed" => "0x0", + "logs" => [], + "logsBloom" => "0x" <> String.duplicate("0", 512), + "status" => "0x1", + "transactionHash" => "0x39ee8fea8d4e719a75a5d64a4d5e34bdbd62f2543c88d00d0f00f91c8f1d8a20", + "transactionIndex" => "0x11", + "type" => "0x7d" + }) + + assert %{ + cumulative_gas_used: 21_000, + gas_used: 0, + status: :ok, + transaction_hash: "0x39ee8fea8d4e719a75a5d64a4d5e34bdbd62f2543c88d00d0f00f91c8f1d8a20", + transaction_index: 17 + } = Receipt.elixir_to_params(receipt) + end end diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/transaction_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/transaction_test.exs index 7c75fa187e3..2e21e6bb5c0 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/transaction_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/transaction_test.exs @@ -13,4 +13,72 @@ defmodule EthereumJSONRPC.TransactionTest do assert %{ignore: :ignore} = Transaction.to_elixir(map) end end + + describe "elixir_to_params/1" do + test "adds compatibility defaults to minimal PostExec transactions only on OP Stack chains" do + transaction = %{ + "blockHash" => "0xab6dfcf5ad132602e939db5f84f56945b1be4e136daab002f9bf9ff0c7f9f7a5", + "blockNumber" => 27, + "gas" => 0, + "hash" => "0x39ee8fea8d4e719a75a5d64a4d5e34bdbd62f2543c88d00d0f00f91c8f1d8a20", + "input" => "0xc3011b80", + "transactionIndex" => 17, + "type" => 0x7D, + "value" => 0 + } + + if Application.get_env(:explorer, :chain_type) == :optimism do + assert Transaction.elixir_to_params(transaction) == %{ + block_hash: "0xab6dfcf5ad132602e939db5f84f56945b1be4e136daab002f9bf9ff0c7f9f7a5", + block_number: 27, + from_address_hash: "0x0000000000000000000000000000000000000000", + gas: 0, + gas_price: 0, + hash: "0x39ee8fea8d4e719a75a5d64a4d5e34bdbd62f2543c88d00d0f00f91c8f1d8a20", + index: 17, + input: "0xc3011b80", + nonce: 0, + r: 0, + s: 0, + to_address_hash: nil, + transaction_index: 17, + type: 0x7D, + v: 0, + value: 0 + } + else + assert_raise FunctionClauseError, fn -> Transaction.elixir_to_params(transaction) end + end + end + + test "preserves standard fields for type 0x7D transactions" do + authorization_list = [%{"address" => "0x0000000000000000000000000000000000000001", "nonce" => 1}] + + assert %{ + created_contract_address_hash: "0x0000000000000000000000000000000000000002", + from_address_hash: "0x0000000000000000000000000000000000000003", + max_fee_per_gas: 30, + max_priority_fee_per_gas: 2, + authorization_list: ^authorization_list, + type: 0x7D + } = + Transaction.elixir_to_params(%{ + "authorizationList" => authorization_list, + "blockHash" => "0xab6dfcf5ad132602e939db5f84f56945b1be4e136daab002f9bf9ff0c7f9f7a5", + "blockNumber" => 27, + "creates" => "0x0000000000000000000000000000000000000002", + "from" => "0x0000000000000000000000000000000000000003", + "gas" => 21_000, + "gasPrice" => 10, + "hash" => "0x39ee8fea8d4e719a75a5d64a4d5e34bdbd62f2543c88d00d0f00f91c8f1d8a20", + "input" => "0x", + "maxFeePerGas" => 30, + "maxPriorityFeePerGas" => 2, + "nonce" => 1, + "transactionIndex" => 17, + "type" => 0x7D, + "value" => 1 + }) + end + end end From e1adb829b388cfd6e9b16d4706fff42d3fc27d20 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:43:08 +0400 Subject: [PATCH 34/52] fix: Group JSON RPC requests by url type (#14806) --- .../lib/ethereum_jsonrpc/http.ex | 95 ++++++++++++++----- .../ethereum_jsonrpc/request_coordinator.ex | 39 +++++--- .../lib/ethereum_jsonrpc/transport.ex | 4 + .../test/ethereum_jsonrpc/http/mox_test.exs | 80 ++++++++++++++++ apps/explorer/lib/explorer/eth_rpc.ex | 30 +++--- apps/explorer/test/explorer/eth_rpc_test.exs | 77 +++++++++++++++ cspell.json | 1 + 7 files changed, 276 insertions(+), 50 deletions(-) create mode 100644 apps/explorer/test/explorer/eth_rpc_test.exs diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex index f169aab08f9..8358bcb8c41 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex @@ -25,7 +25,8 @@ defmodule EthereumJSONRPC.HTTP do def json_rpc(%{method: method} = request, options) when is_map(request) do json = encode_json(request) http = Keyword.fetch!(options, :http) - {url_type, url} = url(options, method) + url_type = url_type(options, method) + url = CommonHelper.get_available_url(options, url_type) http_options = Keyword.fetch!(options, :http_options) with {:ok, %{body: body, status_code: code}} <- http.json_rpc(url, json, headers(), http_options), @@ -41,25 +42,75 @@ defmodule EthereumJSONRPC.HTTP do end def json_rpc([batch | _] = chunked_batch_request, options) when is_list(batch) do - chunked_json_rpc(chunked_batch_request, options, []) + chunked_batch_request + |> Enum.flat_map(&group_by_url_type(&1, options)) + |> chunked_json_rpc(options, []) end def json_rpc(batch_request, options) when is_list(batch_request) do batch_size = Application.get_env(:ethereum_jsonrpc, __MODULE__)[:batch_size] - chunked_batch_request = Enum.chunk_every(batch_request, batch_size) - maybe_log_big_batch(chunked_batch_request) + maybe_log_big_batch(batch_request, batch_size) + + chunked_batch_request = + batch_request + |> group_by_url_type(options) + |> Enum.flat_map(fn {url_type, requests} -> + requests |> Enum.chunk_every(batch_size) |> Enum.map(&{url_type, &1}) + end) + chunked_json_rpc(chunked_batch_request, options, []) end - defp maybe_log_big_batch([]), do: :ok - defp maybe_log_big_batch([_]), do: :ok + # Requests within a single batch can be mapped to different url types (e.g. `eth_call` + # requests are sent to `eth_call_urls`), so the batch has to be split by the url type + # its methods are mapped to. Url types that are configured with the same urls are kept + # together in order to not send redundant requests. + # + # Note that this reorders the requests, so the responses of a batch are not returned in + # the order of the requests and have to be matched by their id (see + # `t:EthereumJSONRPC.Transport.batch_response/0`). + @spec group_by_url_type([Transport.request()], Keyword.t()) :: [{atom(), [Transport.request()]}] + defp group_by_url_type(requests, options) do + requests + |> Enum.group_by(& &1[:method]) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.reduce([], fn {method, method_requests}, acc -> + merge_url_type_requests(url_type(options, method), method_requests, acc, options) + end) + |> Enum.map(fn {_urls, url_type, grouped_requests} -> {url_type, grouped_requests} end) + end + + defp merge_url_type_requests(url_type, requests, acc, options) do + urls = { + CommonHelper.url_type_to_urls(url_type, options), + CommonHelper.url_type_to_urls(url_type, options, :fallback) + } + + case List.keyfind(acc, urls, 0) do + nil -> + acc ++ [{urls, url_type, requests}] - defp maybe_log_big_batch([first_chunk | _] = batch) do - Logger.warning( - "Big amount of node requests in batch: #{batch |> Enum.map(&length/1) |> Enum.sum()}, 1st_chunk_1st_request: #{inspect(List.first(first_chunk))}" - ) + {_urls, existing_url_type, existing_requests} -> + # `:http` is preferred as the representative of a merged group, so that endpoint + # availability is tracked under the type these urls are primarily configured for + merged_url_type = if :http in [existing_url_type, url_type], do: :http, else: existing_url_type + + List.keyreplace(acc, urls, 0, {urls, merged_url_type, existing_requests ++ requests}) + end end + defp maybe_log_big_batch(batch_request, batch_size) do + count = Enum.count(batch_request) + + if count > batch_size do + Logger.warning( + "Big amount of node requests in batch: #{count}, 1st_chunk_1st_request: #{inspect(List.first(batch_request))}" + ) + end + end + + # An empty batch produces no chunks, which matches the JSONRPC 2.0 standard saying that an empty batch (`[]`) returns + # an empty response (`""`): an empty response isn't valid JSON, so instead act like it returns an empty list (`[]`) defp chunked_json_rpc([], _options, decoded_response_bodies) when is_list(decoded_response_bodies) do list = decoded_response_bodies @@ -70,16 +121,10 @@ defmodule EthereumJSONRPC.HTTP do {:ok, list} end - # JSONRPC 2.0 standard says that an empty batch (`[]`) returns an empty response (`""`), but an empty response isn't - # valid JSON, so instead act like it returns an empty list (`[]`) - defp chunked_json_rpc([[] | tail], options, decoded_response_bodies) do - chunked_json_rpc(tail, options, decoded_response_bodies) - end - - defp chunked_json_rpc([[%{method: method} | _] = batch | tail] = chunks, options, decoded_response_bodies) + defp chunked_json_rpc([{url_type, batch} | tail] = chunks, options, decoded_response_bodies) when is_list(tail) and is_list(decoded_response_bodies) do http = Keyword.fetch!(options, :http) - {url_type, url} = url(options, method) + url = CommonHelper.get_available_url(options, url_type) http_options = Keyword.fetch!(options, :http_options) json = encode_json(batch) @@ -110,7 +155,7 @@ defmodule EthereumJSONRPC.HTTP do end end - defp rechunk_json_rpc([batch | tail], options, response, decoded_response_bodies) do + defp rechunk_json_rpc([{url_type, batch} | tail], options, response, decoded_response_bodies) do case length(batch) do # it can't be made any smaller 1 -> @@ -134,7 +179,7 @@ defmodule EthereumJSONRPC.HTTP do batch_size -> split_size = div(batch_size, 2) {first_chunk, second_chunk} = Enum.split(batch, split_size) - new_chunks = [first_chunk, second_chunk | tail] + new_chunks = [{url_type, first_chunk}, {url_type, second_chunk} | tail] chunked_json_rpc(new_chunks, options, decoded_response_bodies) end end @@ -260,17 +305,19 @@ defmodule EthereumJSONRPC.HTTP do end end - defp url(options, method) when is_list(options) and is_binary(method) do + @spec url_type(Keyword.t(), String.t() | nil) :: atom() + defp url_type(options, method) when is_list(options) and is_binary(method) do with {:ok, method_to_url} <- Keyword.fetch(options, :method_to_url), {:ok, method_atom} <- to_existing_atom(method), {:ok, url_type} <- Keyword.fetch(method_to_url, method_atom) do - {url_type, CommonHelper.get_available_url(options, url_type)} + url_type else - _ -> - {:http, CommonHelper.get_available_url(options, :http)} + _ -> :http end end + defp url_type(_options, _method), do: :http + defp to_existing_atom(string) do {:ok, String.to_existing_atom(string)} rescue diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex index aac43018a75..56da6eb3c1d 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/request_coordinator.ex @@ -74,9 +74,9 @@ defmodule EthereumJSONRPC.RequestCoordinator do @spec perform(Transport.batch_request(), Transport.t(), Transport.options(), non_neg_integer()) :: {:ok, Transport.batch_response()} | {:error, term()} def perform(request, transport, transport_options, throttle_timeout) do - request_method = request_method(request) + request_methods = request_methods(request) - sleep_time = sleep_time(request_method) + sleep_time = sleep_time(request_methods) if sleep_time <= throttle_timeout do :timer.sleep(sleep_time) @@ -88,7 +88,7 @@ defmodule EthereumJSONRPC.RequestCoordinator do trace_request(request, fn -> request |> transport.json_rpc(transport_options) - |> handle_transport_response(request_method) + |> handle_transport_response(request_methods) end) :error -> @@ -113,24 +113,29 @@ defmodule EthereumJSONRPC.RequestCoordinator do defp trace_request(_, fun), do: fun.() - defp request_method([request | _]), do: request_method(request) - defp request_method(%{method: method}), do: method - defp request_method(_), do: nil + defp request_methods(requests) when is_list(requests) do + requests + |> Enum.flat_map(&request_methods/1) + |> Enum.uniq() + end + + defp request_methods(%{method: method}), do: [method] + defp request_methods(_), do: [] - defp handle_transport_response({:error, {error_type, _}} = error, method) + defp handle_transport_response({:error, {error_type, _}} = error, methods) when error_type in [:bad_gateway, :bad_response] do - RollingWindow.inc(table(), method_error_key(method)) + inc_methods_error_count(methods) inc_throttle_table() error end - defp handle_transport_response({:error, :timeout} = error, method) do - RollingWindow.inc(table(), method_error_key(method)) + defp handle_transport_response({:error, :timeout} = error, methods) do + inc_methods_error_count(methods) inc_throttle_table() error end - defp handle_transport_response(response, _method) do + defp handle_transport_response(response, _methods) do inc_throttle_table() response end @@ -162,8 +167,16 @@ defmodule EthereumJSONRPC.RequestCoordinator do end end - defp sleep_time(request_method) do - wait_coefficient = RollingWindow.count(table(), method_error_key(request_method)) + defp inc_methods_error_count(methods) do + Enum.each(methods, &RollingWindow.inc(table(), method_error_key(&1))) + end + + defp sleep_time(request_methods) do + wait_coefficient = + Enum.reduce(request_methods, 0, fn request_method, acc -> + max(acc, RollingWindow.count(table(), method_error_key(request_method))) + end) + jitter = :rand.uniform(config!(:max_jitter)) wait_per_timeout = config!(:wait_per_timeout) diff --git a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transport.ex b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transport.ex index 3f8e38b75d5..28357c3b4fe 100644 --- a/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transport.ex +++ b/apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/transport.ex @@ -69,6 +69,10 @@ defmodule EthereumJSONRPC.Transport do @typedoc """ A batch of `t:response/0`. Each `t:response/0` will have an `"id"` corresponding to the `"id"` in the `t:request/0`. + + The order of the responses is **not** guaranteed to match the order of the requests: the + [JSONRPC specification](https://www.jsonrpc.org/specification#batch) allows a server to respond in any order and + `EthereumJSONRPC.HTTP` can split a batch into several requests. Callers must match responses to requests by `"id"`. """ @type batch_response :: [response] diff --git a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs index ed0a2dc39a0..bdf832ab7ef 100644 --- a/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs +++ b/apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs @@ -344,6 +344,86 @@ defmodule EthereumJSONRPC.HTTP.MoxTest do assert log =~ "Big amount of node requests in batch: 10, 1st_chunk_1st_request: %{" end + + test "sends requests of a batch to the urls their methods are mapped to" do + json_rpc_named_arguments = method_to_url_named_arguments(["http://storage.url"], ["http://eth-call.url"]) + + expect(EthereumJSONRPC.HTTP.Mox, :json_rpc, 2, fn url, json, _headers, _options -> + requests = decode_requests(json) + + case url do + "http://eth-call.url" -> assert Enum.map(requests, & &1["method"]) == ["eth_call"] + "http://storage.url" -> assert Enum.map(requests, & &1["method"]) == ["eth_getStorageAt", "eth_getStorageAt"] + end + + {:ok, %{body: encode_results(requests), status_code: 200}} + end) + + assert {:ok, responses} = EthereumJSONRPC.json_rpc(mixed_batch_request(), json_rpc_named_arguments) + assert_results_match_ids(responses) + end + + test "keeps requests in a single batch when their url types are configured with the same urls" do + json_rpc_named_arguments = method_to_url_named_arguments([url()], [url()]) + + expect(EthereumJSONRPC.HTTP.Mox, :json_rpc, 1, fn _url, json, _headers, _options -> + requests = decode_requests(json) + + assert Enum.map(requests, & &1["method"]) == ["eth_call", "eth_getStorageAt", "eth_getStorageAt"] + + {:ok, %{body: encode_results(requests), status_code: 200}} + end) + + assert {:ok, responses} = EthereumJSONRPC.json_rpc(mixed_batch_request(), json_rpc_named_arguments) + assert_results_match_ids(responses) + end + end + + # Batch responses are not returned in the order of the requests, so the only guarantee + # is that every request gets its own result back under its own id + defp assert_results_match_ids(responses) do + assert Map.new(responses, &{&1.id, &1.result}) == %{ + 0 => "0x0", + 1 => "0x1", + 2 => "0x2" + } + end + + defp method_to_url_named_arguments(urls, eth_call_urls) do + [ + transport: EthereumJSONRPC.HTTP, + transport_options: [ + http: EthereumJSONRPC.HTTP.Mox, + urls: urls, + eth_call_urls: eth_call_urls, + fallback_urls: nil, + fallback_eth_call_urls: nil, + method_to_url: [eth_call: :eth_call], + http_options: http_options() + ], + # Which one does not matter, so pick one + variant: EthereumJSONRPC.Nethermind + ] + end + + defp mixed_batch_request do + [ + request(%{id: 0, method: "eth_getStorageAt", params: ["0x0", "0x0", "latest"]}), + request(%{id: 1, method: "eth_call", params: [%{to: "0x0", data: "0x0"}, "latest"]}), + request(%{id: 2, method: "eth_getStorageAt", params: ["0x0", "0x1", "latest"]}) + ] + end + + defp decode_requests(json) do + json |> IO.iodata_to_binary() |> Jason.decode!() + end + + # the result is derived from the request id, so that a response can be traced back to the + # request it belongs to + defp encode_results(requests) do + requests + |> Enum.map(&%{jsonrpc: "2.0", id: &1["id"], result: "0x#{&1["id"]}"}) + |> Jason.encode!() end defp assert_payload_too_large(payload, json_rpc_named_arguments) do diff --git a/apps/explorer/lib/explorer/eth_rpc.ex b/apps/explorer/lib/explorer/eth_rpc.ex index 1f5da9b0cb7..ad222a21ae1 100644 --- a/apps/explorer/lib/explorer/eth_rpc.ex +++ b/apps/explorer/lib/explorer/eth_rpc.ex @@ -659,6 +659,7 @@ defmodule Explorer.EthRPC do } @incorrect_number_of_params "Incorrect number of params." + @no_result "No result" @spec responses([map()]) :: [map()] def responses(requests) do @@ -729,29 +730,32 @@ defmodule Explorer.EthRPC do end) end + # Responses of a batch request are not guaranteed to be returned in the order of the + # requests, so they are matched by id. The index of the request is used as the id sent + # to the node, since the id provided by the user can be duplicated or nil. The user's + # id is attached to the response by `responses/1` anyway. defp json_rpc(map) when is_map(map) do to_request = - Enum.flat_map(Map.values(map), fn - {:error, _} -> + Enum.flat_map(map, fn + {_index, {:error, _}} -> [] - map when is_map(map) -> - [request_to_elixir(map)] + {index, request} when is_map(request) -> + [request |> request_to_elixir() |> Map.put(:id, index)] end) - with [_ | _] = to_request <- to_request, + with [_ | _] <- to_request, {:ok, responses} <- EthereumJSONRPC.json_rpc(to_request, Application.get_env(:explorer, :json_rpc_named_arguments)) do - {map, []} = - Enum.map_reduce(map, responses, fn - {_index, {:error, _}} = elem, responses -> - {elem, responses} + index_to_response = Map.new(responses, &{&1.id, &1}) - {index, _request}, [response | other_responses] -> - {{index, response}, other_responses} - end) + Map.new(map, fn + {_index, {:error, _}} = elem -> + elem - Enum.into(map, %{}) + {index, _request} -> + {index, Map.get(index_to_response, index, {:error, @no_result})} + end) else [] -> map diff --git a/apps/explorer/test/explorer/eth_rpc_test.exs b/apps/explorer/test/explorer/eth_rpc_test.exs new file mode 100644 index 00000000000..52973c170f5 --- /dev/null +++ b/apps/explorer/test/explorer/eth_rpc_test.exs @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.EthRPCTest do + # the tested requests are proxied to the node, so no database is involved + use ExUnit.Case, async: false + + import Mox + + alias Explorer.EthRPC + + setup :verify_on_exit! + setup :set_mox_global + + @address_hash "0x1643E812aE58766192Cf7D2Cf9567dF2C37e9B7F" + + describe "responses/1" do + test "matches responses of a batch to the requests by id, not by their order" do + requests = [ + %{ + "jsonrpc" => "2.0", + "id" => "storage", + "method" => "eth_getStorageAt", + "params" => [@address_hash, "0x0", "latest"] + }, + %{ + "jsonrpc" => "2.0", + "id" => "call", + "method" => "eth_call", + "params" => [%{"to" => @address_hash, "input" => "0xd4aae0c4"}, "latest"] + } + ] + + # the node is free to respond in any order, and `EthereumJSONRPC.HTTP` splits a batch + # by the url type its methods are mapped to, so the responses are reversed here + expect(EthereumJSONRPC.Mox, :json_rpc, fn batch, _options -> + assert [%{method: "eth_getStorageAt"}, %{method: "eth_call"}] = batch + + responses = + Enum.map(batch, fn + %{id: id, method: "eth_getStorageAt"} -> %{jsonrpc: "2.0", id: id, result: "0x123"} + %{id: id, method: "eth_call"} -> %{jsonrpc: "2.0", id: id, result: "0x234"} + end) + + {:ok, Enum.reverse(responses)} + end) + + assert [ + %{id: "storage", result: "0x123"}, + %{id: "call", result: "0x234"} + ] = EthRPC.responses(requests) + end + + test "matches responses of a batch with duplicated user ids" do + request = fn id, storage_pointer -> + %{ + "jsonrpc" => "2.0", + "id" => id, + "method" => "eth_getStorageAt", + "params" => [@address_hash, storage_pointer, "latest"] + } + end + + expect(EthereumJSONRPC.Mox, :json_rpc, fn batch, _options -> + responses = + Enum.map(batch, fn %{id: id, params: [_, storage_pointer, _]} -> + %{jsonrpc: "2.0", id: id, result: storage_pointer} + end) + + {:ok, Enum.reverse(responses)} + end) + + assert [ + %{id: 1, result: "0x0"}, + %{id: 1, result: "0x1"} + ] = EthRPC.responses([request.(1, "0x0"), request.(1, "0x1")]) + end + end +end diff --git a/cspell.json b/cspell.json index 6fdccc2ddc9..e336bd03db6 100644 --- a/cspell.json +++ b/cspell.json @@ -385,6 +385,7 @@ "keyfind", "keymember", "keyout", + "keyreplace", "kittencream", "KnxbUejwY", "labelledby", From 5eef09c7e9cdd3df6002a17a33d7386b639a9b97 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 10 Sep 2026 15:47:15 +0300 Subject: [PATCH 35/52] v11.3.0 --- .github/workflows/generate-swagger.yml | 2 +- .github/workflows/pre-release-arbitrum.yml | 2 +- .github/workflows/pre-release-celo.yml | 2 +- .github/workflows/pre-release-eden.yml | 2 +- .github/workflows/pre-release-eth.yml | 2 +- .github/workflows/pre-release-filecoin.yml | 2 +- .github/workflows/pre-release-fuse.yml | 2 +- .github/workflows/pre-release-gnosis.yml | 2 +- .github/workflows/pre-release-optimism.yml | 2 +- .github/workflows/pre-release-rootstock.yml | 2 +- .github/workflows/pre-release-scroll.yml | 2 +- .github/workflows/pre-release-zilliqa.yml | 2 +- .github/workflows/pre-release-zksync.yml | 2 +- .github/workflows/pre-release.yml | 2 +- .../publish-docker-image-custom-build.yml | 2 +- .../publish-docker-image-every-push.yml | 2 +- .../publish-docker-image-for-arbitrum.yml | 2 +- .../publish-docker-image-for-celo.yml | 2 +- .../publish-docker-image-for-eden.yml | 2 +- .../publish-docker-image-for-eth-sepolia.yml | 2 +- .../publish-docker-image-for-eth.yml | 2 +- .../publish-docker-image-for-filecoin.yml | 2 +- .../publish-docker-image-for-fuse.yml | 2 +- .../publish-docker-image-for-gnosis-chain.yml | 2 +- ...ocker-image-for-optimism-exeperimental.yml | 2 +- .../publish-docker-image-for-optimism.yml | 2 +- .../publish-docker-image-for-rootstock.yml | 2 +- .../publish-docker-image-for-scroll.yml | 2 +- .../publish-docker-image-for-zetachain.yml | 2 +- .../publish-docker-image-for-zilliqa.yml | 2 +- .../publish-docker-image-for-zksync.yml | 2 +- .../workflows/publish-docker-image-old-ui.yml | 2 +- .github/workflows/release-arbitrum.yml | 2 +- .github/workflows/release-celo.yml | 2 +- .github/workflows/release-default.yml | 2 +- .github/workflows/release-eden.yml | 2 +- .github/workflows/release-eth.yml | 2 +- .github/workflows/release-filecoin.yml | 2 +- .github/workflows/release-fuse.yml | 2 +- .github/workflows/release-gnosis.yml | 2 +- .github/workflows/release-optimism.yml | 2 +- .github/workflows/release-rootstock.yml | 2 +- .github/workflows/release-scroll.yml | 2 +- .github/workflows/release-zetachain.yml | 2 +- .github/workflows/release-zilliqa.yml | 2 +- .github/workflows/release-zksync.yml | 2 +- CHANGELOG.md | 33 +++++++++++++++++++ apps/block_scout_web/mix.exs | 2 +- apps/ethereum_jsonrpc/mix.exs | 2 +- .../lib/explorer/token/metadata_retriever.ex | 2 +- apps/explorer/mix.exs | 2 +- apps/indexer/mix.exs | 2 +- apps/nft_media_handler/mix.exs | 2 +- apps/utils/mix.exs | 2 +- docker/Makefile | 2 +- mix.exs | 2 +- rel/config.exs | 2 +- 57 files changed, 89 insertions(+), 56 deletions(-) diff --git a/.github/workflows/generate-swagger.yml b/.github/workflows/generate-swagger.yml index 867b724072c..a606a14d9c9 100644 --- a/.github/workflows/generate-swagger.yml +++ b/.github/workflows/generate-swagger.yml @@ -17,7 +17,7 @@ on: env: OTP_VERSION: '27.3.4.6' ELIXIR_VERSION: '1.19.4' - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 jobs: matrix-builder: diff --git a/.github/workflows/pre-release-arbitrum.yml b/.github/workflows/pre-release-arbitrum.yml index c634c0a9c9e..6a9af42927d 100644 --- a/.github/workflows/pre-release-arbitrum.yml +++ b/.github/workflows/pre-release-arbitrum.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-celo.yml b/.github/workflows/pre-release-celo.yml index 7c367402fb4..1d1ab8ef114 100644 --- a/.github/workflows/pre-release-celo.yml +++ b/.github/workflows/pre-release-celo.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 API_GRAPHQL_MAX_COMPLEXITY: 10400 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/pre-release-eden.yml b/.github/workflows/pre-release-eden.yml index dbe00a95596..bd31991cec1 100644 --- a/.github/workflows/pre-release-eden.yml +++ b/.github/workflows/pre-release-eden.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-eth.yml b/.github/workflows/pre-release-eth.yml index e2dfa5c79b9..1cf3e9dcbad 100644 --- a/.github/workflows/pre-release-eth.yml +++ b/.github/workflows/pre-release-eth.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-filecoin.yml b/.github/workflows/pre-release-filecoin.yml index 87a39303e5f..dda754e3bbc 100644 --- a/.github/workflows/pre-release-filecoin.yml +++ b/.github/workflows/pre-release-filecoin.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-fuse.yml b/.github/workflows/pre-release-fuse.yml index 9c9b9a28599..a54d79fe0bf 100644 --- a/.github/workflows/pre-release-fuse.yml +++ b/.github/workflows/pre-release-fuse.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-gnosis.yml b/.github/workflows/pre-release-gnosis.yml index c79c0c0f405..792aa183b16 100644 --- a/.github/workflows/pre-release-gnosis.yml +++ b/.github/workflows/pre-release-gnosis.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-optimism.yml b/.github/workflows/pre-release-optimism.yml index 3cf6e91c009..547c239a13e 100644 --- a/.github/workflows/pre-release-optimism.yml +++ b/.github/workflows/pre-release-optimism.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-rootstock.yml b/.github/workflows/pre-release-rootstock.yml index aaf92953cbc..a713602d9e0 100644 --- a/.github/workflows/pre-release-rootstock.yml +++ b/.github/workflows/pre-release-rootstock.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-scroll.yml b/.github/workflows/pre-release-scroll.yml index f0dec4d685a..a1e2a40d0f2 100644 --- a/.github/workflows/pre-release-scroll.yml +++ b/.github/workflows/pre-release-scroll.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-zilliqa.yml b/.github/workflows/pre-release-zilliqa.yml index f553925c55f..204364ac65f 100644 --- a/.github/workflows/pre-release-zilliqa.yml +++ b/.github/workflows/pre-release-zilliqa.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-zksync.yml b/.github/workflows/pre-release-zksync.yml index a07fd848e32..ef057c79c82 100644 --- a/.github/workflows/pre-release-zksync.yml +++ b/.github/workflows/pre-release-zksync.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index c88c8e00ed6..770af881b8d 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/publish-docker-image-custom-build.yml b/.github/workflows/publish-docker-image-custom-build.yml index 1b28ab91041..2658e567c01 100644 --- a/.github/workflows/publish-docker-image-custom-build.yml +++ b/.github/workflows/publish-docker-image-custom-build.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/publish-docker-image-every-push.yml b/.github/workflows/publish-docker-image-every-push.yml index afe5d319374..6e92217ef68 100644 --- a/.github/workflows/publish-docker-image-every-push.yml +++ b/.github/workflows/publish-docker-image-every-push.yml @@ -12,7 +12,7 @@ on: env: OTP_VERSION: '27.3.4.6' ELIXIR_VERSION: '1.19.4' - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 permissions: contents: read diff --git a/.github/workflows/publish-docker-image-for-arbitrum.yml b/.github/workflows/publish-docker-image-for-arbitrum.yml index 8aa2cc5d23a..3e67e81020b 100644 --- a/.github/workflows/publish-docker-image-for-arbitrum.yml +++ b/.github/workflows/publish-docker-image-for-arbitrum.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: arbitrum steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-celo.yml b/.github/workflows/publish-docker-image-for-celo.yml index a6d8a19611c..f8e1e4cbb6f 100644 --- a/.github/workflows/publish-docker-image-for-celo.yml +++ b/.github/workflows/publish-docker-image-for-celo.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: celo CHAIN_TYPE: optimism-celo API_GRAPHQL_MAX_COMPLEXITY: 10400 diff --git a/.github/workflows/publish-docker-image-for-eden.yml b/.github/workflows/publish-docker-image-for-eden.yml index ba1c3ed409e..ff16b3f30aa 100644 --- a/.github/workflows/publish-docker-image-for-eden.yml +++ b/.github/workflows/publish-docker-image-for-eden.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: eden steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-eth-sepolia.yml b/.github/workflows/publish-docker-image-for-eth-sepolia.yml index 85558f950e1..1e7a28c5138 100644 --- a/.github/workflows/publish-docker-image-for-eth-sepolia.yml +++ b/.github/workflows/publish-docker-image-for-eth-sepolia.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: eth-sepolia steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-eth.yml b/.github/workflows/publish-docker-image-for-eth.yml index 6188d8cfd06..4084d4db8bf 100644 --- a/.github/workflows/publish-docker-image-for-eth.yml +++ b/.github/workflows/publish-docker-image-for-eth.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: ethereum steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-filecoin.yml b/.github/workflows/publish-docker-image-for-filecoin.yml index 29797dfda88..5df2e635404 100644 --- a/.github/workflows/publish-docker-image-for-filecoin.yml +++ b/.github/workflows/publish-docker-image-for-filecoin.yml @@ -13,7 +13,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: filecoin steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-fuse.yml b/.github/workflows/publish-docker-image-for-fuse.yml index 514ad5f9b50..d4ea1a103cf 100644 --- a/.github/workflows/publish-docker-image-for-fuse.yml +++ b/.github/workflows/publish-docker-image-for-fuse.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: fuse steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-gnosis-chain.yml b/.github/workflows/publish-docker-image-for-gnosis-chain.yml index 3040fd63d46..e18cd5b4588 100644 --- a/.github/workflows/publish-docker-image-for-gnosis-chain.yml +++ b/.github/workflows/publish-docker-image-for-gnosis-chain.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: xdai steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml b/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml index 9c8a78e99e8..d78b6bdbfa6 100644 --- a/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml +++ b/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: optimism steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-optimism.yml b/.github/workflows/publish-docker-image-for-optimism.yml index e47d6fd16c2..723abc91016 100644 --- a/.github/workflows/publish-docker-image-for-optimism.yml +++ b/.github/workflows/publish-docker-image-for-optimism.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: optimism steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-rootstock.yml b/.github/workflows/publish-docker-image-for-rootstock.yml index 2fb971c5301..2d7fc653378 100644 --- a/.github/workflows/publish-docker-image-for-rootstock.yml +++ b/.github/workflows/publish-docker-image-for-rootstock.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: rsk steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-scroll.yml b/.github/workflows/publish-docker-image-for-scroll.yml index fa76dc51acd..5f7ad99671d 100644 --- a/.github/workflows/publish-docker-image-for-scroll.yml +++ b/.github/workflows/publish-docker-image-for-scroll.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: scroll steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zetachain.yml b/.github/workflows/publish-docker-image-for-zetachain.yml index b295199a5c3..cdac7272132 100644 --- a/.github/workflows/publish-docker-image-for-zetachain.yml +++ b/.github/workflows/publish-docker-image-for-zetachain.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: zetachain steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zilliqa.yml b/.github/workflows/publish-docker-image-for-zilliqa.yml index baaa69fa14a..b2fa1c14524 100644 --- a/.github/workflows/publish-docker-image-for-zilliqa.yml +++ b/.github/workflows/publish-docker-image-for-zilliqa.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: zilliqa steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zksync.yml b/.github/workflows/publish-docker-image-for-zksync.yml index c92d2705adc..6b6b03498a6 100644 --- a/.github/workflows/publish-docker-image-for-zksync.yml +++ b/.github/workflows/publish-docker-image-for-zksync.yml @@ -13,7 +13,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 DOCKER_CHAIN_NAME: zksync steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-old-ui.yml b/.github/workflows/publish-docker-image-old-ui.yml index 4287847298a..936716fe496 100644 --- a/.github/workflows/publish-docker-image-old-ui.yml +++ b/.github/workflows/publish-docker-image-old-ui.yml @@ -16,7 +16,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-arbitrum.yml b/.github/workflows/release-arbitrum.yml index 972c1e5342d..36a47383139 100644 --- a/.github/workflows/release-arbitrum.yml +++ b/.github/workflows/release-arbitrum.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-celo.yml b/.github/workflows/release-celo.yml index 09b5f1d6064..f1ec3b3e261 100644 --- a/.github/workflows/release-celo.yml +++ b/.github/workflows/release-celo.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 API_GRAPHQL_MAX_COMPLEXITY: 10400 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/release-default.yml b/.github/workflows/release-default.yml index e3ba72ae5e4..d19d5cd9b19 100644 --- a/.github/workflows/release-default.yml +++ b/.github/workflows/release-default.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-eden.yml b/.github/workflows/release-eden.yml index 702fba733ca..ee832c06205 100644 --- a/.github/workflows/release-eden.yml +++ b/.github/workflows/release-eden.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-eth.yml b/.github/workflows/release-eth.yml index 86e0d5fff43..f7678ab9963 100644 --- a/.github/workflows/release-eth.yml +++ b/.github/workflows/release-eth.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-filecoin.yml b/.github/workflows/release-filecoin.yml index c8e8d092144..0839eb590a4 100644 --- a/.github/workflows/release-filecoin.yml +++ b/.github/workflows/release-filecoin.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-fuse.yml b/.github/workflows/release-fuse.yml index 0dc36bba601..c5077ed3a0e 100644 --- a/.github/workflows/release-fuse.yml +++ b/.github/workflows/release-fuse.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-gnosis.yml b/.github/workflows/release-gnosis.yml index e6b993fd217..161e81b2f44 100644 --- a/.github/workflows/release-gnosis.yml +++ b/.github/workflows/release-gnosis.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-optimism.yml b/.github/workflows/release-optimism.yml index 5950116ce3b..44dabf542f2 100644 --- a/.github/workflows/release-optimism.yml +++ b/.github/workflows/release-optimism.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-rootstock.yml b/.github/workflows/release-rootstock.yml index 3ea8fc688b9..0497c1239b8 100644 --- a/.github/workflows/release-rootstock.yml +++ b/.github/workflows/release-rootstock.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-scroll.yml b/.github/workflows/release-scroll.yml index e45e92b1b15..76d67bca855 100644 --- a/.github/workflows/release-scroll.yml +++ b/.github/workflows/release-scroll.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zetachain.yml b/.github/workflows/release-zetachain.yml index 5af2996935d..fca42686e85 100644 --- a/.github/workflows/release-zetachain.yml +++ b/.github/workflows/release-zetachain.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zilliqa.yml b/.github/workflows/release-zilliqa.yml index 8680c3469a6..caef5f2b4f6 100644 --- a/.github/workflows/release-zilliqa.yml +++ b/.github/workflows/release-zilliqa.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zksync.yml b/.github/workflows/release-zksync.yml index 0d4166997ac..f375ba5e254 100644 --- a/.github/workflows/release-zksync.yml +++ b/.github/workflows/release-zksync.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.2.8 + RELEASE_VERSION: 11.3.0 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/CHANGELOG.md b/CHANGELOG.md index ff58a4687cc..7156055dfd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## 11.3.0 + +### 🚀 Features + +- Support OP Stack post-exec transactions ([#14734](https://github.com/blockscout/blockscout/issues/14734)) +- Support API key for CryptoCompare market source ([#14746](https://github.com/blockscout/blockscout/issues/14746)) + +### 🐛 Bug Fixes + +- Group JSON RPC requests by url type ([#14806](https://github.com/blockscout/blockscout/issues/14806)) +- Decouple cache propagation to API nodes from block import ([#14778](https://github.com/blockscout/blockscout/issues/14778)) +- Support hexadecimal block number in eth_getBalance ([#14804](https://github.com/blockscout/blockscout/issues/14804)) +- Disable old broadcast for token transfers without subscribers ([#14777](https://github.com/blockscout/blockscout/issues/14777)) +- Forward [api?: true] where missed ([#14740](https://github.com/blockscout/blockscout/issues/14740)) + +### ⚡ Performance + +- Deduplicate preloads ([#14797](https://github.com/blockscout/blockscout/issues/14797)) +- Optimize join_associations ([#14775](https://github.com/blockscout/blockscout/issues/14775)) +- Make Postgrex prepared statements mode configurable per repo ([#14799](https://github.com/blockscout/blockscout/issues/14799)) +- Limit logs before joining transactions in topic-only getLogs ([#14780](https://github.com/blockscout/blockscout/issues/14780)) +- Scam addresses ETS cache ([#14781](https://github.com/blockscout/blockscout/issues/14781)) +- Preload only rendered transaction fields in v1 tokentx endpoints ([#14782](https://github.com/blockscout/blockscout/issues/14782)) +- Switch token counters to incremental consolidation ([#14773](https://github.com/blockscout/blockscout/issues/14773)) +- Switch address counters to incremental consolidation ([#14759](https://github.com/blockscout/blockscout/issues/14759)) +- Batch uncataloged token transfers scan via Migrator framework ([#14774](https://github.com/blockscout/blockscout/issues/14774)) +- Remove unused GasUsageSum cache with heavy DB query ([#14771](https://github.com/blockscout/blockscout/issues/14771)) +- Deduplicate addresses preloads ([#14758](https://github.com/blockscout/blockscout/issues/14758)) +- Reduce addresses preload queries count ([#14755](https://github.com/blockscout/blockscout/issues/14755)) +- Remove redundant preloads in api/v2/addresses/* ([#14751](https://github.com/blockscout/blockscout/issues/14751)) +- Remove unconditional contract_address preload for tokens ([#14750](https://github.com/blockscout/blockscout/issues/14750)) + + ## 11.2.8 ### 🚀 Features diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index 02eb9583afc..8e84ff28c10 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -20,7 +20,7 @@ defmodule BlockScoutWeb.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.2.8", + version: "11.3.0", xref: [ exclude: [ Explorer.Chain.Beacon.Reader, diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 9012839b1bf..9408864f38d 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -20,7 +20,7 @@ defmodule EthereumJSONRPC.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.2.8" + version: "11.3.0" ] end diff --git a/apps/explorer/lib/explorer/token/metadata_retriever.ex b/apps/explorer/lib/explorer/token/metadata_retriever.ex index 41cc04a1811..4f1a250fb5c 100644 --- a/apps/explorer/lib/explorer/token/metadata_retriever.ex +++ b/apps/explorer/lib/explorer/token/metadata_retriever.ex @@ -15,7 +15,7 @@ defmodule Explorer.Token.MetadataRetriever do @vm_execution_error "VM execution error" @invalid_base64_data "invalid data:application/json;base64" @invalid_ipfs_path "invalid ipfs path" - @default_headers [{"User-Agent", "blockscout-11.2.8"}] + @default_headers [{"User-Agent", "blockscout-11.3.0"}] # https://eips.ethereum.org/EIPS/eip-1155#metadata @erc1155_token_id_placeholder "{id}" diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index f29eebbf3e9..0d73ae29c1e 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -21,7 +21,7 @@ defmodule Explorer.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.2.8", + version: "11.3.0", xref: [exclude: [BlockScoutWeb.Routers.WebRouter.Helpers, Indexer.Helper, Indexer.Fetcher.InternalTransaction]] ] end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index dcebfe8e14c..9d7b8454939 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -15,7 +15,7 @@ defmodule Indexer.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.2.8", + version: "11.3.0", xref: [ exclude: [ Explorer.Chain.Optimism.Deposit, diff --git a/apps/nft_media_handler/mix.exs b/apps/nft_media_handler/mix.exs index 4f555517dfb..2ca0a375f6c 100644 --- a/apps/nft_media_handler/mix.exs +++ b/apps/nft_media_handler/mix.exs @@ -5,7 +5,7 @@ defmodule NFTMediaHandler.MixProject do def project do [ app: :nft_media_handler, - version: "11.2.8", + version: "11.3.0", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/utils/mix.exs b/apps/utils/mix.exs index 28791ffd11f..35bc091e48c 100644 --- a/apps/utils/mix.exs +++ b/apps/utils/mix.exs @@ -5,7 +5,7 @@ defmodule Utils.MixProject do def project do [ app: :utils, - version: "11.2.8", + version: "11.3.0", build_path: "../../_build", # config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/docker/Makefile b/docker/Makefile index fb2c95989c8..01c60d54b12 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,7 +10,7 @@ STATS_CONTAINER_NAME := stats STATS_DB_CONTAINER_NAME := stats-db PROXY_CONTAINER_NAME := proxy PG_CONTAINER_NAME := postgres -RELEASE_VERSION ?= '11.2.8' +RELEASE_VERSION ?= '11.3.0' TAG := $(RELEASE_VERSION)-commit-$(shell git log -1 --pretty=format:"%h") STABLE_TAG := $(RELEASE_VERSION) diff --git a/mix.exs b/mix.exs index e96986bb480..628ab10b711 100644 --- a/mix.exs +++ b/mix.exs @@ -8,7 +8,7 @@ defmodule BlockScout.Mixfile do [ # app: :block_scout, # aliases: aliases(config_env()), - version: "11.2.8", + version: "11.3.0", apps_path: "apps", deps: deps(), dialyzer: dialyzer(), diff --git a/rel/config.exs b/rel/config.exs index 58d2d5fe9c9..5446ef721ed 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -72,7 +72,7 @@ end # will be used by default release :blockscout do - set version: "11.2.8" + set version: "11.3.0" set applications: [ :runtime_tools, block_scout_web: :permanent, From 43af7ea84797e2f3a55ac1191d9cbe67436eb3e8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Thu, 10 Sep 2026 17:00:00 +0300 Subject: [PATCH 36/52] Update CHANGELOG --- CHANGELOG.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7156055dfd5..15dfe241e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,58 @@ - Remove redundant preloads in api/v2/addresses/* ([#14751](https://github.com/blockscout/blockscout/issues/14751)) - Remove unconditional contract_address preload for tokens ([#14750](https://github.com/blockscout/blockscout/issues/14750)) +### New ENV variables + +| Variable | Description | Parameters | +|-----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------| +| `DATABASE_PREPARE_MODE` | Postgrex prepared statements mode for the main database connection. `named` uses one round trip per query and caches parsed statements per connection; `unnamed` is required behind a transaction-mode pooler without prepared statements support. | Version: v11.3.0\+
Default: `unnamed`
Applications: API, Indexer | +| `DATABASE_API_PREPARE_MODE` | Same as `DATABASE_PREPARE_MODE`, but for the API read-only database connection (`DATABASE_READ_ONLY_API_URL`). | Version: v11.3.0\+
Default: value of `DATABASE_PREPARE_MODE`
Applications: API | +| `MIGRATION_BACKFILL_ADDRESS_COUNTERS_DISABLED` | If `true`, the backfill migration that recomputes address counters from scratch is disabled. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `false`
Applications: Indexer | +| `MIGRATION_BACKFILL_ADDRESS_COUNTERS_BATCH_SIZE` | Number of addresses to process in each batch during address counters backfill migration. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `10`
Applications: Indexer | +| `MIGRATION_BACKFILL_ADDRESS_COUNTERS_CONCURRENCY` | Number of parallel processes for address counters backfill migration. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `2`
Applications: Indexer | +| `MIGRATION_BACKFILL_ADDRESS_COUNTERS_TIMEOUT` | Timeout between address counters backfill batches processing. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `0s`
Applications: Indexer | +| `MIGRATION_BACKFILL_TOKEN_COUNTERS_DISABLED` | If `true`, the backfill migration that recalculates token counters from scratch is disabled. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `false`
Applications: Indexer | +| `MIGRATION_BACKFILL_TOKEN_COUNTERS_BATCH_SIZE` | Number of tokens to process in each batch during token counters backfill migration. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `50`
Applications: Indexer | +| `MIGRATION_BACKFILL_TOKEN_COUNTERS_CONCURRENCY` | Number of parallel processes for token counters backfill migration. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `4`
Applications: Indexer | +| `MIGRATION_BACKFILL_TOKEN_COUNTERS_TIMEOUT` | Timeout between token counters backfill batches processing. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `0s`
Applications: Indexer | +| `MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_BATCH_SIZE` | Number of blocks to reindex in the batch. Implemented in [#14774](https://github.com/blockscout/blockscout/pull/14774). | Version: v11.3.0\+
Default: `1000`
Applications: Indexer | +| `MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_CONCURRENCY` | Number of parallel reindexing block batches processing. Implemented in [#14774](https://github.com/blockscout/blockscout/pull/14774). | Version: v11.3.0\+
Default: `1`
Applications: Indexer | +| `MIGRATION_REINDEX_BLOCKS_WITH_UNCATALOGED_TOKEN_TRANSFERS_TIMEOUT` | Timeout between reindexing block batches processing. Implemented in [#14774](https://github.com/blockscout/blockscout/pull/14774). | Version: v11.3.0\+
Default: `0s`
Applications: Indexer | +| `MARKET_CRYPTOCOMPARE_API_KEY` | CryptoCompare API key. If set to a non-empty value, it is sent in the `Authorization: Apikey {key}` request header. If unset or empty, no authorization header is added. Implemented in [#14746](https://github.com/blockscout/blockscout/pull/14746). | Version: v11.3.0\+
Default: (empty)
Applications: API, Indexer | +| `CACHE_PROPAGATION_FLUSH_INTERVAL` | Interval between cache propagation flushes. Lower values reduce propagation latency, while higher values allow more cache writes to be coalesced into a batch. Implemented in [#14778](https://github.com/blockscout/blockscout/pull/14778). | Version: v11.3.0\+
Default: `100ms`
Applications: API, Indexer | +| `CACHE_PROPAGATION_SEND_TIMEOUT` | Maximum time allowed for a cache propagation batch to be sent to remote nodes before the sender is stopped and the batch is dropped. Implemented in [#14778](https://github.com/blockscout/blockscout/pull/14778). | Version: v11.3.0\+
Default: `30s`
Applications: API, Indexer | +| `CACHE_COUNTERS_CONSOLIDATION_SAFE_BLOCK_LAG` | Number of blocks behind the chain head to use as safe block for consolidation (prevents corrupting ranges during reorgs). Used by both address and token consolidators. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759), extended in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `10`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_TTL` | Time to live of address counters cache (transactions count, token transfers count, gas used). [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `1h`
Applications: API | +| `CACHE_ADDRESS_COUNTERS_MAX_DIRTY_MARKERS` | Maximum number of dirty address markers to accumulate before forcing a consolidation cycle. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `10000`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_CONSOLIDATION_DISABLED` | If `true`, disables background consolidation of address counters. Useful during rolling deploys when old and new API pods are running together. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `false`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_CONSOLIDATION_INTERVAL` | Interval between address counters consolidation cycles. [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `10s`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_CONSOLIDATION_BATCH_SIZE` | Number of addresses to consolidate counters for in a single batch. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `100`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_CONSOLIDATION_CONCURRENCY` | Number of parallel consolidation workers. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `4`
Applications: API, Indexer | +| `CACHE_ADDRESS_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT` | Query timeout for address counters consolidation. [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | Version: v11.3.0\+
Default: `30s`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_TTL` | Time to live of token counters cache (transfer count and holder count). [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `1h`
Applications: API | +| `CACHE_TOKEN_COUNTERS_MAX_DIRTY_MARKERS` | Maximum number of dirty token markers to accumulate before forcing a consolidation cycle. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `10000`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_CONSOLIDATION_DISABLED` | If `true`, disables background consolidation of token counters. Useful during rolling deploys when old and new API pods are running together. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `false`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_CONSOLIDATION_INTERVAL` | Interval between token counters consolidation cycles. [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `10s`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_CONSOLIDATION_BATCH_SIZE` | Number of tokens to consolidate counters for in a single batch. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `50`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_CONSOLIDATION_CONCURRENCY` | Number of parallel token consolidation workers. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `4`
Applications: API, Indexer | +| `CACHE_TOKEN_COUNTERS_CONSOLIDATION_QUERY_TIMEOUT` | Query timeout for token counters consolidation. [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | Version: v11.3.0\+
Default: `30s`
Applications: API, Indexer | +| `CACHE_SCAM_ADDRESSES_UPDATE_INTERVAL` | Interval between updating scam addresses cache. Implemented in [#14781](https://github.com/blockscout/blockscout/pull/14781). | Version: v11.3.0\+
Default: `5m`
Applications: API | +| `CACHE_SCAM_ADDRESSES_MAX_SIZE` | Max number of records in `scam_address_badge_mappings` table after which cache will be turned off. Implemented in [#14781](https://github.com/blockscout/blockscout/pull/14781). | Version: v11.3.0\+
Default: `200000`
Applications: API | + + +### Deprecated ENV variables + +| Variable | Description | Default | Version | Need recompile | Deprecated in Version | +| -------- | ----------- | ------- | ------- | -------------- | --------------------- | +| Deprecated `CACHE_ADDRESS_TRANSACTIONS_COUNTER_PERIOD` | Time to live of address' transaction counter. Replaced with incremental consolidation via `CACHE_ADDRESS_COUNTERS_TTL` and related env vars. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | `1h` | v3.4.0\+ | | v11.3.0+ | +| Deprecated `CACHE_ADDRESS_TOKEN_TRANSFERS_COUNTER_PERIOD` | Interval to restart the task, which calculates the number of token transfers at the address. Replaced with incremental consolidation via `CACHE_ADDRESS_COUNTERS_TTL` and related env vars. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | `1h` | v4.0.0\+ | | v11.3.0+ | +| Deprecated `CACHE_ADDRESS_TRANSACTIONS_GAS_USAGE_COUNTER_PERIOD` | Interval to restart the task which calculates gas usage at the address. Replaced with incremental consolidation via `CACHE_ADDRESS_COUNTERS_TTL` and related env vars. Implemented in [#14759](https://github.com/blockscout/blockscout/pull/14759). | `30m` | v4.1.3\+ | | v11.3.0+ | +| Deprecated `CACHE_TOKEN_HOLDERS_COUNTER_PERIOD` | Interval to restart the task which calculates holders count of the token. Replaced with delta-exact consolidation via `CACHE_TOKEN_COUNTERS_TTL` and related env vars. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | `1h` | v4.1.3\+ | | v11.3.0+ | +| Deprecated `CACHE_TOKEN_TRANSFERS_COUNTER_PERIOD` | Interval to restart the task which calculates transfers count of the token. Replaced with incremental consolidation via `CACHE_TOKEN_COUNTERS_TTL` and related env vars. Implemented in [#14773](https://github.com/blockscout/blockscout/pull/14773). | `1h` | v4.1.3\+ | | v11.3.0+ | +| Deprecated `CACHE_TOTAL_GAS_USAGE_PERIOD` | Interval to restart the task which calculates the total gas usage. Starting from release v6.8.0+, if the value is not set, ttl value gradually increases until the default value with growth of the block numbers. Removed in [#14771](https://github.com/blockscout/blockscout/pull/14771). | `2h` | v4.1.3+ | | v11.3.0+ | +| Deprecated `CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED` | If `true`, enables cache for total gas usage counter. Removed in [#14771](https://github.com/blockscout/blockscout/pull/14771). | `false` | v5.1.3+ | | v11.3.0+ | + + ## 11.2.8 From 1e61e6e885e145b032841dfef0fd4a03e9cde31d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:23:39 +0300 Subject: [PATCH 37/52] chore(deps): bump tesla from 1.21.2 to 1.21.3 (#14783) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 2b0d5c12394..d22532e172d 100644 --- a/mix.lock +++ b/mix.lock @@ -171,7 +171,7 @@ "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, - "tesla": {:hex, :tesla, "1.21.2", "71b3a8a000802fa82f0b87c22acbca1e8f6a03f23d727e01b66687111681debc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.21 or >= 4.0.2 and < 5.0.0-0", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.5.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "296bbafd1a328533c57e6b0e93078983cf40a827546e58a20464ce75485d5254"}, + "tesla": {:hex, :tesla, "1.21.3", "377e74491164595d89ad4e24b738a448c743f830a491a6c638b9e46995bca23d", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.21 or >= 4.0.2 and < 5.0.0-0", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.5.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "558aca0d763aaed4cba418b6078c290ffd55ddd1b03e58d6163cb7438851855e"}, "timex": {:hex, :timex, "3.7.13", "0688ce11950f5b65e154e42b47bf67b15d3bc0e0c3def62199991b8a8079a1e2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.26", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "09588e0522669328e973b8b4fd8741246321b3f0d32735b589f78b136e6d4c54"}, "ton": {:hex, :ton, "0.5.1", "79745434a93e5f7de3572fdcf04feb048620f0edab9794fc412a73528672927d", [:mix], [{:cafezinho, "~> 0.4.4", [hex: :cafezinho, repo: "hexpm", optional: false]}, {:evil_crc32c, "~> 0.2.9", [hex: :evil_crc32c, repo: "hexpm", optional: false]}, {:ex_pbkdf2, "~> 0.8.4", [hex: :ex_pbkdf2, repo: "hexpm", optional: false]}, {:mnemoniac, "~> 0.1.4", [hex: :mnemoniac, repo: "hexpm", optional: false]}], "hexpm", "916f656c870902a61690347da9500c5ce27f04c02e02441363bac7b128030f07"}, "typed_ecto_schema": {:hex, :typed_ecto_schema, "0.5.0", "cf0e387727dd898b6a67c80c8528d54bdb96e2318ee07dfbd65848fec2cde91f", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "061c500aa5ce03ed4defd85a2fa8ba9a53d036fc096459d570e7a4de05171ed1"}, From 069104c20eaba3e40dabfd63ceaa4965f3fd5586 Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:34:47 +0400 Subject: [PATCH 38/52] perf: Optimize realtime events broadcast for legacy topics (#14810) --- .../lib/block_scout_web/notifier.ex | 138 ++++++++++++------ .../notifier_subscriber_filter_test.exs | 134 +++++++++++++++++ 2 files changed, 227 insertions(+), 45 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index d6840236b12..fe1a8fd3f08 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -119,10 +119,19 @@ defmodule BlockScoutWeb.Notifier do exchange_rate = Market.get_coin_exchange_rate() - addresses - |> Stream.reject(fn %Address{fetched_coin_balance: fetched_coin_balance} -> is_nil(fetched_coin_balance) end) - |> Stream.filter(fn %Address{hash: hash} -> address_has_subscribers?(hash) end) - |> Enum.each(&broadcast_balance(&1, exchange_rate)) + addresses_with_balance = + Enum.reject(addresses, fn %Address{fetched_coin_balance: fetched_coin_balance} -> + is_nil(fetched_coin_balance) + end) + + addresses_with_balance + |> Enum.filter(fn %Address{hash: hash} -> address_has_v2_subscribers?(hash) end) + |> Enum.each(&broadcast_balance_v2(&1, exchange_rate)) + + # TODO: delete duplicated event when old UI becomes deprecated + addresses_with_balance + |> Enum.filter(fn %Address{hash: hash} -> address_has_old_subscribers?(hash) end) + |> Enum.each(&broadcast_balance_v1(&1, exchange_rate)) end def handle_event({:chain_event, :address_coin_balances, type, address_coin_balances}) @@ -131,7 +140,6 @@ defmodule BlockScoutWeb.Notifier do address_coin_balances |> Enum.reject(fn balance -> is_nil(balance[:value]) end) - |> Enum.filter(fn balance -> address_has_subscribers?(balance[:address_hash]) end) # Only the most recent balance of an address is worth a query and a message, # the earlier ones are superseded by it right away. |> Enum.sort_by(& &1[:block_number], :desc) @@ -142,8 +150,13 @@ defmodule BlockScoutWeb.Notifier do def handle_event({:chain_event, :address_token_balances, type, address_token_balances}) when type in [:realtime, :on_demand] do address_token_balances - |> Enum.filter(fn balance -> address_has_subscribers?(balance.address_hash) end) - |> Enum.each(&broadcast_address_token_balance/1) + |> Enum.filter(fn balance -> address_has_v2_subscribers?(balance.address_hash) end) + |> Enum.each(&broadcast_address_token_balance_v2/1) + + # TODO: delete duplicated event when old UI becomes deprecated + address_token_balances + |> Enum.filter(fn balance -> address_has_old_subscribers?(balance.address_hash) end) + |> Enum.each(&broadcast_address_token_balance_v1/1) end def handle_event( @@ -590,51 +603,69 @@ defmodule BlockScoutWeb.Notifier do end end + # Subscribers of both channels are checked here rather than filtered upstream: + # the coin balance query below feeds both payloads, so it has to run once for + # an address subscribed to either of them. defp broadcast_address_coin_balance(%{address_hash: address_hash, block_number: block_number}, exchange_rate) do - coin_balance = CoinBalance.get_coin_balance(address_hash, block_number, @api_true) + old_subscribers? = address_has_old_subscribers?(address_hash) + v2_subscribers? = address_has_v2_subscribers?(address_hash) + + coin_balance = + if old_subscribers? or v2_subscribers?, + do: CoinBalance.get_coin_balance(address_hash, block_number, @api_true) if coin_balance && coin_balance.delta && !Decimal.eq?(coin_balance.delta, Decimal.new(0)) do # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:#{address_hash}", "coin_balance", %{ - block_number: block_number, - coin_balance: coin_balance - }) + if old_subscribers? do + Endpoint.broadcast("addresses_old:#{address_hash}", "coin_balance", %{ + block_number: block_number, + coin_balance: coin_balance + }) + end + + if v2_subscribers? and coin_balance.value do + broadcast_address_coin_balance_v2(address_hash, block_number, coin_balance, exchange_rate) + end end + end - if coin_balance && coin_balance.value && coin_balance.delta && !Decimal.eq?(coin_balance.delta, Decimal.new(0)) do - rendered_coin_balance = AddressView.render("coin_balance.json", %{coin_balance: coin_balance}) + defp broadcast_address_coin_balance_v2(address_hash, block_number, coin_balance, exchange_rate) do + rendered_coin_balance = AddressView.render("coin_balance.json", %{coin_balance: coin_balance}) - Endpoint.broadcast("addresses:#{address_hash}", "coin_balance", %{ - coin_balance: rendered_coin_balance - }) + Endpoint.broadcast("addresses:#{address_hash}", "coin_balance", %{ + coin_balance: rendered_coin_balance + }) - Endpoint.broadcast("addresses:#{address_hash}", "current_coin_balance", %{ - coin_balance: coin_balance.value, - exchange_rate: exchange_rate.fiat_value, - block_number: block_number - }) - end + Endpoint.broadcast("addresses:#{address_hash}", "current_coin_balance", %{ + coin_balance: coin_balance.value, + exchange_rate: exchange_rate.fiat_value, + block_number: block_number + }) end - defp broadcast_address_token_balance(%{address_hash: address_hash, block_number: block_number}) do - # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:#{address_hash}", "token_balance", %{ + defp broadcast_address_token_balance_v2(%{address_hash: address_hash, block_number: block_number}) do + Endpoint.broadcast("addresses:#{address_hash}", "token_balance", %{ block_number: block_number }) + end - Endpoint.broadcast("addresses:#{address_hash}", "token_balance", %{ + # TODO: delete this function when old UI becomes deprecated + defp broadcast_address_token_balance_v1(%{address_hash: address_hash, block_number: block_number}) do + Endpoint.broadcast("addresses_old:#{address_hash}", "token_balance", %{ block_number: block_number }) end - defp broadcast_balance(%Address{hash: address_hash} = address, exchange_rate) do - v2_params = %{ + defp broadcast_balance_v2(%Address{hash: address_hash} = address, exchange_rate) do + Endpoint.broadcast("addresses:#{address_hash}", "balance", %{ balance: address.fetched_coin_balance.value, block_number: address.fetched_coin_balance_block_number, exchange_rate: exchange_rate.fiat_value - } + }) + end - # TODO: delete duplicated event when old UI becomes deprecated + # TODO: delete this function when old UI becomes deprecated + defp broadcast_balance_v1(%Address{hash: address_hash} = address, exchange_rate) do Endpoint.broadcast( "addresses_old:#{address_hash}", "balance_update", @@ -643,8 +674,6 @@ defmodule BlockScoutWeb.Notifier do exchange_rate: exchange_rate } ) - - Endpoint.broadcast("addresses:#{address_hash}", "balance", v2_params) end defp broadcast_block(block) do @@ -728,21 +757,35 @@ defmodule BlockScoutWeb.Notifier do defp merge_enriched_miner_field(block, _enriched_block, _field), do: block defp broadcast_rewards(rewards) do - preloaded_rewards = Repo.preload(rewards, [:address, :block]) - emission_reward = Enum.find(preloaded_rewards, fn reward -> reward.address_type == :emission_funds end) - - preloaded_rewards_except_emission = - Enum.reject(preloaded_rewards, fn reward -> reward.address_type == :emission_funds end) - - Enum.each(preloaded_rewards_except_emission, fn reward -> - # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("rewards_old:#{to_string(reward.address_hash)}", "new_reward", %{ - emission_funds: emission_reward, - validator: reward - }) + {emission_rewards, validator_rewards} = + Enum.split_with(rewards, fn reward -> reward.address_type == :emission_funds end) + Enum.each(validator_rewards, fn reward -> Endpoint.broadcast("rewards:#{to_string(reward.address_hash)}", "new_reward", %{reward: 1}) end) + + # TODO: delete duplicated event when old UI becomes deprecated + broadcast_rewards_v1(validator_rewards, emission_rewards) + end + + # TODO: delete this function when old UI becomes deprecated + defp broadcast_rewards_v1(validator_rewards, emission_rewards) do + case Enum.filter(validator_rewards, &reward_has_old_subscribers?/1) do + [] -> + :ok + + relevant_rewards -> + emission_reward = emission_rewards |> Repo.preload([:address, :block]) |> List.first() + + relevant_rewards + |> Repo.preload([:address, :block]) + |> Enum.each(fn reward -> + Endpoint.broadcast("rewards_old:#{to_string(reward.address_hash)}", "new_reward", %{ + emission_funds: emission_reward, + validator: reward + }) + end) + end end defp broadcast_internal_transaction(internal_transaction) do @@ -982,6 +1025,11 @@ defmodule BlockScoutWeb.Notifier do has_subscribers?("addresses_old:" <> to_string(address_hash)) end + # TODO: delete this function when old UI becomes deprecated + defp reward_has_old_subscribers?(reward) do + has_subscribers?("rewards_old:" <> to_string(reward.address_hash)) + end + defp token_transfer_has_subscribers?(tt) do token_transfer_has_v2_subscribers?(tt) or token_transfer_has_old_subscribers?(tt) diff --git a/apps/block_scout_web/test/block_scout_web/notifier_subscriber_filter_test.exs b/apps/block_scout_web/test/block_scout_web/notifier_subscriber_filter_test.exs index 54c18fc2fd0..a407b998b48 100644 --- a/apps/block_scout_web/test/block_scout_web/notifier_subscriber_filter_test.exs +++ b/apps/block_scout_web/test/block_scout_web/notifier_subscriber_filter_test.exs @@ -67,6 +67,72 @@ defmodule BlockScoutWeb.NotifierSubscriberFilterTest do Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) end + + # TODO: delete this test when old UI becomes deprecated + test "broadcasts balance to the old UI channel when subscribed to it only" do + {:ok, balance} = Wei.cast(1) + address = insert(:address, fetched_coin_balance: balance, fetched_coin_balance_block_number: 1) + + topic = "addresses_old:#{address.hash}" + @endpoint.subscribe(topic) + + start_supervised!(AddressesCount) + AddressesCount.consolidate() + + Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) + + assert_receive %Phoenix.Socket.Broadcast{ + topic: ^topic, + event: "balance_update", + payload: %{address: %{hash: hash}} + }, + :timer.seconds(5) + + assert hash == address.hash + end + + # TODO: delete this test when old UI becomes deprecated + test "broadcasts balance only to the old UI channel of the subscribed address in a batch" do + {:ok, balance} = Wei.cast(1) + + subscribed = insert(:address, fetched_coin_balance: balance, fetched_coin_balance_block_number: 1) + unsubscribed = insert(:address, fetched_coin_balance: balance, fetched_coin_balance_block_number: 1) + + subscribed_topic = "addresses_old:#{subscribed.hash}" + unsubscribed_topic = "addresses_old:#{unsubscribed.hash}" + @endpoint.subscribe(subscribed_topic) + @endpoint.subscribe(unsubscribed_topic) + + start_supervised!(AddressesCount) + AddressesCount.consolidate() + + Phoenix.PubSub.unsubscribe(BlockScoutWeb.PubSub, unsubscribed_topic) + + Notifier.handle_event({:chain_event, :addresses, :realtime, [subscribed, unsubscribed]}) + + assert_receive %Phoenix.Socket.Broadcast{topic: ^subscribed_topic, event: "balance_update"}, :timer.seconds(5) + refute_receive %Phoenix.Socket.Broadcast{topic: ^unsubscribed_topic, event: "balance_update"}, 100 + end + + # TODO: delete this test when old UI becomes deprecated + test "broadcasts the old UI address count when subscribed to that channel" do + {:ok, balance} = Wei.cast(1) + address = insert(:address, fetched_coin_balance: balance, fetched_coin_balance_block_number: 1) + + @endpoint.subscribe("addresses_old:new_address") + + start_supervised!(AddressesCount) + AddressesCount.consolidate() + + Notifier.handle_event({:chain_event, :addresses, :realtime, [address]}) + + assert_receive %Phoenix.Socket.Broadcast{ + topic: "addresses_old:new_address", + event: "count", + payload: %{count: _} + }, + :timer.seconds(5) + end end describe "address_coin_balances event: subscriber filtering" do @@ -104,6 +170,33 @@ defmodule BlockScoutWeb.NotifierSubscriberFilterTest do assert_receive %Phoenix.Socket.Broadcast{topic: ^subscribed_topic, event: "coin_balance"}, :timer.seconds(5) end + + # TODO: delete this test when old UI becomes deprecated + test "broadcasts to the old UI channel when subscribed to it only" do + address = insert(:address) + + coin_balance = + insert(:address_coin_balance, + address: address, + address_hash: address.hash, + delta: 500 + ) + + topic = "addresses_old:#{address.hash}" + @endpoint.subscribe(topic) + + Notifier.handle_event( + {:chain_event, :address_coin_balances, :realtime, + [%{address_hash: address.hash, block_number: coin_balance.block_number, value: 1}]} + ) + + assert_receive %Phoenix.Socket.Broadcast{ + topic: ^topic, + event: "coin_balance", + payload: %{coin_balance: _, block_number: _} + }, + :timer.seconds(5) + end end describe "address_token_balances event: subscriber filtering" do @@ -115,6 +208,47 @@ defmodule BlockScoutWeb.NotifierSubscriberFilterTest do {:chain_event, :address_token_balances, :realtime, [%{address_hash: address.hash, block_number: block.number}]} ) end + + test "broadcasts only to subscribed addresses in a batch" do + subscribed_address = insert(:address) + unsubscribed_address = insert(:address) + block = insert(:block) + + subscribed_topic = "addresses:#{subscribed_address.hash}" + unsubscribed_topic = "addresses:#{unsubscribed_address.hash}" + @endpoint.subscribe(subscribed_topic) + + Notifier.handle_event( + {:chain_event, :address_token_balances, :realtime, + [ + %{address_hash: subscribed_address.hash, block_number: block.number}, + %{address_hash: unsubscribed_address.hash, block_number: block.number} + ]} + ) + + assert_receive %Phoenix.Socket.Broadcast{topic: ^subscribed_topic, event: "token_balance"}, :timer.seconds(5) + refute_receive %Phoenix.Socket.Broadcast{topic: ^unsubscribed_topic}, 100 + end + + # TODO: delete this test when old UI becomes deprecated + test "broadcasts to the old UI channel when subscribed to it only" do + address = insert(:address) + block = insert(:block) + + topic = "addresses_old:#{address.hash}" + @endpoint.subscribe(topic) + + Notifier.handle_event( + {:chain_event, :address_token_balances, :realtime, [%{address_hash: address.hash, block_number: block.number}]} + ) + + assert_receive %Phoenix.Socket.Broadcast{ + topic: ^topic, + event: "token_balance", + payload: %{block_number: _} + }, + :timer.seconds(5) + end end describe "internal_transactions event: subscriber filtering" do From dca0feed1792d34c87b1f858007aafdee311a7a0 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Mon, 14 Sep 2026 19:31:14 +0300 Subject: [PATCH 39/52] fix: support OP Stack Upgrade 20 (Super Root games, SystemConfig v4) (#14818) --- .../schemas/api/v2/optimism/game.ex | 8 +- .../views/api/v2/optimism_view.ex | 11 +- .../explorer/chain/optimism/dispute_game.ex | 90 +++++++++++++- .../lib/explorer/chain/optimism/withdrawal.ex | 40 +++++- .../chain/optimism/dispute_game_test.exs | 114 ++++++++++++++++++ .../fetcher/optimism/transaction_batch.ex | 89 +++++++------- cspell.json | 3 + 7 files changed, 300 insertions(+), 55 deletions(-) create mode 100644 apps/explorer/test/explorer/chain/optimism/dispute_game_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/optimism/game.ex b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/optimism/game.ex index 2aa97fc3adf..a7d43450e29 100644 --- a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/optimism/game.ex +++ b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/optimism/game.ex @@ -16,7 +16,12 @@ defmodule BlockScoutWeb.Schemas.API.V2.Optimism.Game do created_at: General.Timestamp, game_type: %Schema{type: :integer}, index: %Schema{type: :integer}, - l2_block_number: %Schema{type: :integer}, + l2_block_number: %Schema{ + type: :integer, + description: "L2 block number the game is bound to. `null` for the games with Super Root claim.", + nullable: true + }, + l2_timestamp: General.TimestampNullable, resolved_at: General.TimestampNullable, status: %Schema{ type: :string, @@ -30,6 +35,7 @@ defmodule BlockScoutWeb.Schemas.API.V2.Optimism.Game do :game_type, :index, :l2_block_number, + :l2_timestamp, :resolved_at, :status ], diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/optimism_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/optimism_view.ex index 6faebc9b279..9bf545c7493 100644 --- a/apps/block_scout_web/lib/block_scout_web/views/api/v2/optimism_view.ex +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/optimism_view.ex @@ -122,13 +122,22 @@ defmodule BlockScoutWeb.API.V2.OptimismView do 2 -> "Defender wins" end - l2_block_number = DisputeGame.l2_block_number_from_extra_data(g.extra_data) + # Games with Output Root claim are bound to L2 block number, whereas games with Super Root claim + # (OP Stack Upgrade 20) are bound to L2 timestamp + {l2_block_number, l2_timestamp} = + case DisputeGame.l2_sequence_number(g) do + {:block_number, block_number} -> {block_number, nil} + # zero timestamp means the extra data is unknown or malformed + {:timestamp, 0} -> {nil, nil} + {:timestamp, timestamp} -> {nil, DateTime.from_unix!(timestamp)} + end %{ "index" => g.index, "game_type" => g.game_type, "contract_address_hash" => g.address_hash, "l2_block_number" => l2_block_number, + "l2_timestamp" => l2_timestamp, "created_at" => g.created_at, "status" => status, "resolved_at" => g.resolved_at diff --git a/apps/explorer/lib/explorer/chain/optimism/dispute_game.ex b/apps/explorer/lib/explorer/chain/optimism/dispute_game.ex index 25666afb677..ca6ab27c6d4 100644 --- a/apps/explorer/lib/explorer/chain/optimism/dispute_game.ex +++ b/apps/explorer/lib/explorer/chain/optimism/dispute_game.ex @@ -19,11 +19,24 @@ defmodule Explorer.Chain.Optimism.DisputeGame do @chain_id_megaeth_mainnet 4326 @chain_id_megaeth_testnet_v2 6343 + # Game types whose root claim is a Super Root rather than an Output Root (introduced by OP Stack Upgrade 20). + # Mirrors `GameTypes.isSuperGame` from the OP Stack contracts: + # SUPER_CANNON (4), SUPER_PERMISSIONED (5), SUPER_ASTERISC_KONA (7), SUPER_CANNON_KONA (9), ZK_DISPUTE_GAME (10). + @super_game_types [4, 5, 7, 9, 10] + + # The only supported version of the Super Root proof encoding stored in the `extraData` of a Super Root game + @super_root_proof_version 1 + + # The size (in bytes) of one output root entry in the Super Root proof: 32 bytes of chain ID + 32 bytes of output root + @super_root_output_root_size 64 + @typedoc """ * `index` - A unique index of the dispute game. * `game_type` - A number encoding a type of the dispute game. * `address_hash` - The dispute game contract address. - * `extra_data` - An extra data of the dispute game (contains L2 block number). + * `extra_data` - An extra data of the dispute game. For the games with Output Root claim it contains L2 block number. + For the games with Super Root claim (see `super_game_type?/1`) it contains the Super Root proof + (version byte, L2 timestamp, and the list of output roots with their chain IDs). Equals to `nil` when the game is written to database but the rest data is not known yet. * `created_at` - UTC timestamp of when the dispute game was created. * `resolved_at` - UTC timestamp of when the dispute game was resolved. @@ -89,14 +102,52 @@ defmodule Explorer.Chain.Optimism.DisputeGame do end @doc """ - Retrieves L2 block number from the `extraData` field of the dispute game. The L2 block number can be encoded in - different ways depending on the chain. + Checks whether the given game type is a Super Root game type, i.e. the game's root claim is a Super Root + (a commitment to the output roots of several chains at the same L2 timestamp) rather than an Output Root + of a single chain. Such games were introduced by OP Stack Upgrade 20. + + ## Parameters + - `game_type`: The game type number. + + ## Returns + - `true` if the game type is a Super Root game type, `false` otherwise. + """ + @spec super_game_type?(non_neg_integer() | nil) :: boolean() + def super_game_type?(game_type), do: game_type in @super_game_types + + @doc """ + Retrieves the L2 sequence number of the dispute game from its `extraData` field. + + For the games with Output Root claim the sequence number is the L2 block number. + For the games with Super Root claim (see `super_game_type?/1`) the sequence number is the L2 timestamp + of the Super Root. + + ## Parameters + - `game`: A map (or `t:Explorer.Chain.Optimism.DisputeGame.t/0`) with `game_type` and `extra_data` fields. + + ## Returns + - `{:block_number, l2_block_number}` tuple for the games with Output Root claim. + - `{:timestamp, l2_timestamp}` tuple for the games with Super Root claim (the timestamp is in UNIX seconds). + """ + @spec l2_sequence_number(%{:game_type => non_neg_integer() | nil, :extra_data => Data.t() | nil, any() => any()}) :: + {:block_number, non_neg_integer()} | {:timestamp, non_neg_integer()} + def l2_sequence_number(%{game_type: game_type, extra_data: extra_data}) do + if super_game_type?(game_type) do + {:timestamp, l2_timestamp_from_extra_data(extra_data)} + else + {:block_number, l2_block_number_from_extra_data(extra_data)} + end + end + + @doc """ + Retrieves L2 block number from the `extraData` field of the dispute game with Output Root claim. + The L2 block number can be encoded in different ways depending on the chain. ## Parameters - `extra_data`: The byte sequence of the extra data to retrieve L2 block number from. ## Returns - - L2 block number of the dispute game. + - L2 block number of the dispute game. Zero if the extra data is unknown or has unexpected format. """ @spec l2_block_number_from_extra_data(Data.t() | nil) :: non_neg_integer() def l2_block_number_from_extra_data(nil), do: 0 @@ -114,10 +165,37 @@ defmodule Explorer.Chain.Optimism.DisputeGame do 256 end - <> = extra_data - l2_block_number + case extra_data do + <> -> l2_block_number + _ -> 0 + end end + @doc """ + Retrieves L2 timestamp from the `extraData` field of the dispute game with Super Root claim. + + The extra data of such a game contains the Super Root proof encoded as follows: + 1 byte of version (must be equal to 1), 8 bytes of the L2 timestamp (big-endian uint64), + and 64 bytes for each output root (32 bytes of chain ID and 32 bytes of the output root). + See `Encoding.decodeSuperRootProof` in the OP Stack contracts. + + ## Parameters + - `extra_data`: The byte sequence of the extra data to retrieve L2 timestamp from. + + ## Returns + - L2 timestamp (in UNIX seconds) of the Super Root. Zero if the extra data is unknown or has unexpected format. + """ + @spec l2_timestamp_from_extra_data(Data.t() | nil) :: non_neg_integer() + def l2_timestamp_from_extra_data(nil), do: 0 + + def l2_timestamp_from_extra_data(%Data{ + bytes: <<@super_root_proof_version, l2_timestamp::size(64), output_roots::binary>> + }) + when byte_size(output_roots) > 0 and rem(byte_size(output_roots), @super_root_output_root_size) == 0, + do: l2_timestamp + + def l2_timestamp_from_extra_data(%Data{}), do: 0 + defp page_dispute_games(query, %PagingOptions{key: nil}), do: query defp page_dispute_games(query, %PagingOptions{key: {index}}) do diff --git a/apps/explorer/lib/explorer/chain/optimism/withdrawal.ex b/apps/explorer/lib/explorer/chain/optimism/withdrawal.ex index 5980f9ed4bb..d98df84e9b2 100644 --- a/apps/explorer/lib/explorer/chain/optimism/withdrawal.ex +++ b/apps/explorer/lib/explorer/chain/optimism/withdrawal.ex @@ -169,6 +169,7 @@ defmodule Explorer.Chain.Optimism.Withdrawal do select: %{ hash: w.hash, l2_block_number: w.l2_block_number, + l2_timestamp: l2_block.timestamp, l1_transaction_hash: we.l1_transaction_hash, msg_nonce: w.msg_nonce, msg_log_sender_address_hash: log.third_topic, @@ -226,7 +227,7 @@ defmodule Explorer.Chain.Optimism.Withdrawal do if proven_events == [] do cond do - appropriate_games_found(w.l2_block_number, respected_games) -> + appropriate_games_found(w, respected_games) -> {@withdrawal_status_ready_to_prove, nil} appropriate_root_found(w.l2_block_number) -> @@ -282,14 +283,41 @@ defmodule Explorer.Chain.Optimism.Withdrawal do @proof_maturity_delay_seconds end - defp appropriate_games_found(withdrawal_l2_block_number, respected_games) do - respected_games - |> Enum.any?(fn game -> - l2_block_number = DisputeGame.l2_block_number_from_extra_data(game.extra_data) - withdrawal_l2_block_number <= l2_block_number + # Checks whether there is a respected dispute game the given withdrawal can be proven against. + # + # For the games with Output Root claim the withdrawal's L2 block number is compared with the L2 block number + # of the game. For the games with Super Root claim (OP Stack Upgrade 20) the game's `extraData` contains + # the L2 timestamp of the Super Root instead of L2 block number, so the timestamp of the withdrawal's L2 block + # is compared with it. + # + # ## Parameters + # - `w`: A map with the withdrawal info. Must contain `l2_block_number` and (for Super Root games) `l2_timestamp`. + # - `respected_games`: A list of games returned by the `respected_games(options)` function. + # + # ## Returns + # - `true` if an appropriate game is found, `false` otherwise. + @spec appropriate_games_found(map(), list()) :: boolean() + defp appropriate_games_found(w, respected_games) do + Enum.any?(respected_games, fn game -> + case DisputeGame.l2_sequence_number(game) do + {:block_number, game_l2_block_number} -> + w.l2_block_number <= game_l2_block_number + + {:timestamp, game_l2_timestamp} -> + withdrawal_covered_by_timestamp?(Map.get(w, :l2_timestamp), game_l2_timestamp) + end end) end + # Checks whether the withdrawal's L2 block timestamp is covered by the Super Root game bound to the given L2 timestamp. + # Returns `false` if the withdrawal's L2 block timestamp is unknown (e.g. the L2 block is not indexed yet). + @spec withdrawal_covered_by_timestamp?(DateTime.t() | nil, non_neg_integer()) :: boolean() + defp withdrawal_covered_by_timestamp?(%DateTime{} = withdrawal_l2_timestamp, game_l2_timestamp) do + DateTime.to_unix(withdrawal_l2_timestamp) <= game_l2_timestamp + end + + defp withdrawal_covered_by_timestamp?(_withdrawal_l2_timestamp, _game_l2_timestamp), do: false + defp appropriate_root_found(withdrawal_l2_block_number) do last_root_l2_block_number = Repo.replica().one( diff --git a/apps/explorer/test/explorer/chain/optimism/dispute_game_test.exs b/apps/explorer/test/explorer/chain/optimism/dispute_game_test.exs new file mode 100644 index 00000000000..358aae7f49a --- /dev/null +++ b/apps/explorer/test/explorer/chain/optimism/dispute_game_test.exs @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Optimism.DisputeGameTest do + use Explorer.DataCase + + alias Explorer.Chain.Cache.ChainId + alias Explorer.Chain.Data + alias Explorer.Chain.Optimism.DisputeGame + + if Application.compile_env(:explorer, :chain_type) == :optimism do + # game types are taken from `GameTypes` library of the OP Stack contracts + @cannon_game_type 0 + @permissioned_cannon_game_type 1 + @super_cannon_game_type 4 + @super_permissioned_game_type 5 + @super_asterisc_kona_game_type 7 + @super_cannon_kona_game_type 9 + @zk_dispute_game_type 10 + + @l2_block_number 123_456_789 + @l2_timestamp 1_758_000_000 + + # `extraData` of a game with Output Root claim: abi.encode(uint256 l2BlockNumber) + @output_root_extra_data %Data{bytes: <<@l2_block_number::size(256)>>} + + # `extraData` of a game with Super Root claim (see `Encoding.decodeSuperRootProof` in the OP Stack contracts): + # 1 byte of version, 8 bytes of timestamp, (32 bytes of chain ID + 32 bytes of output root) per chain + @super_root_extra_data %Data{ + bytes: <<1, @l2_timestamp::size(64), 10::size(256), 0xAA::size(256), 8453::size(256), 0xBB::size(256)>> + } + + setup do + ChainId.set_id(10) + :ok + end + + describe "super_game_type?/1" do + test "returns true only for Super Root game types" do + assert DisputeGame.super_game_type?(@super_cannon_game_type) + assert DisputeGame.super_game_type?(@super_permissioned_game_type) + assert DisputeGame.super_game_type?(@super_asterisc_kona_game_type) + assert DisputeGame.super_game_type?(@super_cannon_kona_game_type) + assert DisputeGame.super_game_type?(@zk_dispute_game_type) + + refute DisputeGame.super_game_type?(@cannon_game_type) + refute DisputeGame.super_game_type?(@permissioned_cannon_game_type) + refute DisputeGame.super_game_type?(nil) + end + end + + describe "l2_sequence_number/1" do + test "returns L2 block number for a game with Output Root claim" do + game = %{game_type: @cannon_game_type, extra_data: @output_root_extra_data} + assert DisputeGame.l2_sequence_number(game) == {:block_number, @l2_block_number} + end + + test "returns L2 timestamp for a game with Super Root claim" do + for game_type <- [@super_permissioned_game_type, @super_cannon_kona_game_type] do + game = %{game_type: game_type, extra_data: @super_root_extra_data} + assert DisputeGame.l2_sequence_number(game) == {:timestamp, @l2_timestamp} + end + end + + test "returns zero when extra data is unknown" do + assert DisputeGame.l2_sequence_number(%{game_type: @cannon_game_type, extra_data: nil}) == {:block_number, 0} + + assert DisputeGame.l2_sequence_number(%{game_type: @super_cannon_kona_game_type, extra_data: nil}) == + {:timestamp, 0} + end + end + + describe "l2_block_number_from_extra_data/1" do + test "returns zero when extra data is shorter than expected" do + assert DisputeGame.l2_block_number_from_extra_data(%Data{bytes: <<1, 2, 3>>}) == 0 + end + end + + describe "l2_timestamp_from_extra_data/1" do + test "parses the timestamp of the Super Root proof" do + assert DisputeGame.l2_timestamp_from_extra_data(@super_root_extra_data) == @l2_timestamp + end + + test "returns zero for unsupported Super Root proof version" do + extra_data = %Data{bytes: <<2, @l2_timestamp::size(64), 10::size(256), 0xAA::size(256)>>} + assert DisputeGame.l2_timestamp_from_extra_data(extra_data) == 0 + end + + test "returns zero when extra data is shorter than expected" do + assert DisputeGame.l2_timestamp_from_extra_data(%Data{bytes: <<1, 2, 3>>}) == 0 + end + + test "returns zero when the list of output roots is empty" do + assert DisputeGame.l2_timestamp_from_extra_data(%Data{bytes: <<1, @l2_timestamp::size(64)>>}) == 0 + end + + test "returns zero when an output root entry is truncated" do + # 32 bytes of chain ID followed by only 16 bytes of the output root + extra_data = %Data{bytes: <<1, @l2_timestamp::size(64), 10::size(256), 0xAA::size(128)>>} + assert DisputeGame.l2_timestamp_from_extra_data(extra_data) == 0 + + # one complete entry followed by a truncated one + extra_data = %Data{ + bytes: <<1, @l2_timestamp::size(64), 10::size(256), 0xAA::size(256), 8453::size(256), 0xBB::size(64)>> + } + + assert DisputeGame.l2_timestamp_from_extra_data(extra_data) == 0 + end + + test "accepts a single complete output root entry" do + extra_data = %Data{bytes: <<1, @l2_timestamp::size(64), 10::size(256), 0xAA::size(256)>>} + assert DisputeGame.l2_timestamp_from_extra_data(extra_data) == @l2_timestamp + end + end + end +end diff --git a/apps/indexer/lib/indexer/fetcher/optimism/transaction_batch.ex b/apps/indexer/lib/indexer/fetcher/optimism/transaction_batch.ex index 7d1e476a89b..0b0b3976d48 100644 --- a/apps/indexer/lib/indexer/fetcher/optimism/transaction_batch.ex +++ b/apps/indexer/lib/indexer/fetcher/optimism/transaction_batch.ex @@ -1562,8 +1562,12 @@ defmodule Indexer.Fetcher.Optimism.TransactionBatch do # Gets the number of a start block (from which this fetcher should start), # the inbox address, and the batcher (batch submitter) address. # - # If SystemConfig has obsolete implementation, the values are fallen back from the corresponding - # env variables (INDEXER_OPTIMISM_L1_START_BLOCK, INDEXER_OPTIMISM_L1_BATCH_INBOX, INDEXER_OPTIMISM_L1_BATCH_SUBMITTER). + # Each getter is handled independently: if a getter is unavailable (the call reverts or the whole request fails), + # only that value is fallen back from the corresponding env variable (INDEXER_OPTIMISM_L1_START_BLOCK, + # INDEXER_OPTIMISM_L1_BATCH_INBOX, INDEXER_OPTIMISM_L1_BATCH_SUBMITTER) while the others are still taken + # from the contract. In particular, SystemConfig v4.0.0 (OP Stack Upgrade 20) removed the `batchInbox()` getter, + # so the inbox address must be defined through INDEXER_OPTIMISM_L1_BATCH_INBOX for such chains, whereas + # `startBlock()` and `batcherHash()` are still readable. # # Moreover, if INDEXER_OPTIMISM_L1_BATCH_INBOX and/or INDEXER_OPTIMISM_L1_BATCH_SUBMITTER are explicitly set, # they take precedence over the corresponding values read from the SystemConfig contract. This is needed when @@ -1579,15 +1583,18 @@ defmodule Indexer.Fetcher.Optimism.TransactionBatch do # - `nil` in case of error. @spec read_system_config(String.t(), EthereumJSONRPC.json_rpc_named_arguments()) :: {non_neg_integer(), String.t(), String.t()} | nil - # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity defp read_system_config(contract_address, json_rpc_named_arguments) do + start_block_request_id = 0 + batch_inbox_request_id = 1 + batch_submitter_request_id = 2 + requests = [ # startBlock() public getter - Contract.eth_call_request("0x48cd4cb1", contract_address, 0, nil, nil), - # batchInbox() public getter - Contract.eth_call_request("0xdac6e63a", contract_address, 1, nil, nil), + Contract.eth_call_request("0x48cd4cb1", contract_address, start_block_request_id, nil, nil), + # batchInbox() public getter (removed in SystemConfig v4.0.0, so the call can revert) + Contract.eth_call_request("0xdac6e63a", contract_address, batch_inbox_request_id, nil, nil), # batcherHash() public getter - Contract.eth_call_request("0xe81b2c6d", contract_address, 2, nil, nil) + Contract.eth_call_request("0xe81b2c6d", contract_address, batch_submitter_request_id, nil, nil) ] error_message = &"Cannot call public getters of SystemConfig. Error: #{inspect(&1)}" @@ -1595,57 +1602,57 @@ defmodule Indexer.Fetcher.Optimism.TransactionBatch do env = Application.get_all_env(:indexer)[__MODULE__] fallback_start_block = Application.get_all_env(:indexer)[Indexer.Fetcher.Optimism][:start_block_l1] - {start_block, batch_inbox, batch_submitter} = + result_by_id = case Helper.repeated_call( &json_rpc/2, [requests, json_rpc_named_arguments], error_message, Helper.finite_retries_number() ) do - {:ok, responses} -> - start_block = - responses - |> Enum.at(0) - |> Map.get(:result, fallback_start_block) - |> quantity_to_integer() - - inbox_result = Map.get(Enum.at(responses, 1), :result) - submitter_result = Map.get(Enum.at(responses, 2), :result) - - {batch_inbox, batch_submitter} = - with {:nil_result, true, _, _} <- - {:nil_result, is_nil(inbox_result) or is_nil(submitter_result), inbox_result, submitter_result}, - {:fallback_defined, true} <- - {:fallback_defined, - Helper.address_correct?(env[:inbox]) and Helper.address_correct?(env[:submitter])} do - {env[:inbox], env[:submitter]} - else - {:nil_result, false, inbox, submitter} -> - "0x000000000000000000000000" <> batch_inbox = inbox - "0x000000000000000000000000" <> batch_submitter = submitter - {"0x" <> batch_inbox, "0x" <> batch_submitter} - - {:fallback_defined, false} -> - {nil, nil} - end - - {start_block, batch_inbox, batch_submitter} - - _ -> - {fallback_start_block, env[:inbox], env[:submitter]} + {:ok, responses} -> Map.new(responses, &{&1.id, Map.get(&1, :result)}) + _ -> %{} end + start_block = + result_by_id + |> Map.get(start_block_request_id) + |> Kernel.||(fallback_start_block) + |> quantity_to_integer() + # An explicitly configured inbox/submitter overrides the value read from the SystemConfig contract. # Only kicks in when the corresponding env variable holds a correct address, so the on-chain value # is still used by default. - batch_inbox = if Helper.address_correct?(env[:inbox]), do: env[:inbox], else: batch_inbox - batch_submitter = if Helper.address_correct?(env[:submitter]), do: env[:submitter], else: batch_submitter + batch_inbox = system_config_address(result_by_id[batch_inbox_request_id], env[:inbox]) + batch_submitter = system_config_address(result_by_id[batch_submitter_request_id], env[:submitter]) + + if is_nil(batch_inbox) and Map.has_key?(result_by_id, batch_submitter_request_id) do + Logger.error( + "The batchInbox() getter of SystemConfig is not available (it was removed in SystemConfig v4.0.0 by OP Stack Upgrade 20). Please, define the batch inbox address via INDEXER_OPTIMISM_L1_BATCH_INBOX env variable." + ) + end if !is_nil(start_block) and Helper.address_correct?(batch_inbox) and Helper.address_correct?(batch_submitter) do {start_block, String.downcase(batch_inbox), String.downcase(batch_submitter)} end end + # Returns an address read from a SystemConfig getter unless the corresponding env variable + # defines a correct address (which takes precedence). Returns `nil` if the getter result + # is absent (e.g. the call reverted) and the env variable is not defined. + @spec system_config_address(String.t() | nil, String.t() | nil) :: String.t() | nil + defp system_config_address(getter_result, env_address) do + cond do + Helper.address_correct?(env_address) -> + env_address + + is_binary(getter_result) and String.starts_with?(getter_result, "0x000000000000000000000000") -> + "0x" <> String.slice(getter_result, 26..-1//1) + + true -> + nil + end + end + # Retrieves Alt-DA signature from L1 transaction input as described in # https://github.com/ethereum-optimism/specs/blob/main/specs/experimental/alt-da.md#example-commitments # diff --git a/cspell.json b/cspell.json index e336bd03db6..c14c328058d 100644 --- a/cspell.json +++ b/cspell.json @@ -58,6 +58,7 @@ "Asearch", "Asfpp", "Astar", + "asterisc", "asyncable", "atoken", "audix", @@ -387,6 +388,7 @@ "keyout", "keyreplace", "kittencream", + "kona", "KnxbUejwY", "labelledby", "lastmod", @@ -522,6 +524,7 @@ "peekers", "pendingtxlist", "perc", + "permissioned", "permissionless", "persistable", "PGDATABASE", From 6037429b93685d7aa9dec4cbe4cc8ec9c6b424b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:34:54 +0300 Subject: [PATCH 40/52] chore(deps-dev): bump dialyxir from 1.4.7 to 1.4.8 (#14822) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index d22532e172d..764c84f7276 100644 --- a/mix.lock +++ b/mix.lock @@ -40,14 +40,14 @@ "decimal": {:hex, :decimal, "2.4.1", "6c0fbede12fb122ba685e9ab41c6a40c129e322b3aa192f9e072e61f3a6ffaf2", [:mix], [], "hexpm", "7e618897933a8455f19a727d7c5e50a2c071a544b700e5e724298ecb4340187f"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, - "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, + "dialyxir": {:hex, :dialyxir, "1.4.8", "7ef671a8aff9948b091d8c30f09467fbb16e77305cda451bce48109a0f5e021c", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "cbd5a851571e5dfeb32aaf2e840bfa98b7864cb3071bf2ef5d95d1276b12e072"}, "digital_token": {:hex, :digital_token, "1.0.0", "454a4444061943f7349a51ef74b7fb1ebd19e6a94f43ef711f7dae88c09347df", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "8ed6f5a8c2fa7b07147b9963db506a1b4c7475d9afca6492136535b064c9e9e6"}, "dns": {:hex, :dns, "2.4.0", "44790a0375b28bdc7b59fc894460bfcb03ffeec4c5984e2c3e8b0797b1518327", [:mix], [], "hexpm", "e178e353c469820d02ba889d6a80d01c8c27b47dfcda4016a9cbc6218e3eed64"}, "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "ecto": {:hex, :ecto, "3.13.6", "352135b474f91d1ab99a1b502171d207e9db60421c9e3d0ecab4c7ab96b24d14", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8afa059bc16cd2c94739ec0a11e3e5df69d828125119109bef35f20a21a76af2"}, "ecto_sql": {:hex, :ecto_sql, "3.13.5", "2f8282b2ad97bf0f0d3217ea0a6fff320ead9e2f8770f810141189d182dc304e", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aa36751f4e6a2b56ae79efb0e088042e010ff4935fc8684e74c23b1f49e25fdc"}, "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, - "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "evil_crc32c": {:hex, :evil_crc32c, "0.2.9", "9e4082418c4cc27c88869c1773f114f15a9257121ebb019037bea60c75c601d2", [:mix], [{:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.8", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "86ba14f8aad0fe55eda0b57e79e3d8dffe39ea326b753eac12463ae2bec757bf"}, "evision": {:hex, :evision, "0.2.17", "0d5aed4fdb6945f100642cfd41531d187963af984c60ec367719106dda7ed864", [:make, :mix, :rebar3], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:kino, "~> 0.11", [hex: :kino, repo: "hexpm", optional: true]}, {:nx, "~> 0.6", [hex: :nx, repo: "hexpm", optional: false]}, {:progress_bar, "~> 2.0 or ~> 3.0", [hex: :progress_bar, repo: "hexpm", optional: true]}], "hexpm", "565bc8700d23cd2429444387a1cf0e53fe7a85892e9276ce62a9cef23350c11f"}, From 051fbf19b957053b9a278d5d468e0f438ef696aa Mon Sep 17 00:00:00 2001 From: Qwerty5Uiop <105209995+Qwerty5Uiop@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:50:18 +0400 Subject: [PATCH 41/52] perf: Use Endpoint.local_broadcast instead of Endpoint.broadcast (#14819) --- .../lib/block_scout_web/notifier.ex | 107 ++++++++++-------- .../lib/block_scout_web/notifiers/arbitrum.ex | 4 +- .../lib/block_scout_web/notifiers/optimism.ex | 4 +- .../notifier_broadcast_test.exs | 81 +++++++++++++ 4 files changed, 142 insertions(+), 54 deletions(-) create mode 100644 apps/block_scout_web/test/block_scout_web/notifier_broadcast_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/notifier.ex b/apps/block_scout_web/lib/block_scout_web/notifier.ex index fe1a8fd3f08..d58223e9d0b 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifier.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifier.ex @@ -2,6 +2,12 @@ defmodule BlockScoutWeb.Notifier do @moduledoc """ Responds to events by sending appropriate channel updates to front-end. + + Every node handles the chain events itself (API nodes receive them from the + database through `Explorer.Chain.Events.Listener`), so the updates are sent to + the subscribers of the current node only. Forwarding them to the other nodes + of the cluster would duplicate the messages there and suspend the handling + process whenever the distribution buffer to one of those nodes is full. """ use Utils.CompileTimeEnvHelper, chain_type: [:explorer, :chain_type], @@ -114,8 +120,8 @@ defmodule BlockScoutWeb.Notifier do addresses_count = AddressesCount.fetch() # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:new_address", "count", %{count: addresses_count}) - Endpoint.broadcast("addresses:new_address", "count", %{count: addresses_count}) + Endpoint.local_broadcast("addresses_old:new_address", "count", %{count: addresses_count}) + Endpoint.local_broadcast("addresses:new_address", "count", %{count: addresses_count}) exchange_rate = Market.get_coin_exchange_rate() @@ -166,7 +172,7 @@ defmodule BlockScoutWeb.Notifier do v2_params = verification_result_params_v2(contract_verification_result) # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast( + Endpoint.local_broadcast( "addresses_old:#{address_hash}", "verification_result", %{ @@ -174,7 +180,7 @@ defmodule BlockScoutWeb.Notifier do } ) - Endpoint.broadcast("addresses:#{address_hash}", "verification_result", v2_params) + Endpoint.local_broadcast("addresses:#{address_hash}", "verification_result", v2_params) end def handle_event( @@ -207,7 +213,7 @@ defmodule BlockScoutWeb.Notifier do end # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast( + Endpoint.local_broadcast( "addresses_old:#{address_hash}", "verification_result", %{ @@ -215,7 +221,7 @@ defmodule BlockScoutWeb.Notifier do } ) - Endpoint.broadcast("addresses:#{address_hash}", "verification_result", v2_params) + Endpoint.local_broadcast("addresses:#{address_hash}", "verification_result", v2_params) end def handle_event({:chain_event, :block_rewards, :realtime, rewards}) do @@ -252,12 +258,12 @@ defmodule BlockScoutWeb.Notifier do end # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("exchange_rate_old:new_rate", "new_rate", %{ + Endpoint.local_broadcast("exchange_rate_old:new_rate", "new_rate", %{ exchange_rate: exchange_rate_with_available_supply, market_history_data: market_history_data }) - Endpoint.broadcast("exchange_rate:new_rate", "new_rate", %{ + Endpoint.local_broadcast("exchange_rate:new_rate", "new_rate", %{ exchange_rate: exchange_rate_with_available_supply.fiat_value, available_supply: exchange_rate_with_available_supply.available_supply, chart_data: market_history_data @@ -273,7 +279,7 @@ defmodule BlockScoutWeb.Notifier do |> Map.get(:transaction_hash) # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("transactions_old:#{transaction_hash}", "raw_trace", %{raw_trace_origin: transaction_hash}) + Endpoint.local_broadcast("transactions_old:#{transaction_hash}", "raw_trace", %{raw_trace_origin: transaction_hash}) internal_transactions = InternalTransaction.all_transaction_to_internal_transactions(transaction_hash) @@ -281,7 +287,7 @@ defmodule BlockScoutWeb.Notifier do raw_trace: TransactionView.render("raw_trace.json", %{internal_transactions: internal_transactions}) } - Endpoint.broadcast("transactions:#{transaction_hash}", "raw_trace", v2_params) + Endpoint.local_broadcast("transactions:#{transaction_hash}", "raw_trace", v2_params) end # internal transactions broadcast disabled on the indexer level, therefore it out of scope of the refactoring within https://github.com/blockscout/blockscout/pull/7474 @@ -306,7 +312,8 @@ defmodule BlockScoutWeb.Notifier do Enum.group_by(all_token_transfers, fn tt -> to_string(tt.token_contract_address_hash) end) for {token_contract_address_hash, token_transfers} <- all_transfers_by_token do - Subscription.publish( + # Local only, like the channel updates (see the moduledoc) + Subscription.Local.publish_mutation( Endpoint, token_transfers, token_transfers: token_contract_address_hash @@ -352,7 +359,7 @@ defmodule BlockScoutWeb.Notifier do date_range = TransactionStats.by_date_range(x_days_back, today) stats = Enum.map(date_range, fn item -> Map.drop(item, [:__meta__]) end) - Endpoint.broadcast("transactions_old:stats", "update", %{stats: stats}) + Endpoint.local_broadcast("transactions_old:stats", "update", %{stats: stats}) end def handle_event( @@ -361,20 +368,20 @@ defmodule BlockScoutWeb.Notifier do ) when not is_nil(total_supply) do # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("tokens_old:#{to_string(contract_address_hash)}", "token_total_supply", %{token: token}) + Endpoint.local_broadcast("tokens_old:#{to_string(contract_address_hash)}", "token_total_supply", %{token: token}) - Endpoint.broadcast("tokens:#{to_string(contract_address_hash)}", "total_supply", %{ + Endpoint.local_broadcast("tokens:#{to_string(contract_address_hash)}", "total_supply", %{ total_supply: to_string(total_supply) }) end def handle_event({:chain_event, :fetched_bytecode, :on_demand, [address_hash, fetched_bytecode]}) do # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:#{to_string(address_hash)}", "fetched_bytecode", %{ + Endpoint.local_broadcast("addresses_old:#{to_string(address_hash)}", "fetched_bytecode", %{ fetched_bytecode: fetched_bytecode }) - Endpoint.broadcast("addresses:#{to_string(address_hash)}", "fetched_bytecode", %{ + Endpoint.local_broadcast("addresses:#{to_string(address_hash)}", "fetched_bytecode", %{ fetched_bytecode: fetched_bytecode }) end @@ -383,7 +390,7 @@ defmodule BlockScoutWeb.Notifier do {:chain_event, :fetched_token_instance_metadata, :on_demand, [token_contract_address_hash_string, token_id, fetched_token_instance_metadata]} ) do - Endpoint.broadcast( + Endpoint.local_broadcast( "token_instances:#{token_contract_address_hash_string}", "fetched_token_instance_metadata", %{token_id: to_string(token_id), fetched_metadata: fetched_token_instance_metadata} @@ -394,7 +401,7 @@ defmodule BlockScoutWeb.Notifier do {:chain_event, :not_fetched_token_instance_metadata, :on_demand, [token_contract_address_hash_string, token_id, reason]} ) do - Endpoint.broadcast( + Endpoint.local_broadcast( "token_instances:#{token_contract_address_hash_string}", "not_fetched_token_instance_metadata", %{token_id: to_string(token_id), reason: reason} @@ -403,8 +410,8 @@ defmodule BlockScoutWeb.Notifier do def handle_event({:chain_event, :changed_bytecode, :on_demand, [address_hash]}) do # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:#{to_string(address_hash)}", "changed_bytecode", %{}) - Endpoint.broadcast("addresses:#{to_string(address_hash)}", "changed_bytecode", %{}) + Endpoint.local_broadcast("addresses_old:#{to_string(address_hash)}", "changed_bytecode", %{}) + Endpoint.local_broadcast("addresses:#{to_string(address_hash)}", "changed_bytecode", %{}) end def handle_event({:chain_event, :smart_contract_was_verified = event, :on_demand, [address_hash]}) do @@ -459,7 +466,7 @@ defmodule BlockScoutWeb.Notifier do blocks |> Enum.group_by(&(&1 |> BlockScoutWeb.BlockView.block_type() |> String.downcase())) |> Enum.each(fn {type, blocks_list} -> - Endpoint.broadcast("blocks:new_block", "new_blocks_count", %{count: Enum.count(blocks_list), type: type}) + Endpoint.local_broadcast("blocks:new_block", "new_blocks_count", %{count: Enum.count(blocks_list), type: type}) end) end @@ -538,7 +545,7 @@ defmodule BlockScoutWeb.Notifier do event = "updated_token_balances_" <> event_postfix - Endpoint.broadcast("addresses:#{address_hash}", event, %{ + Endpoint.local_broadcast("addresses:#{address_hash}", event, %{ token_balances: AddressView.render("token_balances.json", %{ token_balances: Enum.take(sorted, @current_token_balances_limit) @@ -567,7 +574,7 @@ defmodule BlockScoutWeb.Notifier do @spec broadcast_indexed_ratio(String.t(), Decimal.t()) :: :ok | {:error, term()} def broadcast_indexed_ratio(msg, ratio) do - Endpoint.broadcast(msg, "index_status", %{ + Endpoint.local_broadcast(msg, "index_status", %{ ratio: Decimal.to_string(ratio), finished: Chain.finished_indexing_from_ratio?(ratio) }) @@ -617,7 +624,7 @@ defmodule BlockScoutWeb.Notifier do if coin_balance && coin_balance.delta && !Decimal.eq?(coin_balance.delta, Decimal.new(0)) do # TODO: delete duplicated event when old UI becomes deprecated if old_subscribers? do - Endpoint.broadcast("addresses_old:#{address_hash}", "coin_balance", %{ + Endpoint.local_broadcast("addresses_old:#{address_hash}", "coin_balance", %{ block_number: block_number, coin_balance: coin_balance }) @@ -632,11 +639,11 @@ defmodule BlockScoutWeb.Notifier do defp broadcast_address_coin_balance_v2(address_hash, block_number, coin_balance, exchange_rate) do rendered_coin_balance = AddressView.render("coin_balance.json", %{coin_balance: coin_balance}) - Endpoint.broadcast("addresses:#{address_hash}", "coin_balance", %{ + Endpoint.local_broadcast("addresses:#{address_hash}", "coin_balance", %{ coin_balance: rendered_coin_balance }) - Endpoint.broadcast("addresses:#{address_hash}", "current_coin_balance", %{ + Endpoint.local_broadcast("addresses:#{address_hash}", "current_coin_balance", %{ coin_balance: coin_balance.value, exchange_rate: exchange_rate.fiat_value, block_number: block_number @@ -644,20 +651,20 @@ defmodule BlockScoutWeb.Notifier do end defp broadcast_address_token_balance_v2(%{address_hash: address_hash, block_number: block_number}) do - Endpoint.broadcast("addresses:#{address_hash}", "token_balance", %{ + Endpoint.local_broadcast("addresses:#{address_hash}", "token_balance", %{ block_number: block_number }) end # TODO: delete this function when old UI becomes deprecated defp broadcast_address_token_balance_v1(%{address_hash: address_hash, block_number: block_number}) do - Endpoint.broadcast("addresses_old:#{address_hash}", "token_balance", %{ + Endpoint.local_broadcast("addresses_old:#{address_hash}", "token_balance", %{ block_number: block_number }) end defp broadcast_balance_v2(%Address{hash: address_hash} = address, exchange_rate) do - Endpoint.broadcast("addresses:#{address_hash}", "balance", %{ + Endpoint.local_broadcast("addresses:#{address_hash}", "balance", %{ balance: address.fetched_coin_balance.value, block_number: address.fetched_coin_balance_block_number, exchange_rate: exchange_rate.fiat_value @@ -666,7 +673,7 @@ defmodule BlockScoutWeb.Notifier do # TODO: delete this function when old UI becomes deprecated defp broadcast_balance_v1(%Address{hash: address_hash} = address, exchange_rate) do - Endpoint.broadcast( + Endpoint.local_broadcast( "addresses_old:#{address_hash}", "balance_update", %{ @@ -691,12 +698,12 @@ defmodule BlockScoutWeb.Notifier do average_block_time = AverageBlockTime.average_block_time() # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("blocks_old:new_block", "new_block", %{ + Endpoint.local_broadcast("blocks_old:new_block", "new_block", %{ block: preloaded_block, average_block_time: average_block_time }) - Endpoint.broadcast("blocks_old:#{to_string(block.miner_hash)}", "new_block", %{ + Endpoint.local_broadcast("blocks_old:#{to_string(block.miner_hash)}", "new_block", %{ block: preloaded_block, average_block_time: average_block_time }) @@ -710,8 +717,8 @@ defmodule BlockScoutWeb.Notifier do }) } - Endpoint.broadcast("blocks:new_block", "new_block", block_params_v2) - Endpoint.broadcast("blocks:#{to_string(block.miner_hash)}", "new_block", block_params_v2) + Endpoint.local_broadcast("blocks:new_block", "new_block", block_params_v2) + Endpoint.local_broadcast("blocks:#{to_string(block.miner_hash)}", "new_block", block_params_v2) end defp maybe_preload_enrichment_for_broadcast(block) do @@ -761,7 +768,7 @@ defmodule BlockScoutWeb.Notifier do Enum.split_with(rewards, fn reward -> reward.address_type == :emission_funds end) Enum.each(validator_rewards, fn reward -> - Endpoint.broadcast("rewards:#{to_string(reward.address_hash)}", "new_reward", %{reward: 1}) + Endpoint.local_broadcast("rewards:#{to_string(reward.address_hash)}", "new_reward", %{reward: 1}) end) # TODO: delete duplicated event when old UI becomes deprecated @@ -780,7 +787,7 @@ defmodule BlockScoutWeb.Notifier do relevant_rewards |> Repo.preload([:address, :block]) |> Enum.each(fn reward -> - Endpoint.broadcast("rewards_old:#{to_string(reward.address_hash)}", "new_reward", %{ + Endpoint.local_broadcast("rewards_old:#{to_string(reward.address_hash)}", "new_reward", %{ emission_funds: emission_reward, validator: reward }) @@ -789,13 +796,13 @@ defmodule BlockScoutWeb.Notifier do end defp broadcast_internal_transaction(internal_transaction) do - Endpoint.broadcast("addresses_old:#{internal_transaction.from_address_hash}", "internal_transaction", %{ + Endpoint.local_broadcast("addresses_old:#{internal_transaction.from_address_hash}", "internal_transaction", %{ address: internal_transaction.from_address, internal_transaction: internal_transaction }) if internal_transaction.to_address_hash != internal_transaction.from_address_hash do - Endpoint.broadcast("addresses_old:#{internal_transaction.to_address_hash}", "internal_transaction", %{ + Endpoint.local_broadcast("addresses_old:#{internal_transaction.to_address_hash}", "internal_transaction", %{ address: internal_transaction.to_address, internal_transaction: internal_transaction }) @@ -820,7 +827,7 @@ defmodule BlockScoutWeb.Notifier do defp broadcast_transactions_websocket_v2_inner(transactions, default_channel, event) do if not Enum.empty?(transactions) do - Endpoint.broadcast(default_channel, event, %{ + Endpoint.local_broadcast(default_channel, event, %{ String.to_existing_atom(event) => Enum.count(transactions) }) end @@ -892,19 +899,19 @@ defmodule BlockScoutWeb.Notifier do end defp broadcast_transaction(transaction, transaction_channel, event) do - Endpoint.broadcast("transactions_old:#{transaction.hash}", "collated", %{}) + Endpoint.local_broadcast("transactions_old:#{transaction.hash}", "collated", %{}) - Endpoint.broadcast(transaction_channel, event, %{ + Endpoint.local_broadcast(transaction_channel, event, %{ transaction: transaction }) - Endpoint.broadcast("addresses_old:#{transaction.from_address_hash}", event, %{ + Endpoint.local_broadcast("addresses_old:#{transaction.from_address_hash}", event, %{ address: transaction.from_address, transaction: transaction }) if transaction.to_address_hash != transaction.from_address_hash do - Endpoint.broadcast("addresses_old:#{transaction.to_address_hash}", event, %{ + Endpoint.local_broadcast("addresses_old:#{transaction.to_address_hash}", event, %{ address: transaction.to_address, transaction: transaction }) @@ -919,7 +926,7 @@ defmodule BlockScoutWeb.Notifier do relevant_token_transfers -> for {token_contract_address_hash, token_transfers} <- Enum.group_by(relevant_token_transfers, fn tt -> to_string(tt.token_contract_address_hash) end) do - Endpoint.broadcast("tokens:#{token_contract_address_hash}", "token_transfer", %{ + Endpoint.local_broadcast("tokens:#{token_contract_address_hash}", "token_transfer", %{ token_transfer: Enum.count(token_transfers) }) end @@ -952,18 +959,18 @@ defmodule BlockScoutWeb.Notifier do end defp broadcast_token_transfer(token_transfer, event) do - Endpoint.broadcast("addresses_old:#{token_transfer.from_address_hash}", event, %{ + Endpoint.local_broadcast("addresses_old:#{token_transfer.from_address_hash}", event, %{ address: token_transfer.from_address, token_transfer: token_transfer }) - Endpoint.broadcast("tokens_old:#{token_transfer.token_contract_address_hash}", event, %{ + Endpoint.local_broadcast("tokens_old:#{token_transfer.token_contract_address_hash}", event, %{ address: token_transfer.token_contract_address_hash, token_transfer: token_transfer }) if token_transfer.to_address_hash != token_transfer.from_address_hash do - Endpoint.broadcast("addresses_old:#{token_transfer.to_address_hash}", event, %{ + Endpoint.local_broadcast("addresses_old:#{token_transfer.to_address_hash}", event, %{ address: token_transfer.to_address, token_transfer: token_transfer }) @@ -982,7 +989,7 @@ defmodule BlockScoutWeb.Notifier do grouped = Map.merge(grouped_by_to, grouped_by_from, fn _k, v1, v2 -> Enum.uniq_by(v1 ++ v2, uniq_function) end) for {address_hash, elements} <- grouped do - Endpoint.broadcast("addresses:#{address_hash}", event, %{map_key => elements}) + Endpoint.local_broadcast("addresses:#{address_hash}", event, %{map_key => elements}) end end @@ -997,8 +1004,8 @@ defmodule BlockScoutWeb.Notifier do defp broadcast_automatic_verification_events(event, address_hash) do log_broadcast_smart_contract_event(address_hash, event) # TODO: delete duplicated event when old UI becomes deprecated - Endpoint.broadcast("addresses_old:#{to_string(address_hash)}", to_string(event), %{}) - Endpoint.broadcast("addresses:#{to_string(address_hash)}", to_string(event), %{}) + Endpoint.local_broadcast("addresses_old:#{to_string(address_hash)}", to_string(event), %{}) + Endpoint.local_broadcast("addresses:#{to_string(address_hash)}", to_string(event), %{}) end defp has_subscribers?(topic) when is_binary(topic) do diff --git a/apps/block_scout_web/lib/block_scout_web/notifiers/arbitrum.ex b/apps/block_scout_web/lib/block_scout_web/notifiers/arbitrum.ex index b6bf0149aa1..42d135d4674 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifiers/arbitrum.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifiers/arbitrum.ex @@ -13,14 +13,14 @@ defmodule BlockScoutWeb.Notifiers.Arbitrum do batches |> Enum.sort_by(& &1.number, :asc) |> Enum.each(fn batch -> - Endpoint.broadcast("arbitrum:new_batch", "new_arbitrum_batch", %{ + Endpoint.local_broadcast("arbitrum:new_batch", "new_arbitrum_batch", %{ batch: ArbitrumView.render_base_info_for_batch(batch) }) end) end def handle_event({:chain_event, :new_messages_to_arbitrum_amount, :realtime, new_messages_amount}) do - Endpoint.broadcast("arbitrum:new_messages_to_rollup_amount", "new_messages_to_rollup_amount", %{ + Endpoint.local_broadcast("arbitrum:new_messages_to_rollup_amount", "new_messages_to_rollup_amount", %{ new_messages_to_rollup_amount: new_messages_amount }) end diff --git a/apps/block_scout_web/lib/block_scout_web/notifiers/optimism.ex b/apps/block_scout_web/lib/block_scout_web/notifiers/optimism.ex index 0e9648bff51..9043a8b1cce 100644 --- a/apps/block_scout_web/lib/block_scout_web/notifiers/optimism.ex +++ b/apps/block_scout_web/lib/block_scout_web/notifiers/optimism.ex @@ -12,7 +12,7 @@ defmodule BlockScoutWeb.Notifiers.Optimism do batches |> Enum.sort_by(& &1.number, :asc) |> Enum.each(fn batch -> - Endpoint.broadcast("optimism:new_batch", "new_optimism_batch", %{ + Endpoint.local_broadcast("optimism:new_batch", "new_optimism_batch", %{ batch: batch }) end) @@ -22,7 +22,7 @@ defmodule BlockScoutWeb.Notifiers.Optimism do deposits_count = Enum.count(deposits) if deposits_count > 0 do - Endpoint.broadcast("optimism:new_deposits", "new_optimism_deposits", %{ + Endpoint.local_broadcast("optimism:new_deposits", "new_optimism_deposits", %{ deposits: deposits_count }) end diff --git a/apps/block_scout_web/test/block_scout_web/notifier_broadcast_test.exs b/apps/block_scout_web/test/block_scout_web/notifier_broadcast_test.exs new file mode 100644 index 00000000000..3589a3fe26e --- /dev/null +++ b/apps/block_scout_web/test/block_scout_web/notifier_broadcast_test.exs @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule BlockScoutWeb.NotifierBroadcastTest do + use BlockScoutWeb.SubscriptionCase, + async: false + + alias BlockScoutWeb.Notifier + + defmodule ClusterBroadcastSpy do + @moduledoc false + + # Takes the place of the PubSub adapter, whose `broadcast/4` sends a broadcast + # to the other nodes of the cluster, and reports such broadcasts to the test + # process given as the adapter name. + def broadcast(test_pid, topic, message, _dispatcher) do + send(test_pid, {:cluster_broadcast, topic, message}) + :ok + end + + def node_name(_test_pid), do: node() + end + + setup do + {:ok, {adapter, adapter_name, dispatcher}} = Registry.meta(BlockScoutWeb.PubSub, :pubsub) + Registry.put_meta(BlockScoutWeb.PubSub, :pubsub, {ClusterBroadcastSpy, self(), dispatcher}) + on_exit(fn -> Registry.put_meta(BlockScoutWeb.PubSub, :pubsub, {adapter, adapter_name, dispatcher}) end) + + :ok + end + + # Makes sure that the refutations in the other tests cannot pass vacuously. + test "the spy detects a broadcast sent to the cluster" do + topic = "test:#{System.unique_integer([:positive])}" + + @endpoint.broadcast(topic, "event", %{}) + + assert_received {:cluster_broadcast, ^topic, _} + end + + test "sends channel updates to the subscribers of the current node only" do + address = insert(:address) + topic = "addresses:#{address.hash}" + @endpoint.subscribe(topic) + + Notifier.handle_event({:chain_event, :changed_bytecode, :on_demand, [address.hash]}) + + assert_receive %Phoenix.Socket.Broadcast{topic: ^topic, event: "changed_bytecode"} + refute_received {:cluster_broadcast, ^topic, _} + end + + test "publishes GraphQL subscription updates on the current node only", %{socket: socket} do + transaction = insert(:transaction) + token_transfer = insert(:token_transfer, transaction: transaction) + token_contract_address_hash = to_string(token_transfer.token_contract_address_hash) + + subscription = """ + subscription ($hash: AddressHash!) { + token_transfers(token_contract_address_hash: $hash) { + transaction_hash + } + } + """ + + ref = push_doc(socket, subscription, variables: %{"hash" => token_contract_address_hash}) + assert_reply(ref, :ok, %{subscriptionId: subscription_id}) + + Notifier.handle_event({:chain_event, :token_transfers, :realtime, [token_transfer]}) + + assert_push("subscription:data", push) + + assert push == %{ + result: %{ + data: %{ + "token_transfers" => [%{"transaction_hash" => to_string(token_transfer.transaction_hash)}] + } + }, + subscriptionId: subscription_id + } + + refute_received {:cluster_broadcast, "__absinthe__:proxy:" <> _, %{mutation_result: [^token_transfer]}} + end +end From 982fb44d95b6171a1f4d347c00ac7a72040a47be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:50:42 +0300 Subject: [PATCH 42/52] chore(deps): bump redix from 1.8.2 to 1.9.1 (#14823) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 764c84f7276..222323e7649 100644 --- a/mix.lock +++ b/mix.lock @@ -156,7 +156,7 @@ "que": {:hex, :que, "0.12.0", "4629c156036b8a3bf9ffdd79aa4069ab414822144d3ef031b5cc61fc805942d3", [:mix], [{:ex_utils, "~> 0.1", [hex: :ex_utils, repo: "hexpm", optional: false]}, {:memento, "~> 0.6", [hex: :memento, repo: "hexpm", optional: false]}], "hexpm", "906651d60b1036840f1afacc0036541583eda9ba4b0b03f322530a1c84100507"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"}, - "redix": {:hex, :redix, "1.8.2", "7f0bb263c7c74b62ae5db3922146cbaa5d953993f8d678fce28ab9a24b12cd79", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c6912716ae2f9aeb4f867383fc9b8522317528c6d183daefdc12c9cb7696dbd9"}, + "redix": {:hex, :redix, "1.9.1", "99473ccc538b97eb169beea133e788c236850e84ec3bcc785ef414ffca9fb2e2", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b873953d06f893f88bb92a515ff9f9c0b65662576de27fb810e76cff3609ebab"}, "remote_ip": {:hex, :remote_ip, "1.2.0", "fb078e12a44414f4cef5a75963c33008fe169b806572ccd17257c208a7bc760f", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2ff91de19c48149ce19ed230a81d377186e4412552a597d6a5137373e5877cb7"}, "req": {:hex, :req, "0.5.6", "8fe1eead4a085510fe3d51ad854ca8f20a622aae46e97b302f499dfb84f726ac", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cfaa8e720945d46654853de39d368f40362c2641c4b2153c886418914b372185"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"}, From 4b99cc9416979261c0109853532bc348d1d58260 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:51:04 +0300 Subject: [PATCH 43/52] chore(deps): bump mox from 1.1.0 to 1.3.2 (#14825) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- apps/ethereum_jsonrpc/mix.exs | 2 +- apps/explorer/mix.exs | 2 +- apps/indexer/mix.exs | 2 +- mix.lock | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 9408864f38d..ed0e59b226e 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -70,7 +70,7 @@ defmodule EthereumJSONRPC.MixProject do {:logger_file_backend, "~> 0.0.10"}, {:logger_json, "~> 7.0"}, # Mocking `EthereumJSONRPC.Transport` and `EthereumJSONRPC.HTTP` so we avoid hitting real chains for local testing - {:mox, "~> 1.1.0", only: [:test]}, + {:mox, "~> 1.3.2", only: [:test]}, {:prometheus_ex, "~> 5.1.0", override: true}, # Tracing {:spandex, "~> 3.0"}, diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index 0d73ae29c1e..eef800239da 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -93,7 +93,7 @@ defmodule Explorer.Mixfile do {:logger_json, "~> 7.0"}, {:math, "~> 0.7.0"}, {:mock, "~> 0.3.0", only: [:test], runtime: false}, - {:mox, "~> 1.1.0"}, + {:mox, "~> 1.3.2"}, {:poison, "~> 5.0.0"}, {:nimble_csv, "~> 1.1"}, {:postgrex, ">= 0.0.0"}, diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index 9d7b8454939..8d2fb44980a 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -66,7 +66,7 @@ defmodule Indexer.MixProject do {:logger_file_backend, "~> 0.0.10"}, {:logger_json, "~> 7.0"}, # Mocking `EthereumJSONRPC.Transport`, so we avoid hitting real chains for local testing - {:mox, "~> 1.1.0"}, + {:mox, "~> 1.3.2"}, {:prometheus_ex, "~> 5.1.0", override: true}, # Tracing {:spandex, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 222323e7649..5802ec8ebfe 100644 --- a/mix.lock +++ b/mix.lock @@ -115,7 +115,7 @@ "mint": {:hex, :mint, "1.10.0", "85af3353bfc504f5bdfe494bd92b8490f87a306dc659ee1ad0af435107e898dc", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "8b16fb72aaa7531d206a1f05e4cc85509ba531ccec7a17a22736c9c95cbb24d1"}, "mnemoniac": {:hex, :mnemoniac, "0.1.5", "d10bf14abb94adc1a98f6cee4d81d197e1b3986cad8ec43b2e21d2cd9f4d99b6", [:mix], [], "hexpm", "518ae4b6ddda5285a5c02bf83e1eb6ce1af7e4d65d3f7f5bff3270689a00bcc3"}, "mock": {:hex, :mock, "0.3.9", "10e44ad1f5962480c5c9b9fa779c6c63de9bd31997c8e04a853ec990a9d841af", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "9e1b244c4ca2551bb17bb8415eed89e40ee1308e0fbaed0a4fdfe3ec8a4adbd3"}, - "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "mox": {:hex, :mox, "1.3.2", "f34ca4331b1cce3125609c6de674739fe5f3df0d68df25510d64b6a94b2246de", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "97918a185e727f3128a826f01827b5b8168e1b815bda19eb4192c3ffdb6a8054"}, "msgpax": {:hex, :msgpax, "2.4.0", "4647575c87cb0c43b93266438242c21f71f196cafa268f45f91498541148c15d", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "ca933891b0e7075701a17507c61642bf6e0407bb244040d5d0a58597a06369d2"}, "nimble_csv": {:hex, :nimble_csv, "1.3.0", "b7f998dc62b222bce9596e46f028c7a5af04cb5dde6df2ea197c583227c54971", [:mix], [], "hexpm", "41ccdc18f7c8f8bb06e84164fc51635321e80d5a3b450761c4997d620925d619"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, From 5221aff987b01e4929dcf1baee95b44584ac71fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:51:29 +0300 Subject: [PATCH 44/52] chore(deps): bump hammer_backend_redis from 7.1.1 to 7.2.0 (#14826) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 5802ec8ebfe..eacff3db9de 100644 --- a/mix.lock +++ b/mix.lock @@ -84,7 +84,7 @@ "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "hammer": {:hex, :hammer, "7.5.0", "7f9621ebb137f94d1bce60175cf2dd1be26e6e5c40f561124307fdd053fe8693", [:mix], [{:igniter, "~> 0.8", [hex: :igniter, repo: "hexpm", optional: true]}], "hexpm", "cb6fb12e5b5a7f1631d5753e9c45493b225bc11f4f09f11ec7f28140c9f0905f"}, - "hammer_backend_redis": {:hex, :hammer_backend_redis, "7.1.1", "979362db9e6f30b9b71f671b28550b5192cd5c2614e60da6ee9347d8085b962b", [:mix], [{:hammer, "~> 7.0", [hex: :hammer, repo: "hexpm", optional: false]}, {:redix, "~> 1.5", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "717bb15f14e709dcae67ba67f90229d2aaee2b6a9bfcc2ca96212f414cd474dc"}, + "hammer_backend_redis": {:hex, :hammer_backend_redis, "7.2.0", "90cec559a78af3128c7f73788b19d5a1040b1eba7fabaf82ac7a23383ce68c6a", [:mix], [{:hammer, "~> 7.0", [hex: :hammer, repo: "hexpm", optional: false]}, {:redix, "~> 1.5", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm", "dd33cc70ae0678602f363e80b302d7454722d12d6bef8f9db5cbb858bb319248"}, "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "httpoison": {:hex, :httpoison, "2.3.0", "10eef046405bc44ba77dc5b48957944df8952cc4966364b3cf6aa71dce6de587", [:mix], [{:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "d388ee70be56d31a901e333dbcdab3682d356f651f93cf492ba9f06056436a2c"}, From bcc8d173899c4eba307976561657081f05bd70fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:00:19 +0300 Subject: [PATCH 45/52] chore(deps-dev): bump cowboy from 2.18.0 to 2.19.0 (#14824) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index eacff3db9de..3465bcad2c2 100644 --- a/mix.lock +++ b/mix.lock @@ -29,9 +29,9 @@ "con_cache": {:hex, :con_cache, "1.1.1", "9f47a68dfef5ac3bbff8ce2c499869dbc5ba889dadde6ac4aff8eb78ddaf6d82", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1def4d1bec296564c75b5bbc60a19f2b5649d81bfa345a2febcc6ae380e8ae15"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "cors_plug": {:hex, :cors_plug, "3.0.3", "7c3ac52b39624bc616db2e937c282f3f623f25f8d550068b6710e58d04a0e330", [:mix], [{:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "3f2d759e8c272ed3835fab2ef11b46bddab8c1ab9528167bd463b6452edf830d"}, - "cowboy": {:hex, :cowboy, "2.18.0", "bff388eb4d6356cb3f88c26e65b515976bf04b401b805d31550c3e60eba8fe18", [:make, :rebar3], [{:cowlib, ">= 2.19.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "62d0b26abcf455054972b0da242389c69d5982ce5914afb8c344517f667b9600"}, + "cowboy": {:hex, :cowboy, "2.19.0", "78b9d92d25a23e56d7341040b67946c04067cd77de966e2372865f673a8e6f59", [:make, :rebar3], [{:cowlib, ">= 2.20.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "986dae81f99fcb78ef2d8efc21d738cee189410840eeaf32eca84ada81dcf6d4"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.19.0", "c9d11c9d035472e27a740c9f327786c61ed209269b4be0260d59d3ec07b8949f", [:make, :rebar3], [], "hexpm", "6dc66e3135b229193ea4dcb14294e79520c923d391315c9c962ef0b4bea72356"}, + "cowlib": {:hex, :cowlib, "2.20.0", "bb525377ba634cd6d68bac7bff5d98571f738806139d335daca827717c5dc172", [:make, :rebar3], [], "hexpm", "7d41a0dd2c093041ff3779ac5fe8a1585a68ec7cb2dd1de0536bdd2452fd7ba1"}, "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, "csv": {:hex, :csv, "2.5.0", "c47b5a5221bf2e56d6e8eb79e77884046d7fd516280dc7d9b674251e0ae46246", [:mix], [{:parallel_stream, "~> 1.0.4 or ~> 1.1.0", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "e821f541487045c7591a1963eeb42afff0dfa99bdcdbeb3410795a2f59c77d34"}, "dataloader": {:hex, :dataloader, "2.0.2", "c45075e0692e68638a315e14f747bd8d7065fb5f38705cf980f62d4cd344401f", [:mix], [{:ecto, ">= 3.4.3 and < 4.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:opentelemetry_process_propagator, "~> 0.2.1 or ~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4c6cabc0b55e96e7de74d14bf37f4a5786f0ab69aa06764a1f39dda40079b098"}, From d1c51cfdfaf10c90ab09820ab1750687d5ae5742 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 13:54:57 +0300 Subject: [PATCH 46/52] perf: reuse preloaded proxy and ABI associations in input decoding (#14827) --- .../lib/explorer/chain/smart_contract.ex | 32 +++++ .../lib/explorer/chain/transaction.ex | 127 +++++++++++++----- .../test/explorer/chain/transaction_test.exs | 109 ++++++++++++++- 3 files changed, 234 insertions(+), 34 deletions(-) diff --git a/apps/explorer/lib/explorer/chain/smart_contract.ex b/apps/explorer/lib/explorer/chain/smart_contract.ex index 590a414530e..9f48cebfa27 100644 --- a/apps/explorer/lib/explorer/chain/smart_contract.ex +++ b/apps/explorer/lib/explorer/chain/smart_contract.ex @@ -1295,6 +1295,38 @@ defmodule Explorer.Chain.SmartContract do [] end + @doc """ + Gets ABIs of the verified smart contracts among the given address hashes in a + single query. + + Only the `abi` column is selected. Address hashes without a verified smart + contract are absent from the result. + + ## Parameters + - `address_hashes`: The address hashes to look up. + - `options`: An optional keyword list of options, such as selecting a specific repository. + + ## Returns + - A map of address hash to ABI. + """ + @spec abis_by_address_hashes([Hash.Address.t()], Keyword.t()) :: %{Hash.Address.t() => abi() | nil} + def abis_by_address_hashes(address_hashes, options \\ []) + + def abis_by_address_hashes([], _options), do: %{} + + def abis_by_address_hashes(address_hashes, options) do + query = + from( + smart_contract in __MODULE__, + where: smart_contract.address_hash in ^address_hashes, + select: {smart_contract.address_hash, smart_contract.abi} + ) + + query + |> Chain.select_repo(options).all() + |> Map.new() + end + @doc """ Composes a query for fetching a smart contract by its address hash. diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index 012fca533bb..3970dfdfd61 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -338,6 +338,7 @@ defmodule Explorer.Chain.Transaction do Hash, InternalTransaction, MethodIdentifier, + SmartContract, SmartContract.Proxy, TokenTransfer, Wei @@ -2485,52 +2486,112 @@ defmodule Explorer.Chain.Transaction do defp decode_remaining_transaction({decoded, _}, _, _, _, _), do: decoded + # Builds %{target address hash => combined proxy + implementations ABI} for the + # to_address / created_contract_address of every transaction. + # + # Associations already present on the address structs (`smart_contract` with + # its `abi`, `proxy_implementations`, and the implementations' + # `smart_contracts`) are reused, so a fully preloaded target address costs no + # query at all. Whatever is missing is fetched in at most two queries: one for + # proxy implementations, one for all outstanding ABIs. defp combine_smart_contract_full_abi_map(transactions, opts) do - # parse unique address hashes of smart-contracts from to_address and created_contract_address properties of the transactions list - unique_to_address_hashes = + # unique target addresses of the transactions list: to_address, or + # created_contract_address for contract creations + target_addresses = transactions |> Enum.flat_map(fn - %__MODULE__{to_address: %Address{hash: hash}} -> [hash] - %__MODULE__{created_contract_address: %Address{hash: hash}} -> [hash] + %__MODULE__{to_address: %Address{} = address} -> [address] + %__MODULE__{created_contract_address: %Address{} = address} -> [address] _ -> [] end) - |> Enum.uniq() + |> Enum.uniq_by(& &1.hash) - # query from the DB proxy implementation objects for those address hashes - multiple_proxy_implementations = - Implementation.get_proxy_implementations_for_multiple_proxies(unique_to_address_hashes, opts) - - # query from the DB address objects with smart_contract preload for all found above proxy and implementation addresses - addresses_with_smart_contracts = - multiple_proxy_implementations - |> Enum.flat_map(fn proxy_implementations -> proxy_implementations.address_hashes end) - |> Enum.concat(unique_to_address_hashes) - |> Chain.hashes_to_addresses( - Keyword.merge(Keyword.take(opts, [:api?]), necessity_by_association: %{smart_contract: :optional}) - ) - |> Enum.into(%{}, &{&1.hash, &1}) + # %{target address hash => implementation address hashes} + implementation_hashes_map = implementation_hashes_by_target_address(target_addresses, opts) - # combine map %{proxy_address_hash => implementation address hashes} - proxy_implementations_map = - multiple_proxy_implementations - |> Enum.into(%{}, &{&1.proxy_address_hash, &1.address_hashes}) + # %{address hash => abi} for every target address and its implementations + abis_map = abis_by_address_hash(target_addresses, implementation_hashes_map, opts) - # combine map %{proxy_address_hash => combined proxy abi} - unique_to_address_hashes - |> Enum.into(%{}, fn to_address_hash -> + Map.new(target_addresses, fn %Address{hash: target_hash} -> full_abi = - [to_address_hash | Map.get(proxy_implementations_map, to_address_hash, [])] - |> Enum.map(&Map.get(addresses_with_smart_contracts, &1)) - |> Enum.flat_map(fn - %{smart_contract: %{abi: abi}} when is_list(abi) -> abi - _ -> [] - end) - |> Enum.filter(&(!is_nil(&1))) + [target_hash | Map.get(implementation_hashes_map, target_hash, [])] + |> Enum.flat_map(&abi_from_map(abis_map, &1)) + |> Enum.reject(&is_nil/1) - {to_address_hash, full_abi} + {target_hash, full_abi} end) end + defp abi_from_map(abis_map, address_hash) do + case Map.get(abis_map, address_hash) do + abi when is_list(abi) -> abi + _ -> [] + end + end + + # Takes implementation address hashes from the preloaded `proxy_implementations` + # association where present and queries them in one go for the rest. + defp implementation_hashes_by_target_address(target_addresses, opts) do + {preloaded_map, not_loaded_hashes} = + Enum.reduce(target_addresses, {%{}, []}, fn + %Address{hash: hash, proxy_implementations: %Implementation{address_hashes: address_hashes}}, + {preloaded_map, not_loaded_hashes} -> + {Map.put(preloaded_map, hash, address_hashes), not_loaded_hashes} + + # a loaded `has_one` without a row: the address is not a proxy + %Address{proxy_implementations: nil}, acc -> + acc + + %Address{hash: hash}, {preloaded_map, not_loaded_hashes} -> + {preloaded_map, [hash | not_loaded_hashes]} + end) + + not_loaded_hashes + |> Implementation.get_proxy_implementations_for_multiple_proxies(opts) + |> Enum.reduce(preloaded_map, &Map.put(&2, &1.proxy_address_hash, &1.address_hashes)) + end + + # Takes ABIs from the preloaded `smart_contract` and + # `proxy_implementations.smart_contracts` associations where present and + # fetches the outstanding ones in a single query. + defp abis_by_address_hash(target_addresses, implementation_hashes_map, opts) do + preloaded_abis_map = + Enum.reduce(target_addresses, %{}, fn %Address{hash: hash} = address, acc -> + acc + |> put_preloaded_abi(hash, address.smart_contract) + |> put_preloaded_implementation_abis(address.proxy_implementations) + end) + + target_addresses + |> Enum.flat_map(fn %Address{hash: hash} -> [hash | Map.get(implementation_hashes_map, hash, [])] end) + |> Enum.uniq() + |> Enum.reject(&Map.has_key?(preloaded_abis_map, &1)) + |> SmartContract.abis_by_address_hashes(opts) + |> Map.merge(preloaded_abis_map) + end + + # a loaded `has_one` without a row: the address is not a verified contract + defp put_preloaded_abi(acc, hash, nil), do: Map.put(acc, hash, []) + + # `abi` is a list only when the smart contract was loaded with its ABI; the + # ABI-less preload (`SmartContract.association_without_abi/0`) leaves it `nil` + # and falls through to the fetch below + defp put_preloaded_abi(acc, hash, %SmartContract{abi: abi}) when is_list(abi), do: Map.put(acc, hash, abi) + + defp put_preloaded_abi(acc, _hash, _not_loaded_or_without_abi), do: acc + + defp put_preloaded_implementation_abis( + acc, + %Implementation{address_hashes: address_hashes, smart_contracts: smart_contracts} + ) + when is_list(smart_contracts) do + smart_contracts_map = Map.new(smart_contracts, &{&1.address_hash, &1}) + + Enum.reduce(address_hashes, acc, &put_preloaded_abi(&2, &1, Map.get(smart_contracts_map, &1))) + end + + defp put_preloaded_implementation_abis(acc, _not_loaded), do: acc + @doc """ Receives as input result of decoded_input_data/5, returns either nil or decoded input in format: {:ok, _identifier, _text, _mapping} """ diff --git a/apps/explorer/test/explorer/chain/transaction_test.exs b/apps/explorer/test/explorer/chain/transaction_test.exs index 22e4faacae5..01c2b8cd5ce 100644 --- a/apps/explorer/test/explorer/chain/transaction_test.exs +++ b/apps/explorer/test/explorer/chain/transaction_test.exs @@ -5,7 +5,8 @@ defmodule Explorer.Chain.TransactionTest do import Mox alias Ecto.Changeset - alias Explorer.Chain.{Address, InternalTransaction, Transaction} + alias Explorer.Chain.{Address, InternalTransaction, SmartContract, Transaction} + alias Explorer.Chain.SmartContract.Proxy.Models.Implementation alias Explorer.{PagingOptions, TestHelper} doctest Transaction @@ -392,6 +393,112 @@ defmodule Explorer.Chain.TransactionTest do end end + describe "decode_transactions/3" do + @decoded_set_10 {:ok, "60fe47b1", "set(uint256 x)", [{"x", "uint256", 10}]} + @abi_sources ["proxy_implementations", "smart_contracts", "addresses"] + + test "decodes input with the implementation ABI of a proxy whose associations are not preloaded" do + transaction = + insert_transaction_to_verified_proxy() + |> Repo.preload(to_address: :smart_contract) + + assert [@decoded_set_10] = Transaction.decode_transactions([transaction], true, api?: true) + end + + test "reuses preloaded proxy implementations and ABIs without querying them again" do + transaction = + insert_transaction_to_verified_proxy() + |> Repo.preload( + to_address: [:smart_contract, Implementation.proxy_implementations_smart_contracts_association()] + ) + + {result, sources} = + with_query_sources(fn -> Transaction.decode_transactions([transaction], true, api?: true) end) + + assert [@decoded_set_10] = result + assert Enum.filter(sources, &(&1 in @abi_sources)) == [] + end + + test "skips the proxy implementations query when the association is preloaded and empty" do + smart_contract = insert(:smart_contract) |> Repo.preload(:address) + input_data = "set(uint)" |> ABI.encode([10]) |> Base.encode16(case: :lower) + + transaction = + :transaction + |> insert(to_address: smart_contract.address, input: "0x" <> input_data) + |> Repo.preload(to_address: [:smart_contract, :proxy_implementations]) + + {result, sources} = + with_query_sources(fn -> Transaction.decode_transactions([transaction], true, api?: true) end) + + assert [@decoded_set_10] = result + assert Enum.filter(sources, &(&1 in @abi_sources)) == [] + end + + test "fetches the ABI when the smart contract was preloaded without it" do + transaction = + insert_transaction_to_verified_proxy() + |> Repo.preload(to_address: [SmartContract.association_without_abi(), :proxy_implementations]) + + {result, sources} = + with_query_sources(fn -> Transaction.decode_transactions([transaction], true, api?: true) end) + + assert [@decoded_set_10] = result + assert "smart_contracts" in sources + refute "proxy_implementations" in sources + end + end + + # a verified proxy whose own ABI lacks `set(uint256)`, delegating to a + # verified implementation whose ABI has it + defp insert_transaction_to_verified_proxy do + proxy_smart_contract = :smart_contract |> insert(abi: []) |> Repo.preload(:address) + implementation_smart_contract = insert(:smart_contract) + + insert(:proxy_implementation, + proxy_address_hash: proxy_smart_contract.address_hash, + proxy_type: :eip1967, + address_hashes: [implementation_smart_contract.address_hash], + names: [implementation_smart_contract.name] + ) + + input_data = "set(uint)" |> ABI.encode([10]) |> Base.encode16(case: :lower) + + insert(:transaction, to_address: proxy_smart_contract.address, input: "0x" <> input_data) + end + + # runs `fun` and returns its result with the `source` tables of every + # query issued by the calling process meanwhile + defp with_query_sources(fun) do + handler_id = {__MODULE__, :query_sources, make_ref()} + + :telemetry.attach_many( + handler_id, + [[:explorer, :repo, :query], [:explorer, :repo, :replica1, :query]], + &__MODULE__.handle_query_event/4, + self() + ) + + try do + {fun.(), collect_query_sources([])} + after + :telemetry.detach(handler_id) + end + end + + defp collect_query_sources(acc) do + receive do + {:query_source, source} -> collect_query_sources([source | acc]) + after + 0 -> Enum.reverse(acc) + end + end + + @doc false + def handle_query_event(_event, _measurements, %{source: source}, test_pid) do + if self() == test_pid, do: send(test_pid, {:query_source, source}) + end + describe "Poison.encode!/1" do test "encodes transaction input" do assert %{ From 075ca1cacc5bd67c34e31191a01518b51e53a5a4 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 13:56:16 +0300 Subject: [PATCH 47/52] perf: preload signed_authorizations only for EIP-7702 transactions (#14828) --- .../api/v2/transaction_controller.ex | 20 +++++------ .../api/v2/transaction_controller_test.exs | 31 +++++++++++++++++ .../lib/explorer/chain/transaction.ex | 33 ++++++++++++++++++ apps/explorer/lib/explorer/eth_rpc.ex | 34 ++++++++++++------- 4 files changed, 95 insertions(+), 23 deletions(-) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex index b24bfd79559..fa45a54e3a8 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/transaction_controller.ex @@ -169,21 +169,20 @@ defmodule BlockScoutWeb.API.V2.TransactionController do """ @spec transaction(Plug.Conn.t(), map()) :: Plug.Conn.t() | {atom(), any()} def transaction(conn, %{transaction_hash_param: transaction_hash_string} = params) do - necessity_by_association_with_actions = - @transaction_necessity_by_association - |> Map.put(:signed_authorizations, :optional) - + # `signed_authorizations` are not part of the necessity map on purpose: only + # EIP-7702 (type 4) transactions have any, so they are preloaded below via + # `Transaction.preload_signed_authorizations/2` once the type is known. necessity_by_association = case Application.get_env(:explorer, :chain_type) do :zksync -> - necessity_by_association_with_actions + @transaction_necessity_by_association |> Map.put(:zksync_batch, :optional) |> Map.put(:zksync_commit_transaction, :optional) |> Map.put(:zksync_prove_transaction, :optional) |> Map.put(:zksync_execute_transaction, :optional) :arbitrum -> - necessity_by_association_with_actions + @transaction_necessity_by_association |> Map.put(:arbitrum_batch, :optional) |> Map.put(:arbitrum_commitment_transaction, :optional) |> Map.put(:arbitrum_confirmation_transaction, :optional) @@ -191,14 +190,14 @@ defmodule BlockScoutWeb.API.V2.TransactionController do |> Map.put(:arbitrum_message_from_l2, :optional) :suave -> - necessity_by_association_with_actions + @transaction_necessity_by_association |> Map.put(:logs, :optional) |> Map.put([execution_node: :names], :optional) |> Map.put([wrapped_to_address: :names], :optional) :eden -> Map.put( - necessity_by_association_with_actions, + @transaction_necessity_by_association, [ fee_payer_address: [ :scam_badge, @@ -211,7 +210,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do ) _ -> - necessity_by_association_with_actions + @transaction_necessity_by_association end options = @@ -219,9 +218,10 @@ defmodule BlockScoutWeb.API.V2.TransactionController do |> Keyword.merge(@api_true) with {:ok, transaction, _transaction_hash} <- validate_transaction(transaction_hash_string, params, options), + preloaded <- Transaction.preload_signed_authorizations(transaction, @api_true), preloaded <- Chain.preload_token_transfers( - transaction, + preloaded, @token_transfers_in_transaction_necessity_by_association, @api_true |> fetch_scam_token_toggle(conn) ), diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs index 4710a54773f..be79004eb44 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/transaction_controller_test.exs @@ -224,6 +224,37 @@ defmodule BlockScoutWeb.API.V2.TransactionControllerTest do end describe "/transactions/{transaction_hash}" do + test "returns authorization list for EIP-7702 (type 4) transaction", %{conn: conn} do + transaction = + :transaction + |> insert(type: 4) + |> with_block() + + authorization = insert(:signed_authorization, transaction: transaction, index: 0) + + request = get(conn, "/api/v2/transactions/#{transaction.hash}") + response = json_response(request, 200) + + assert [%{"address_hash" => address_hash, "authority" => authority, "nonce" => "0"}] = + response["authorization_list"] + + assert address_hash == Address.checksum(authorization.address) + assert authority == Address.checksum(authorization.authority) + assert "set_code_transaction" in response["transaction_types"] + end + + test "returns empty authorization list for non-EIP-7702 transaction", %{conn: conn} do + transaction = + :transaction + |> insert(type: 2) + |> with_block() + + request = get(conn, "/api/v2/transactions/#{transaction.hash}") + response = json_response(request, 200) + + assert response["authorization_list"] == [] + end + test "get token-transfers with ok reputation", %{conn: conn} do init_value = Application.get_env(:block_scout_web, :hide_scam_addresses) Application.put_env(:block_scout_web, :hide_scam_addresses, true) diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index 3970dfdfd61..7b6ccdbc0a0 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -2398,6 +2398,39 @@ defmodule Explorer.Chain.Transaction do ) end + # EIP-7702 set code transaction type: the only type carrying an authorization list. + @set_code_transaction_type 4 + + @doc """ + Preloads `signed_authorizations` for a transaction, but only when it can have + any: EIP-7702 set code transactions (type #{@set_code_transaction_type}). + + Every other transaction type gets an empty list without touching the DB, + which saves one query per transaction on the single-transaction endpoints. + A transaction whose `signed_authorizations` are already loaded is returned + as is. + + ## Parameters + - `transaction`: The transaction to preload. + - `options`: Keyword list with `api?` used to pick the replica repo. + + ## Returns + - The transaction with `signed_authorizations` loaded (or set to `[]`). + """ + @spec preload_signed_authorizations(t(), Keyword.t()) :: t() + def preload_signed_authorizations(%__MODULE__{signed_authorizations: signed_authorizations} = transaction, _options) + when is_list(signed_authorizations) do + transaction + end + + def preload_signed_authorizations(%__MODULE__{type: @set_code_transaction_type} = transaction, options) do + Chain.select_repo(options).preload(transaction, :signed_authorizations) + end + + def preload_signed_authorizations(%__MODULE__{} = transaction, _options) do + %{transaction | signed_authorizations: []} + end + @doc """ Receives as input list of transactions and returns decoded_input_data Where diff --git a/apps/explorer/lib/explorer/eth_rpc.ex b/apps/explorer/lib/explorer/eth_rpc.ex index ad222a21ae1..80d71ae2650 100644 --- a/apps/explorer/lib/explorer/eth_rpc.ex +++ b/apps/explorer/lib/explorer/eth_rpc.ex @@ -853,13 +853,17 @@ defmodule Explorer.EthRPC do """ @spec eth_get_transaction_by_hash(String.t()) :: {:ok, map() | nil} | {:error, String.t()} def eth_get_transaction_by_hash(transaction_hash_string) do - necessity_by_association = - %{signed_authorizations: :optional} - |> Map.merge(chain_type_transaction_necessity_by_association()) + # `signed_authorizations` are preloaded separately, only for EIP-7702 + # (type 4) transactions, to skip the query for every other type. + render_func = fn transaction -> + transaction + |> Transaction.preload_signed_authorizations(api?: true) + |> render_transaction() + end - validate_and_render_transaction(transaction_hash_string, &render_transaction/1, + validate_and_render_transaction(transaction_hash_string, render_func, api?: true, - necessity_by_association: necessity_by_association + necessity_by_association: chain_type_transaction_necessity_by_association() ) end @@ -995,25 +999,29 @@ defmodule Explorer.EthRPC do defp maybe_add_access_list(props, _transaction), do: props - defp maybe_add_chain_type_extra_transaction_info_properties(props, %{beacon_blob_transaction: beacon_blob_transaction}) do - if Application.get_env(:explorer, :chain_type) == :ethereum && beacon_blob_transaction do + # The `beacon_blob_transaction` association exists on the `Transaction` struct + # only when the chain type is `:ethereum` at compile time, so these clauses + # are compiled in under the same condition; otherwise dialyzer reports a + # pattern that can never match. + if @chain_type == :ethereum do + defp maybe_add_chain_type_extra_transaction_info_properties(props, %{ + beacon_blob_transaction: beacon_blob_transaction + }) + when not is_nil(beacon_blob_transaction) do props |> Map.put("maxFeePerBlobGas", Helper.decimal_to_hex(beacon_blob_transaction.max_fee_per_blob_gas)) |> Map.put("blobVersionedHashes", beacon_blob_transaction.blob_versioned_hashes) - else - props end end defp maybe_add_chain_type_extra_transaction_info_properties(props, _transaction), do: props - defp maybe_add_chain_type_extra_receipt_properties(props, %{beacon_blob_transaction: beacon_blob_transaction}) do - if Application.get_env(:explorer, :chain_type) == :ethereum && beacon_blob_transaction do + if @chain_type == :ethereum do + defp maybe_add_chain_type_extra_receipt_properties(props, %{beacon_blob_transaction: beacon_blob_transaction}) + when not is_nil(beacon_blob_transaction) do props |> Map.put("blobGasPrice", Helper.decimal_to_hex(beacon_blob_transaction.blob_gas_price)) |> Map.put("blobGasUsed", Helper.decimal_to_hex(beacon_blob_transaction.blob_gas_used)) - else - props end end From 80113f9c596c3f035fe7bcd619bf9f33c32eb15f Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 14:06:46 +0300 Subject: [PATCH 48/52] perf: Bound and deduplicate on-demand token total supply fetcher (#14811) --- .../controllers/api/v2/token_controller.ex | 2 +- .../controllers/tokens/transfer_controller.ex | 3 +- .../fetcher/on_demand/token_total_supply.ex | 242 ++++++++++++++-- .../on_demand/token_total_supply_test.exs | 267 ++++++++++++++++++ config/runtime.exs | 7 + cspell.json | 1 + docker-compose/envs/common-blockscout.env | 3 + 7 files changed, 499 insertions(+), 26 deletions(-) create mode 100644 apps/indexer/test/indexer/fetcher/on_demand/token_total_supply_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index 09c0a1f3447..b6fa0396407 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -101,7 +101,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do with {:format, {:ok, address_hash}} <- {:format, Chain.string_to_address_hash(address_hash_string)}, {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params), {:not_found, {:ok, token}} <- {:not_found, Chain.token_from_address_hash(address_hash, @token_options)} do - TokenTotalSupplyOnDemand.trigger_fetch(ip, address_hash) + TokenTotalSupplyOnDemand.trigger_fetch(ip, token) conn |> token_response(token, address_hash) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex index 941835a8624..ab4f7b7ec8b 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/tokens/transfer_controller.ex @@ -92,13 +92,14 @@ defmodule BlockScoutWeb.Tokens.TransferController do with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string), {:ok, token} <- Chain.token_from_address_hash(address_hash, options), {:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do + TokenTotalSupplyOnDemand.trigger_fetch(ip, token) + render( conn, "index.html", counters_path: token_path(conn, :token_counters, %{"id" => Address.checksum(address_hash)}), current_path: Controller.current_full_path(conn), token: token, - token_total_supply_status: TokenTotalSupplyOnDemand.trigger_fetch(ip, address_hash), tags: get_address_tags(address_hash, current_user(conn)) ) else diff --git a/apps/indexer/lib/indexer/fetcher/on_demand/token_total_supply.ex b/apps/indexer/lib/indexer/fetcher/on_demand/token_total_supply.ex index 80dc1e4b539..4b19ecaf524 100644 --- a/apps/indexer/lib/indexer/fetcher/on_demand/token_total_supply.ex +++ b/apps/indexer/lib/indexer/fetcher/on_demand/token_total_supply.ex @@ -1,28 +1,63 @@ # SPDX-License-Identifier: LicenseRef-Blockscout defmodule Indexer.Fetcher.OnDemand.TokenTotalSupply do @moduledoc """ - Ensures that we have a reasonably up to date token supply. + Ensures that we have a reasonably up to date token supply. + + The fetch is triggered from API request handlers. To keep the request path + cheap and the GenServer mailbox bounded: + + * the staleness check runs in the caller, using the already loaded + `t:Explorer.Chain.Token.t/0`, before anything is enqueued; + * an ETS table records tokens that are in flight or failed recently, so a + token is enqueued at most once per attempt and failing tokens back off; + * the JSON RPC call runs in a task under `__MODULE__.TaskSupervisor`, + never in the GenServer itself, and at most `max_concurrency` tasks run at + a time. Requests arriving above the cap are dropped and can be retried + by a later request. """ use GenServer use Indexer.Fetcher, restart: :permanent + require Logger + alias Explorer.Chain.Cache.BlockNumber alias Explorer.Chain.Events.Publisher - alias Explorer.Chain.{Hash, Token} - alias Explorer.Repo + alias Explorer.Chain.Token alias Explorer.Token.MetadataRetriever alias Explorer.Utility.RateLimiter + @table_name :token_total_supply_on_demand @ttl_in_blocks 1 + @default_max_concurrency 5 + @default_threshold :timer.minutes(5) + @sweep_interval :timer.minutes(1) + + # Non-token entries kept in the same ETS table. + @slots_key :slots + @max_concurrency_key :max_concurrency + + @typep key :: binary() ## Interface - @spec trigger_fetch(String.t() | nil, Hash.Address.t()) :: :ok - def trigger_fetch(caller \\ nil, address_hash) do - case RateLimiter.check_rate(caller, :on_demand) do - :allow -> GenServer.cast(__MODULE__, {:fetch_and_update, address_hash}) - :deny -> :ok + @doc """ + Schedules a total supply refresh for `token` when it is stale. + + Returns `:ok` in all cases. Does nothing when the fetcher is disabled or not + running, when the token is fresh, when the caller is rate limited, or when + the token is already in flight or failed recently. + """ + @spec trigger_fetch(String.t() | nil, Token.t()) :: :ok + def trigger_fetch(caller \\ nil, %Token{} = token) do + with false <- __MODULE__.Supervisor.disabled?(), + true <- table_exists?(), + true <- stale?(token, BlockNumber.get_max()), + :allow <- RateLimiter.check_rate(caller, :on_demand), + true <- claim_and_reserve_slot(key(token)) do + GenServer.cast(__MODULE__, {:fetch, token}) + else + _ -> :ok end end @@ -33,36 +68,195 @@ defmodule Indexer.Fetcher.OnDemand.TokenTotalSupply do end @impl true - def init(init_arg) do - {:ok, init_arg} + def init(init_opts) do + ensure_table() + schedule_sweep() + + max_concurrency = + Keyword.get(init_opts, :max_concurrency) || + Application.get_env(:indexer, __MODULE__, [])[:max_concurrency] || + @default_max_concurrency + + :ets.insert(@table_name, [{@slots_key, 0}, {@max_concurrency_key, max_concurrency}]) + + {:ok, %{running: %{}}} + end + + @impl true + def handle_cast({:fetch, %Token{} = token}, %{running: running} = state) do + key = key(token) + + if key in Map.values(running) do + release_slot() + {:noreply, state} + else + %Task{ref: ref} = Task.Supervisor.async_nolink(__MODULE__.TaskSupervisor, fn -> do_fetch(token) end) + {:noreply, %{state | running: Map.put(running, ref, key)}} + end end @impl true - def handle_cast({:fetch_and_update, address_hash}, state) do - do_fetch(address_hash) + def handle_info({ref, result}, %{running: running} = state) when is_map_key(running, ref) do + Process.demonitor(ref, [:flush]) + {key, running} = Map.pop(running, ref) + finish(key, result) + release_slot() + + {:noreply, %{state | running: running}} + end + + def handle_info({:DOWN, ref, :process, _pid, reason}, %{running: running} = state) when is_map_key(running, ref) do + {key, running} = Map.pop(running, ref) + Logger.error("On-demand token total supply fetch crashed: #{inspect(reason)}") + mark_failed(key) + release_slot() + + {:noreply, %{state | running: running}} + end + + def handle_info(:sweep, %{running: running} = state) do + sweep(Map.values(running)) + schedule_sweep() {:noreply, state} end + def handle_info(_message, state), do: {:noreply, state} + ## Implementation - defp do_fetch(address_hash) when not is_nil(address_hash) do - token = Repo.replica().get_by(Token, contract_address_hash: address_hash) + @spec do_fetch(Token.t()) :: :ok | :error | {:error, Ecto.Changeset.t()} + defp do_fetch(%Token{contract_address_hash: contract_address_hash} = token) do + case MetadataRetriever.get_total_supply_of(to_string(contract_address_hash)) do + %{total_supply: _} = token_params -> + case Token.update(token, token_params) do + {:ok, updated_token} -> + Publisher.broadcast(%{token_total_supply: [updated_token]}, :on_demand) + :ok + + {:error, _changeset} = error -> + error + end + + _ -> + :error + end + end + + defp finish(key, :ok), do: :ets.delete(@table_name, key) + + defp finish(key, error) do + Logger.debug(fn -> "On-demand token total supply fetch failed: #{inspect(error)}" end) + mark_failed(key) + end + + defp mark_failed(key), do: :ets.insert(@table_name, {key, :failed, now_ms()}) + + @spec stale?(Token.t(), non_neg_integer()) :: boolean() + defp stale?(%Token{skip_metadata: true, total_supply_updated_at_block: nil}, _max_block_number), do: false + defp stale?(%Token{total_supply_updated_at_block: nil}, _max_block_number), do: true + + defp stale?(%Token{total_supply_updated_at_block: updated_at_block}, max_block_number), + do: max_block_number - updated_at_block > @ttl_in_blocks + + # Claims the token and a fetch slot. The ETS table is owned by the GenServer, + # so it can disappear between `table_exists?/0` and these calls when the + # fetcher is terminating; treat that as "nothing to do" instead of crashing + # the request process. + @spec claim_and_reserve_slot(key()) :: boolean() + defp claim_and_reserve_slot(key) do + claim(key) and reserve_slot(key) + rescue + ArgumentError -> false + end + + # Marks the token as in flight. Returns `false` when it is already in flight + # or failed less than `threshold` ago. + @spec claim(key()) :: boolean() + defp claim(key) do + now = now_ms() + + case :ets.lookup(@table_name, key) do + [] -> + :ets.insert_new(@table_name, {key, :in_flight, now}) + + [{_key, :in_flight, _at}] -> + false + + [{_key, :failed, _failed_at}] -> + # Atomically replace the failure record only if it is old enough, so + # concurrent callers cannot both claim the same expired token. + expired_before = now - threshold_ms() + + match_spec = [ + {{key, :failed, :"$1"}, [{:"=<", :"$1", expired_before}], [{{{:const, key}, :in_flight, {:const, now}}}]} + ] + + :ets.select_replace(@table_name, match_spec) == 1 + end + end - if (token && !token.skip_metadata && is_nil(token.total_supply_updated_at_block)) or - (token && !is_nil(token.total_supply_updated_at_block) && - BlockNumber.get_max() - token.total_supply_updated_at_block > @ttl_in_blocks) do - token_address_hash_string = to_string(address_hash) + # Reserves one of `max_concurrency` fetch slots. On failure, releases the + # token claim so a later request can retry. + @spec reserve_slot(key()) :: boolean() + defp reserve_slot(key) do + if :ets.update_counter(@table_name, @slots_key, {2, 1}) > max_concurrency() do + release_slot() + :ets.delete(@table_name, key) + false + else + true + end + end + + defp release_slot, do: :ets.update_counter(@table_name, @slots_key, {2, -1}) + + defp max_concurrency do + case :ets.lookup(@table_name, @max_concurrency_key) do + [{_key, max_concurrency}] -> max_concurrency + [] -> @default_max_concurrency + end + end + + # Removes expired failure records and in-flight markers that are not backed + # by a running task (for example, casts lost while the server restarted). + @spec sweep([key()]) :: :ok + defp sweep(running_keys) do + now = now_ms() + threshold = threshold_ms() - token_params = MetadataRetriever.get_total_supply_of(token_address_hash_string) + :ets.foldl( + fn + {key, :failed, failed_at}, acc when now - failed_at >= threshold -> + [key | acc] - if token_params !== %{} do - {:ok, token} = Token.update(token, token_params) + {key, :in_flight, started_at}, acc when now - started_at >= 2 * threshold -> + if key in running_keys, do: acc, else: [key | acc] - Publisher.broadcast(%{token_total_supply: [token]}, :on_demand) - end + _entry, acc -> + acc + end, + [], + @table_name + ) + |> Enum.each(&:ets.delete(@table_name, &1)) + end - :ok + defp ensure_table do + if :ets.whereis(@table_name) == :undefined do + :ets.new(@table_name, [:set, :named_table, :public, read_concurrency: true, write_concurrency: true]) end end + + defp table_exists?, do: :ets.whereis(@table_name) != :undefined + + defp schedule_sweep, do: Process.send_after(self(), :sweep, @sweep_interval) + + defp key(%Token{contract_address_hash: %{bytes: bytes}}), do: bytes + + defp now_ms, do: System.monotonic_time(:millisecond) + + defp threshold_ms do + Application.get_env(:indexer, __MODULE__, [])[:threshold] || @default_threshold + end end diff --git a/apps/indexer/test/indexer/fetcher/on_demand/token_total_supply_test.exs b/apps/indexer/test/indexer/fetcher/on_demand/token_total_supply_test.exs new file mode 100644 index 00000000000..281ec2c4d3e --- /dev/null +++ b/apps/indexer/test/indexer/fetcher/on_demand/token_total_supply_test.exs @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Indexer.Fetcher.OnDemand.TokenTotalSupplyTest do + use EthereumJSONRPC.Case, async: false + use Explorer.DataCase + + import Mox + + alias Explorer.Chain.Cache.BlockNumber + alias Explorer.Chain.Events.Subscriber + alias Explorer.Chain.Token + alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: Fetcher + + @moduletag :capture_log + + @table :token_total_supply_on_demand + @total_supply_selector "0x18160ddd" + @total_supply_hex "0x0000000000000000000000000000000000000000000000000de0b6b3a7640000" + + setup :set_mox_global + setup :verify_on_exit! + + setup do + initial_explorer_retries = Application.get_env(:explorer, :token_functions_reader_max_retries) + initial_fetcher_config = Application.get_env(:indexer, Fetcher) || [] + + Application.put_env(:explorer, :token_functions_reader_max_retries, 1) + Application.put_env(:indexer, Fetcher, Keyword.merge(initial_fetcher_config, threshold: :timer.minutes(5))) + + on_exit(fn -> + Application.put_env(:explorer, :token_functions_reader_max_retries, initial_explorer_retries) + Application.put_env(:indexer, Fetcher, initial_fetcher_config) + end) + + start_supervised!(Fetcher.Supervisor.child_spec([[max_concurrency: 1]])) + Subscriber.to(:token_total_supply, :on_demand) + + %{max_block: BlockNumber.get_max()} + end + + describe "trigger_fetch/2" do + test "fetches, stores and broadcasts total supply of a stale token" do + token = stale_token() + hash = token.contract_address_hash + expect_total_supply(to_string(hash), fn id -> {:ok, [%{id: id, result: @total_supply_hex}]} end) + + assert :ok = Fetcher.trigger_fetch(token) + + assert_receive {:chain_event, :token_total_supply, :on_demand, [%Token{contract_address_hash: ^hash} = updated]}, + 1_000 + + assert Decimal.equal?(updated.total_supply, Decimal.new(1_000_000_000_000_000_000)) + + reloaded = Repo.get_by!(Token, contract_address_hash: hash) + assert Decimal.equal?(reloaded.total_supply, Decimal.new(1_000_000_000_000_000_000)) + assert reloaded.total_supply_updated_at_block == BlockNumber.get_max() + + wait_until(fn -> :ets.lookup(@table, key(token)) == [] end) + assert %{running: running} = :sys.get_state(Fetcher) + assert running == %{} + assert :ets.lookup(@table, :slots) == [{:slots, 0}] + end + + test "does not enqueue a token that is already in flight" do + token = stale_token() + expect_blocking_total_supply(to_string(token.contract_address_hash)) + + assert :ok = Fetcher.trigger_fetch(token) + assert_receive {:rpc, _}, 1_000 + assert_receive {:blocked, task_pid}, 1_000 + assert [{_key, :in_flight, _at}] = :ets.lookup(@table, key(token)) + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + refute_receive {:rpc, _}, 100 + assert %{running: running} = :sys.get_state(Fetcher) + assert map_size(running) == 1 + + send(task_pid, :continue) + assert_receive {:chain_event, :token_total_supply, :on_demand, [%Token{}]}, 1_000 + end + + test "does nothing for a fresh token", %{max_block: max_block} do + token = insert(:token, total_supply_updated_at_block: max_block) + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + assert :ets.lookup(@table, key(token)) == [] + assert %{running: running} = :sys.get_state(Fetcher) + assert running == %{} + refute_receive {:rpc, _}, 100 + end + + test "does nothing for a never fetched token with skip_metadata" do + token = insert(:token, total_supply_updated_at_block: nil, skip_metadata: true) + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + assert :ets.lookup(@table, key(token)) == [] + refute_receive {:rpc, _}, 100 + end + + test "records a failed attempt and backs off until the threshold passes" do + token = stale_token() + hash_string = to_string(token.contract_address_hash) + + expect_total_supply(hash_string, fn id -> + {:ok, [%{id: id, error: %{code: -32015, message: "VM execution error"}}]} + end) + + assert :ok = Fetcher.trigger_fetch(token) + assert_receive {:rpc, ^hash_string}, 1_000 + + wait_until(fn -> match?([{_key, :failed, _at}], :ets.lookup(@table, key(token))) end) + [{_key, :failed, failed_at}] = :ets.lookup(@table, key(token)) + refute_receive {:chain_event, :token_total_supply, :on_demand, _}, 100 + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + assert [{_key, :failed, ^failed_at}] = :ets.lookup(@table, key(token)) + refute_receive {:rpc, _}, 100 + + # Expire the failure record and make sure the token is fetched again. + :ets.insert(@table, {key(token), :failed, failed_at - :timer.minutes(5) - 1}) + expect_total_supply(hash_string, fn id -> {:ok, [%{id: id, result: @total_supply_hex}]} end) + + assert :ok = Fetcher.trigger_fetch(token) + assert_receive {:chain_event, :token_total_supply, :on_demand, [%Token{}]}, 1_000 + wait_until(fn -> :ets.lookup(@table, key(token)) == [] end) + end + + test "drops requests above the concurrency cap and lets them retry later" do + token_a = stale_token() + token_b = stale_token() + hash_b_string = to_string(token_b.contract_address_hash) + expect_blocking_total_supply(to_string(token_a.contract_address_hash)) + + assert :ok = Fetcher.trigger_fetch(token_a) + assert_receive {:blocked, task_pid}, 1_000 + + assert :ok = Fetcher.trigger_fetch(token_b) + sync() + + assert :ets.lookup(@table, key(token_b)) == [] + assert :ets.lookup(@table, :slots) == [{:slots, 1}] + assert %{running: running} = :sys.get_state(Fetcher) + assert map_size(running) == 1 + refute_receive {:rpc, ^hash_b_string}, 100 + + send(task_pid, :continue) + assert_receive {:chain_event, :token_total_supply, :on_demand, [%Token{}]}, 1_000 + wait_until(fn -> map_size(:sys.get_state(Fetcher).running) == 0 end) + assert :ets.lookup(@table, :slots) == [{:slots, 0}] + + expect_total_supply(hash_b_string, fn id -> {:ok, [%{id: id, result: @total_supply_hex}]} end) + assert :ok = Fetcher.trigger_fetch(token_b) + assert_receive {:rpc, ^hash_b_string}, 1_000 + assert_receive {:chain_event, :token_total_supply, :on_demand, [%Token{}]}, 1_000 + end + + test "does nothing when the fetcher is disabled" do + initial = Application.get_env(:indexer, Fetcher.Supervisor) || [] + Application.put_env(:indexer, Fetcher.Supervisor, Keyword.merge(initial, disabled?: true)) + on_exit(fn -> Application.put_env(:indexer, Fetcher.Supervisor, initial) end) + + token = stale_token() + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + assert :ets.lookup(@table, key(token)) == [] + assert %{running: running} = :sys.get_state(Fetcher) + assert running == %{} + end + + test "does not raise when the fetcher is not running" do + token = stale_token() + stop_supervised!(Fetcher.Supervisor) + + assert :ets.whereis(@table) == :undefined + assert :ok = Fetcher.trigger_fetch(token) + end + + test "does not raise when ETS operations fail after the table check" do + token = stale_token() + # Simulates the table being torn down between `table_exists?/0` and the + # ETS calls: the slot counter is gone, so `:ets.update_counter/3` raises. + :ets.delete(@table, :slots) + + assert :ok = Fetcher.trigger_fetch(token) + sync() + + assert %{running: running} = :sys.get_state(Fetcher) + assert running == %{} + end + end + + describe "sweep" do + test "removes expired failures and orphaned in-flight markers" do + now = System.monotonic_time(:millisecond) + threshold = :timer.minutes(5) + + :ets.insert(@table, {"expired_failed", :failed, now - threshold - 1}) + :ets.insert(@table, {"fresh_failed", :failed, now}) + :ets.insert(@table, {"orphaned_in_flight", :in_flight, now - 2 * threshold - 1}) + :ets.insert(@table, {"fresh_in_flight", :in_flight, now}) + + send(Fetcher, :sweep) + sync() + + assert :ets.lookup(@table, "expired_failed") == [] + assert :ets.lookup(@table, "orphaned_in_flight") == [] + assert [{"fresh_failed", :failed, _}] = :ets.lookup(@table, "fresh_failed") + assert [{"fresh_in_flight", :in_flight, _}] = :ets.lookup(@table, "fresh_in_flight") + end + end + + defp stale_token, do: insert(:token, total_supply_updated_at_block: nil) + + defp key(%Token{contract_address_hash: %{bytes: bytes}}), do: bytes + + # Casts are processed in order, so a synchronous call afterwards guarantees + # that all previously sent casts have been handled. + defp sync, do: :sys.get_state(Fetcher) + + defp wait_until(fun, attempts \\ 50) do + cond do + fun.() -> + :ok + + attempts == 0 -> + flunk("condition was not met in time") + + true -> + Process.sleep(20) + wait_until(fun, attempts - 1) + end + end + + defp expect_total_supply(hash_string, reply) do + test_pid = self() + + expect(EthereumJSONRPC.Mox, :json_rpc, fn [%{id: id, method: "eth_call", params: [params, "latest"]}], _opts -> + assert params.data == @total_supply_selector + assert params.to == hash_string + send(test_pid, {:rpc, hash_string}) + reply.(id) + end) + end + + defp expect_blocking_total_supply(hash_string) do + test_pid = self() + + expect(EthereumJSONRPC.Mox, :json_rpc, fn [%{id: id, method: "eth_call", params: [params, "latest"]}], _opts -> + assert params.to == hash_string + send(test_pid, {:rpc, hash_string}) + send(test_pid, {:blocked, self()}) + + receive do + :continue -> {:ok, [%{id: id, result: @total_supply_hex}]} + end + end) + end +end diff --git a/config/runtime.exs b/config/runtime.exs index 0c1217db415..4eeff9818fc 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1217,6 +1217,13 @@ config :indexer, Indexer.Fetcher.OnDemand.TokenBalance, config :indexer, Indexer.Fetcher.OnDemand.TokenBalance.Supervisor, disabled?: ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_TOKEN_BALANCE_ON_DEMAND_FETCHER") +config :indexer, Indexer.Fetcher.OnDemand.TokenTotalSupply, + max_concurrency: ConfigHelper.parse_integer_env_var("TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_CONCURRENCY", 5, min: 1), + threshold: ConfigHelper.parse_time_env_var("TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_THRESHOLD", "5m") + +config :indexer, Indexer.Fetcher.OnDemand.TokenTotalSupply.Supervisor, + disabled?: ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER") + config :indexer, Indexer.Fetcher.OnDemand.CoinBalance, threshold: ConfigHelper.parse_time_env_var("COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD", "1h"), fallback_threshold_in_blocks: 500 diff --git a/cspell.json b/cspell.json index c14c328058d..09edb46dc75 100644 --- a/cspell.json +++ b/cspell.json @@ -291,6 +291,7 @@ "fixidity", "fkey", "Floki", + "foldl", "fontawesome", "fortawesome", "froms", diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index bed0c609861..e19dfed6a6b 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -156,6 +156,9 @@ RELEASE_LINK= # COIN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD= # CONTRACT_CODE_ON_DEMAND_FETCHER_THRESHOLD= # TOKEN_INSTANCE_METADATA_REFETCH_ON_DEMAND_FETCHER_THRESHOLD= +# TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_THRESHOLD= +# TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_CONCURRENCY= +# INDEXER_DISABLE_TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER=false # ADDRESSES_TABS_COUNTERS_TTL=10m # TOKEN_METADATA_UPDATE_INTERVAL=48h # TOKEN_COUNTERS_UPDATE_INTERVAL=3h From fea802fd545ed3c7b9dfd26fc5d86c2fdacd4453 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 18:35:18 +0300 Subject: [PATCH 49/52] perf: cache public address tags and index-driven tags lookup (#14833) --- .../models/get_address_tags.ex | 22 ++- .../models/get_address_tags_test.exs | 42 ++++++ apps/explorer/config/config.exs | 2 + apps/explorer/config/test.exs | 2 + apps/explorer/lib/explorer/application.ex | 142 ++++++++++-------- .../lib/explorer/chain/cache/address_tags.ex | 82 ++++++++++ .../chain/cache/address_tags_test.exs | 83 ++++++++++ config/runtime.exs | 2 + docker-compose/envs/common-blockscout.env | 1 + 9 files changed, 310 insertions(+), 68 deletions(-) create mode 100644 apps/block_scout_web/test/block_scout_web/models/get_address_tags_test.exs create mode 100644 apps/explorer/lib/explorer/chain/cache/address_tags.ex create mode 100644 apps/explorer/test/explorer/chain/cache/address_tags_test.exs diff --git a/apps/block_scout_web/lib/block_scout_web/models/get_address_tags.ex b/apps/block_scout_web/lib/block_scout_web/models/get_address_tags.ex index d97fa17cd61..2a4ad915cfc 100644 --- a/apps/block_scout_web/lib/block_scout_web/models/get_address_tags.ex +++ b/apps/block_scout_web/lib/block_scout_web/models/get_address_tags.ex @@ -9,6 +9,7 @@ defmodule BlockScoutWeb.Models.GetAddressTags do import Explorer.Chain, only: [select_repo: 1] alias Explorer.Account.{TagAddress, WatchlistAddress} + alias Explorer.Chain.Cache.AddressTags, as: AddressTagsCache alias Explorer.Repo alias Explorer.Tags.{AddressTag, AddressToTag} @@ -67,23 +68,28 @@ defmodule BlockScoutWeb.Models.GetAddressTags do def get_tags_on_address(address_hash, opts \\ []) def get_tags_on_address(address_hash, opts) when not is_nil(address_hash) do - common_tags_base_query() - |> where([tt, att], att.address_hash == ^address_hash) - |> select_repo(opts).all() + get_tags_on_addresses([address_hash], opts) end def get_tags_on_address(_, _), do: [] + # Served from `Explorer.Chain.Cache.AddressTags`; only addresses missing from + # the cache reach the database, in one query. defp get_tags_on_addresses(address_hashes, opts) do - common_tags_base_query() - |> where([tt, att], att.address_hash in ^address_hashes) - |> select_repo(opts).all() + AddressTagsCache.fetch(address_hashes, fn missing_address_hashes -> + common_tags_base_query() + |> where([att], att.address_hash in ^missing_address_hashes) + |> select_repo(opts).all() + end) end + # Starts from `address_to_tags` so the `address_hash` filter is applied + # through its index before joining the tags, rather than scanning + # `address_tags` and filtering the joined rows afterwards. defp common_tags_base_query do from( - tt in AddressTag, - left_join: att in AddressToTag, + att in AddressToTag, + join: tt in AddressTag, on: tt.id == att.tag_id, where: tt.label != ^"validator", select: %{label: tt.label, display_name: tt.display_name, address_hash: att.address_hash} diff --git a/apps/block_scout_web/test/block_scout_web/models/get_address_tags_test.exs b/apps/block_scout_web/test/block_scout_web/models/get_address_tags_test.exs new file mode 100644 index 00000000000..072e5f2a282 --- /dev/null +++ b/apps/block_scout_web/test/block_scout_web/models/get_address_tags_test.exs @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule BlockScoutWeb.Models.GetAddressTagsTest do + use BlockScoutWeb.ConnCase, async: false + + alias BlockScoutWeb.Models.GetAddressTags + + describe "get_tags_on_address/2" do + test "returns the public tags of the address except the validator one" do + address = insert(:address) + tag = insert(:address_tag, label: "exchange", display_name: "Exchange") + insert(:address_to_tag, address: address, tag: tag) + validator_tag = insert(:address_tag, label: "validator", display_name: "Validator") + insert(:address_to_tag, address: address, tag: validator_tag) + + assert GetAddressTags.get_tags_on_address(address.hash) == [ + %{label: "exchange", display_name: "Exchange", address_hash: address.hash} + ] + end + + test "returns an empty list for an address without tags" do + assert GetAddressTags.get_tags_on_address(insert(:address).hash) == [] + end + end + + describe "get_address_tags_batch/3" do + test "groups public tags by address and fills in addresses without tags" do + tagged = insert(:address) + untagged = insert(:address) + tag = insert(:address_tag, label: "bridge", display_name: "Bridge") + insert(:address_to_tag, address: tagged, tag: tag) + + assert GetAddressTags.get_address_tags_batch([tagged.hash, untagged.hash], nil) == %{ + tagged.hash => %{ + common_tags: [%{label: "bridge", display_name: "Bridge", address_hash: tagged.hash}], + personal_tags: [], + watchlist_names: [] + }, + untagged.hash => %{common_tags: [], personal_tags: [], watchlist_names: []} + } + end + end +end diff --git a/apps/explorer/config/config.exs b/apps/explorer/config/config.exs index 15a2236a0a5..2dcaaa14556 100644 --- a/apps/explorer/config/config.exs +++ b/apps/explorer/config/config.exs @@ -25,6 +25,8 @@ config :explorer, Explorer.Chain.Cache.BlockNumber, enabled: true config :explorer, Explorer.Chain.Cache.ContractMethods, enabled: true +config :explorer, Explorer.Chain.Cache.AddressTags, enabled: true + config :explorer, Explorer.Chain.Cache.Counters.AddressesCoinBalanceSum, enabled: true, ttl_check_interval: :timer.seconds(1) diff --git a/apps/explorer/config/test.exs b/apps/explorer/config/test.exs index 1a6aaa300a2..1551e68471b 100644 --- a/apps/explorer/config/test.exs +++ b/apps/explorer/config/test.exs @@ -104,6 +104,8 @@ end config :logger, :explorer, path: Path.absname("logs/test/explorer.log") config :explorer, Explorer.Chain.Cache.ContractMethods, enabled: false + +config :explorer, Explorer.Chain.Cache.AddressTags, enabled: false, ttl: :timer.minutes(5) config :explorer, Explorer.Chain.Fetcher.CheckBytecodeMatchingOnDemand, enabled: false config :explorer, Explorer.Chain.Fetcher.FetchValidatorInfoOnDemand, enabled: false config :explorer, Explorer.Tags.AddressTag.Cataloger, enabled: false diff --git a/apps/explorer/lib/explorer/application.ex b/apps/explorer/lib/explorer/application.ex index 9cc8c1f37ca..584438d184a 100644 --- a/apps/explorer/lib/explorer/application.ex +++ b/apps/explorer/lib/explorer/application.ex @@ -31,6 +31,7 @@ defmodule Explorer.Application do TransactionsCount } + alias Explorer.Chain.Cache.AddressTags, as: AddressTagsCache alias Explorer.Chain.Cache.ContractMethods, as: ContractMethodsCache alias Explorer.Chain.Optimism.InteropMessage, as: OptimismInteropMessage alias Explorer.Chain.Supply.RSK @@ -60,66 +61,7 @@ defmodule Explorer.Application do %{} ) - # Children to start in all environments - base_children = - [ - Explorer.Repo, - Explorer.Repo.Replica1, - Explorer.Vault, - Supervisor.child_spec({SpandexDatadog.ApiServer, datadog_opts()}, id: SpandexDatadog.ApiServer), - Supervisor.child_spec({Task.Supervisor, name: Explorer.HistoryTaskSupervisor}, - id: Explorer.HistoryTaskSupervisor - ), - Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, - id: Explorer.MarketTaskSupervisor - ), - Supervisor.child_spec({Task.Supervisor, name: Explorer.GenesisDataTaskSupervisor}, - id: GenesisDataTaskSupervisor - ), - Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor), - Supervisor.child_spec({Task.Supervisor, name: Explorer.LookUpSmartContractSourcesTaskSupervisor}, - id: LookUpSmartContractSourcesTaskSupervisor - ), - Supervisor.child_spec({Task.Supervisor, name: Explorer.WETHMigratorSupervisor}, id: WETHMigratorSupervisor), - {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, - Explorer.Chain.Cache.Propagator, - Accounts, - AddressesCoinBalanceSum, - AddressesCoinBalanceSumMinusBurnt, - BackgroundMigrations, - BlocksCount, - BlockNumber, - Blocks, - ChainId, - GasPriceOracle, - PendingBlockOperationCount, - PendingTransactionOperationCount, - TransactionsCount, - StateChanges, - Transactions, - Uncles, - AddressTabsElementsCount, - con_cache_child_spec(MarketHistoryCache.cache_name()), - con_cache_child_spec(HotSmartContractsCache.cache_name(), - ttl_check_interval: :timer.seconds(1), - global_ttl: :infinity - ), - con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), - con_cache_child_spec(ContractMethodsCache.cache_name(), - ttl_check_interval: :timer.minutes(1), - global_ttl: :infinity - ), - {Redix, redix_opts()}, - {Explorer.Utility.ReplicaAccessibilityManager, []}, - :hackney_pool.child_spec(:default, - recv_timeout: 60_000, - timeout: 60_000, - max_connections: Application.get_env(:explorer, :hackney_default_pool_size) - ), - Explorer.Promo.Autoscout - ] ++ HttpClient.pool_child_specs() - - children = base_children ++ configurable_children() + children = children() opts = [strategy: :one_for_one, name: Explorer.Supervisor, max_restarts: 1_000] @@ -130,6 +72,77 @@ defmodule Explorer.Application do end end + # The child list of the supervision tree for the current mode, exposed so a + # test can check that every entry is something a supervisor accepts. The + # mode- and chain-type-dependent helpers return `[]` for children that do not + # apply, and a bare `[]` crashes the supervisor unless the list holding it is + # flattened first — so those helpers belong in `configurable_children/0`. + @doc false + @spec children() :: [Supervisor.child_spec() | {module(), term()} | module()] + def children do + base_children() ++ configurable_children() + end + + # Children to start in all environments + defp base_children do + [ + Explorer.Repo, + Explorer.Repo.Replica1, + Explorer.Vault, + Supervisor.child_spec({SpandexDatadog.ApiServer, datadog_opts()}, id: SpandexDatadog.ApiServer), + Supervisor.child_spec({Task.Supervisor, name: Explorer.HistoryTaskSupervisor}, + id: Explorer.HistoryTaskSupervisor + ), + Supervisor.child_spec({Task.Supervisor, name: Explorer.MarketTaskSupervisor}, + id: Explorer.MarketTaskSupervisor + ), + Supervisor.child_spec({Task.Supervisor, name: Explorer.GenesisDataTaskSupervisor}, + id: GenesisDataTaskSupervisor + ), + Supervisor.child_spec({Task.Supervisor, name: Explorer.TaskSupervisor}, id: Explorer.TaskSupervisor), + Supervisor.child_spec({Task.Supervisor, name: Explorer.LookUpSmartContractSourcesTaskSupervisor}, + id: LookUpSmartContractSourcesTaskSupervisor + ), + Supervisor.child_spec({Task.Supervisor, name: Explorer.WETHMigratorSupervisor}, id: WETHMigratorSupervisor), + {Registry, keys: :duplicate, name: Registry.ChainEvents, id: Registry.ChainEvents}, + Explorer.Chain.Cache.Propagator, + Accounts, + AddressesCoinBalanceSum, + AddressesCoinBalanceSumMinusBurnt, + BackgroundMigrations, + BlocksCount, + BlockNumber, + Blocks, + ChainId, + GasPriceOracle, + PendingBlockOperationCount, + PendingTransactionOperationCount, + TransactionsCount, + StateChanges, + Transactions, + Uncles, + AddressTabsElementsCount, + con_cache_child_spec(MarketHistoryCache.cache_name()), + con_cache_child_spec(HotSmartContractsCache.cache_name(), + ttl_check_interval: :timer.seconds(1), + global_ttl: :infinity + ), + con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)), + con_cache_child_spec(ContractMethodsCache.cache_name(), + ttl_check_interval: :timer.minutes(1), + global_ttl: :infinity + ), + {Redix, redix_opts()}, + {Explorer.Utility.ReplicaAccessibilityManager, []}, + :hackney_pool.child_spec(:default, + recv_timeout: 60_000, + timeout: 60_000, + max_connections: Application.get_env(:explorer, :hackney_default_pool_size) + ), + Explorer.Promo.Autoscout + ] ++ HttpClient.pool_child_specs() + end + defp configurable_children do configurable_children_set = [ @@ -137,6 +150,15 @@ defmodule Explorer.Application do only_in_mode(Explorer.SmartContract.SolcDownloader, :api), only_in_mode(Explorer.SmartContract.VyperDownloader, :api), only_in_mode({Admin.Recovery, [[], [name: Admin.Recovery]]}, :api), + # only the web layer reads public address tags; this list is flattened, + # so the `[]` returned outside API mode is dropped rather than passed + # to the supervisor + AddressTagsCache.cache_name() + |> con_cache_child_spec( + ttl_check_interval: :timer.minutes(1), + global_ttl: :infinity + ) + |> only_in_mode(:api), configure(Explorer.Utility.VersionUpgrade), configure_mode_dependent_process(Explorer.Utility.VersionConstantsUpdater, :indexer), configure_mode_dependent_process(Explorer.Market.Fetcher.Coin, :api), diff --git a/apps/explorer/lib/explorer/chain/cache/address_tags.ex b/apps/explorer/lib/explorer/chain/cache/address_tags.ex new file mode 100644 index 00000000000..860f47d6f2f --- /dev/null +++ b/apps/explorer/lib/explorer/chain/cache/address_tags.ex @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.AddressTags do + @moduledoc """ + Per-address cache for public address tags (`address_tags` joined with + `address_to_tags`). + + Public tags are read for every address in every transaction, address and + token-transfer response, but change only through the tag cataloger and the + admin endpoints, so each address's tag list is cached with a short TTL. + Addresses without tags are cached too: an empty result is by far the most + common one, and it is exactly the lookup worth skipping. + + The cache stores whatever list the fallback returns for an address and knows + nothing about the query itself, so the filtering that the API applies (for + example hiding the `validator` tag) stays with the caller. + """ + + alias Explorer.Chain.Hash + + @cache_name :address_tags + + @type tag :: %{required(:address_hash) => Hash.Address.t(), optional(atom()) => any()} + + @spec cache_name() :: atom() + def cache_name, do: @cache_name + + @doc """ + Returns the tags of `address_hashes`, serving addresses already in the cache + from it and fetching the rest with `fallback_fn` in a single call. + + `fallback_fn` receives the list of address hashes missing from the cache and + must return the tags of those addresses as maps carrying an `:address_hash` + key. Every missing address is then cached, including the ones the fallback + returned no tags for. + + With the cache disabled the fallback is called with all `address_hashes`. + """ + @spec fetch([Hash.Address.t()], ([Hash.Address.t()] -> [tag()])) :: [tag()] + def fetch([], _fallback_fn), do: [] + + def fetch(address_hashes, fallback_fn) when is_function(fallback_fn, 1) do + if enabled?() do + fetch_with_cache(address_hashes, fallback_fn) + else + fallback_fn.(address_hashes) + end + end + + defp fetch_with_cache(address_hashes, fallback_fn) do + {cached_tags, missing_hashes} = + Enum.reduce(address_hashes, {[], []}, fn address_hash, {cached, missing} -> + case ConCache.get(@cache_name, address_hash) do + nil -> {cached, [address_hash | missing]} + tags -> {tags ++ cached, missing} + end + end) + + fetched_tags = if missing_hashes == [], do: [], else: fallback_fn.(missing_hashes) + fetched_by_hash = Enum.group_by(fetched_tags, & &1.address_hash) + + Enum.each(missing_hashes, fn address_hash -> + ConCache.put(@cache_name, address_hash, %ConCache.Item{ + value: Map.get(fetched_by_hash, address_hash, []), + ttl: ttl() + }) + end) + + cached_tags ++ fetched_tags + end + + defp enabled? do + config()[:enabled] + end + + defp ttl do + config()[:ttl] + end + + defp config do + Application.get_env(:explorer, __MODULE__) + end +end diff --git a/apps/explorer/test/explorer/chain/cache/address_tags_test.exs b/apps/explorer/test/explorer/chain/cache/address_tags_test.exs new file mode 100644 index 00000000000..98680d11442 --- /dev/null +++ b/apps/explorer/test/explorer/chain/cache/address_tags_test.exs @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.Chain.Cache.AddressTagsTest do + use Explorer.DataCase, async: false + + alias Explorer.Chain.Cache.AddressTags + + setup do + config = Application.get_env(:explorer, AddressTags) || [] + Application.put_env(:explorer, AddressTags, Keyword.merge(config, enabled: true)) + on_exit(fn -> Application.put_env(:explorer, AddressTags, config) end) + + :ok + end + + describe "fetch/2" do + test "returns an empty list for no address hashes without calling the fallback" do + assert AddressTags.fetch([], fn hashes -> + send(self(), {:fallback, hashes}) + [] + end) == [] + + refute_received {:fallback, _} + end + + test "calls the fallback only for addresses missing from the cache" do + tagged_hash = build(:address).hash + untagged_hash = build(:address).hash + new_hash = build(:address).hash + tag = %{label: "label", display_name: "Label", address_hash: tagged_hash} + + fallback = fn hashes -> + send(self(), {:fallback, Enum.sort(hashes)}) + for hash <- hashes, hash == tagged_hash, do: tag + end + + assert AddressTags.fetch([tagged_hash, untagged_hash], fallback) == [tag] + assert_received {:fallback, hashes} + assert hashes == Enum.sort([tagged_hash, untagged_hash]) + + # the tagged address and the address without tags are both cached now + assert AddressTags.fetch([tagged_hash, untagged_hash, new_hash], fallback) == [tag] + assert_received {:fallback, [^new_hash]} + end + + test "serves a cached address without touching the fallback" do + hash = build(:address).hash + tag = %{label: "label", display_name: "Label", address_hash: hash} + + assert AddressTags.fetch([hash], fn _ -> [tag] end) == [tag] + + assert AddressTags.fetch([hash], fn hashes -> + send(self(), {:fallback, hashes}) + [] + end) == [tag] + + refute_received {:fallback, _} + end + + test "passes every address to the fallback when disabled" do + Application.put_env( + :explorer, + AddressTags, + Keyword.merge(Application.get_env(:explorer, AddressTags), enabled: false) + ) + + hash = build(:address).hash + + assert AddressTags.fetch([hash], fn hashes -> + send(self(), {:fallback, hashes}) + [] + end) == [] + + assert_received {:fallback, [^hash]} + + assert AddressTags.fetch([hash], fn hashes -> + send(self(), {:fallback, hashes}) + [] + end) == [] + + assert_received {:fallback, [^hash]} + end + end +end diff --git a/config/runtime.exs b/config/runtime.exs index 4eeff9818fc..685d0012e16 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1011,6 +1011,8 @@ config :explorer, Explorer.Chain.Cache.ScamAddresses, update_interval: ConfigHelper.parse_time_env_var("CACHE_SCAM_ADDRESSES_UPDATE_INTERVAL", "5m"), max_size: ConfigHelper.parse_integer_env_var("CACHE_SCAM_ADDRESSES_MAX_SIZE", 200_000, min: 0) +config :explorer, Explorer.Chain.Cache.AddressTags, ttl: ConfigHelper.parse_time_env_var("CACHE_ADDRESS_TAGS_TTL", "5m") + rate_limiter_redis_url = ConfigHelper.parse_url_env_var("RATE_LIMITER_REDIS_URL") rate_limiter_redis_sentinel_urls = ConfigHelper.safe_get_env("RATE_LIMITER_REDIS_SENTINEL_URLS", "") diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index e19dfed6a6b..1b16cc9b153 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -150,6 +150,7 @@ RELEASE_LINK= # CACHE_PENDING_OPERATIONS_COUNT_PERIOD= # CACHE_SCAM_ADDRESSES_UPDATE_INTERVAL= # CACHE_SCAM_ADDRESSES_MAX_SIZE= +# CACHE_ADDRESS_TAGS_TTL= # PENDING_OPERATIONS_HELPER_TRANSACTIONS_BATCH_SIZE=1000 # PENDING_OPERATIONS_HELPER_BLOCKS_BATCH_SIZE=10 # TOKEN_BALANCE_ON_DEMAND_FETCHER_THRESHOLD= From 50cbcf4ea6855ff8bb46cdce67e7cacfe012a6c9 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 18:36:39 +0300 Subject: [PATCH 50/52] Add missing test --- .../test/explorer/application_test.exs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 apps/explorer/test/explorer/application_test.exs diff --git a/apps/explorer/test/explorer/application_test.exs b/apps/explorer/test/explorer/application_test.exs new file mode 100644 index 00000000000..69a064fd947 --- /dev/null +++ b/apps/explorer/test/explorer/application_test.exs @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: LicenseRef-Blockscout +defmodule Explorer.ApplicationTest do + use ExUnit.Case, async: false + + describe "children/0" do + # Regression test: a mode-gated child spec placed in a list that is not + # flattened reaches the supervisor as a bare `[]` outside its mode and + # crashes the application on start. `Supervisor.init/2` runs the same + # child validation as `Supervisor.start_link/2` without starting anything. + for mode <- [:indexer, :api, :all] do + test "every child is accepted by the supervisor in #{mode} mode" do + initial_mode = Application.get_env(:explorer, :mode) + Application.put_env(:explorer, :mode, unquote(mode)) + on_exit(fn -> Application.put_env(:explorer, :mode, initial_mode) end) + + children = Explorer.Application.children() + refute Enum.empty?(children) + + assert {:ok, {_flags, specs}} = Supervisor.init(children, strategy: :one_for_one) + assert length(specs) == length(children) + end + end + end +end From 5efb2aac32f7aa7388d08c0ee1e6610040572bbd Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Tue, 15 Sep 2026 19:23:16 +0300 Subject: [PATCH 51/52] docs: document OP Stack transaction types in Transaction schema (#14835) --- .../schemas/api/v2/transaction.ex | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/transaction.ex b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/transaction.ex index 1322be3f59d..55ce62bb716 100644 --- a/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/transaction.ex +++ b/apps/block_scout_web/lib/block_scout_web/schemas/api/v2/transaction.ex @@ -109,6 +109,14 @@ defmodule BlockScoutWeb.Schemas.API.V2.Transaction.ChainTypeCustomizations do additionalProperties: false } + # OP Stack-specific transaction types appended to the `transaction_types` enum + # for both `optimism` and `optimism-celo` chain types (the latter resolves to + # the `:optimism` chain type with a `{:optimism, :celo}` chain identity). + @optimism_transaction_types [ + "op_stack_l1_attributes_transaction", + "op_stack_post_exec_transaction" + ] + @scroll_schema %Schema{ type: :object, nullable: false, @@ -196,6 +204,7 @@ defmodule BlockScoutWeb.Schemas.API.V2.Transaction.ChainTypeCustomizations do operator_fee: General.IntegerString } ) + |> extend_transaction_types_enum(@optimism_transaction_types) :scroll -> schema |> Helper.extend_schema(properties: %{scroll: @scroll_schema}) @@ -295,6 +304,16 @@ defmodule BlockScoutWeb.Schemas.API.V2.Transaction.ChainTypeCustomizations do |> chain_identity_fields() end + # Appends chain-specific values to the `transaction_types` items enum of the given schema. + @spec extend_transaction_types_enum(map(), [String.t()]) :: map() + defp extend_transaction_types_enum(schema, extra_types) do + update_in( + schema, + [:properties, :transaction_types, Access.key(:items), Access.key(:enum)], + &(&1 ++ extra_types) + ) + end + defp chain_identity_fields(schema) do case chain_identity() do {:optimism, :celo} -> From 07c097c6d4dc00ebc8ac8d38c9a26af09c8372d8 Mon Sep 17 00:00:00 2001 From: Victor Baranov Date: Wed, 16 Sep 2026 14:01:49 +0300 Subject: [PATCH 52/52] v11.3.1 --- .github/workflows/generate-swagger.yml | 2 +- .github/workflows/pre-release-arbitrum.yml | 2 +- .github/workflows/pre-release-celo.yml | 2 +- .github/workflows/pre-release-eden.yml | 2 +- .github/workflows/pre-release-eth.yml | 2 +- .github/workflows/pre-release-filecoin.yml | 2 +- .github/workflows/pre-release-fuse.yml | 2 +- .github/workflows/pre-release-gnosis.yml | 2 +- .github/workflows/pre-release-optimism.yml | 2 +- .github/workflows/pre-release-rootstock.yml | 2 +- .github/workflows/pre-release-scroll.yml | 2 +- .github/workflows/pre-release-zilliqa.yml | 2 +- .github/workflows/pre-release-zksync.yml | 2 +- .github/workflows/pre-release.yml | 2 +- .../publish-docker-image-custom-build.yml | 2 +- .../publish-docker-image-every-push.yml | 2 +- .../publish-docker-image-for-arbitrum.yml | 2 +- .../publish-docker-image-for-celo.yml | 2 +- .../publish-docker-image-for-eden.yml | 2 +- .../publish-docker-image-for-eth-sepolia.yml | 2 +- .../publish-docker-image-for-eth.yml | 2 +- .../publish-docker-image-for-filecoin.yml | 2 +- .../publish-docker-image-for-fuse.yml | 2 +- .../publish-docker-image-for-gnosis-chain.yml | 2 +- ...ocker-image-for-optimism-exeperimental.yml | 2 +- .../publish-docker-image-for-optimism.yml | 2 +- .../publish-docker-image-for-rootstock.yml | 2 +- .../publish-docker-image-for-scroll.yml | 2 +- .../publish-docker-image-for-zetachain.yml | 2 +- .../publish-docker-image-for-zilliqa.yml | 2 +- .../publish-docker-image-for-zksync.yml | 2 +- .../workflows/publish-docker-image-old-ui.yml | 2 +- .github/workflows/release-arbitrum.yml | 2 +- .github/workflows/release-celo.yml | 2 +- .github/workflows/release-default.yml | 2 +- .github/workflows/release-eden.yml | 2 +- .github/workflows/release-eth.yml | 2 +- .github/workflows/release-filecoin.yml | 2 +- .github/workflows/release-fuse.yml | 2 +- .github/workflows/release-gnosis.yml | 2 +- .github/workflows/release-optimism.yml | 2 +- .github/workflows/release-rootstock.yml | 2 +- .github/workflows/release-scroll.yml | 2 +- .github/workflows/release-zetachain.yml | 2 +- .github/workflows/release-zilliqa.yml | 2 +- .github/workflows/release-zksync.yml | 2 +- CHANGELOG.md | 27 +++++++++++++++++++ apps/block_scout_web/mix.exs | 2 +- apps/ethereum_jsonrpc/mix.exs | 2 +- .../lib/explorer/token/metadata_retriever.ex | 2 +- apps/explorer/mix.exs | 2 +- apps/indexer/mix.exs | 2 +- apps/nft_media_handler/mix.exs | 2 +- apps/utils/mix.exs | 2 +- docker/Makefile | 2 +- mix.exs | 2 +- rel/config.exs | 2 +- 57 files changed, 83 insertions(+), 56 deletions(-) diff --git a/.github/workflows/generate-swagger.yml b/.github/workflows/generate-swagger.yml index a606a14d9c9..d056ef0c3b4 100644 --- a/.github/workflows/generate-swagger.yml +++ b/.github/workflows/generate-swagger.yml @@ -17,7 +17,7 @@ on: env: OTP_VERSION: '27.3.4.6' ELIXIR_VERSION: '1.19.4' - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 jobs: matrix-builder: diff --git a/.github/workflows/pre-release-arbitrum.yml b/.github/workflows/pre-release-arbitrum.yml index 6a9af42927d..876921a291b 100644 --- a/.github/workflows/pre-release-arbitrum.yml +++ b/.github/workflows/pre-release-arbitrum.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-celo.yml b/.github/workflows/pre-release-celo.yml index 1d1ab8ef114..da8c0d8b659 100644 --- a/.github/workflows/pre-release-celo.yml +++ b/.github/workflows/pre-release-celo.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 API_GRAPHQL_MAX_COMPLEXITY: 10400 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/pre-release-eden.yml b/.github/workflows/pre-release-eden.yml index bd31991cec1..f2711dca7de 100644 --- a/.github/workflows/pre-release-eden.yml +++ b/.github/workflows/pre-release-eden.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-eth.yml b/.github/workflows/pre-release-eth.yml index 1cf3e9dcbad..65f2a618d43 100644 --- a/.github/workflows/pre-release-eth.yml +++ b/.github/workflows/pre-release-eth.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-filecoin.yml b/.github/workflows/pre-release-filecoin.yml index dda754e3bbc..11ca00a1867 100644 --- a/.github/workflows/pre-release-filecoin.yml +++ b/.github/workflows/pre-release-filecoin.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-fuse.yml b/.github/workflows/pre-release-fuse.yml index a54d79fe0bf..07009665612 100644 --- a/.github/workflows/pre-release-fuse.yml +++ b/.github/workflows/pre-release-fuse.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-gnosis.yml b/.github/workflows/pre-release-gnosis.yml index 792aa183b16..8a2f3057fa2 100644 --- a/.github/workflows/pre-release-gnosis.yml +++ b/.github/workflows/pre-release-gnosis.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-optimism.yml b/.github/workflows/pre-release-optimism.yml index 547c239a13e..ad8e69e7c1b 100644 --- a/.github/workflows/pre-release-optimism.yml +++ b/.github/workflows/pre-release-optimism.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-rootstock.yml b/.github/workflows/pre-release-rootstock.yml index a713602d9e0..2d90904f43e 100644 --- a/.github/workflows/pre-release-rootstock.yml +++ b/.github/workflows/pre-release-rootstock.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-scroll.yml b/.github/workflows/pre-release-scroll.yml index a1e2a40d0f2..38e10622d9b 100644 --- a/.github/workflows/pre-release-scroll.yml +++ b/.github/workflows/pre-release-scroll.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-zilliqa.yml b/.github/workflows/pre-release-zilliqa.yml index 204364ac65f..54d0ee35e60 100644 --- a/.github/workflows/pre-release-zilliqa.yml +++ b/.github/workflows/pre-release-zilliqa.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release-zksync.yml b/.github/workflows/pre-release-zksync.yml index ef057c79c82..c8495e3396f 100644 --- a/.github/workflows/pre-release-zksync.yml +++ b/.github/workflows/pre-release-zksync.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 770af881b8d..d147955ed1b 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -21,7 +21,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/publish-docker-image-custom-build.yml b/.github/workflows/publish-docker-image-custom-build.yml index 2658e567c01..dae72763762 100644 --- a/.github/workflows/publish-docker-image-custom-build.yml +++ b/.github/workflows/publish-docker-image-custom-build.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/publish-docker-image-every-push.yml b/.github/workflows/publish-docker-image-every-push.yml index 6e92217ef68..9e748196480 100644 --- a/.github/workflows/publish-docker-image-every-push.yml +++ b/.github/workflows/publish-docker-image-every-push.yml @@ -12,7 +12,7 @@ on: env: OTP_VERSION: '27.3.4.6' ELIXIR_VERSION: '1.19.4' - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 permissions: contents: read diff --git a/.github/workflows/publish-docker-image-for-arbitrum.yml b/.github/workflows/publish-docker-image-for-arbitrum.yml index 3e67e81020b..b77bf6708bf 100644 --- a/.github/workflows/publish-docker-image-for-arbitrum.yml +++ b/.github/workflows/publish-docker-image-for-arbitrum.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: arbitrum steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-celo.yml b/.github/workflows/publish-docker-image-for-celo.yml index f8e1e4cbb6f..9061d4b788e 100644 --- a/.github/workflows/publish-docker-image-for-celo.yml +++ b/.github/workflows/publish-docker-image-for-celo.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: celo CHAIN_TYPE: optimism-celo API_GRAPHQL_MAX_COMPLEXITY: 10400 diff --git a/.github/workflows/publish-docker-image-for-eden.yml b/.github/workflows/publish-docker-image-for-eden.yml index ff16b3f30aa..6265bf2dc19 100644 --- a/.github/workflows/publish-docker-image-for-eden.yml +++ b/.github/workflows/publish-docker-image-for-eden.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: eden steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-eth-sepolia.yml b/.github/workflows/publish-docker-image-for-eth-sepolia.yml index 1e7a28c5138..27949d4ead7 100644 --- a/.github/workflows/publish-docker-image-for-eth-sepolia.yml +++ b/.github/workflows/publish-docker-image-for-eth-sepolia.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: eth-sepolia steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-eth.yml b/.github/workflows/publish-docker-image-for-eth.yml index 4084d4db8bf..0f61699052d 100644 --- a/.github/workflows/publish-docker-image-for-eth.yml +++ b/.github/workflows/publish-docker-image-for-eth.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: ethereum steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-filecoin.yml b/.github/workflows/publish-docker-image-for-filecoin.yml index 5df2e635404..590dff13ccf 100644 --- a/.github/workflows/publish-docker-image-for-filecoin.yml +++ b/.github/workflows/publish-docker-image-for-filecoin.yml @@ -13,7 +13,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: filecoin steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-fuse.yml b/.github/workflows/publish-docker-image-for-fuse.yml index d4ea1a103cf..28756341ac1 100644 --- a/.github/workflows/publish-docker-image-for-fuse.yml +++ b/.github/workflows/publish-docker-image-for-fuse.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: fuse steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-gnosis-chain.yml b/.github/workflows/publish-docker-image-for-gnosis-chain.yml index e18cd5b4588..5310e3145a4 100644 --- a/.github/workflows/publish-docker-image-for-gnosis-chain.yml +++ b/.github/workflows/publish-docker-image-for-gnosis-chain.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: xdai steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml b/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml index d78b6bdbfa6..da5e1e6c921 100644 --- a/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml +++ b/.github/workflows/publish-docker-image-for-optimism-exeperimental.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: optimism steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-optimism.yml b/.github/workflows/publish-docker-image-for-optimism.yml index 723abc91016..45672e08973 100644 --- a/.github/workflows/publish-docker-image-for-optimism.yml +++ b/.github/workflows/publish-docker-image-for-optimism.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: optimism steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-rootstock.yml b/.github/workflows/publish-docker-image-for-rootstock.yml index 2d7fc653378..d191cccce82 100644 --- a/.github/workflows/publish-docker-image-for-rootstock.yml +++ b/.github/workflows/publish-docker-image-for-rootstock.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: rsk steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-scroll.yml b/.github/workflows/publish-docker-image-for-scroll.yml index 5f7ad99671d..5d81133f35d 100644 --- a/.github/workflows/publish-docker-image-for-scroll.yml +++ b/.github/workflows/publish-docker-image-for-scroll.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: scroll steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zetachain.yml b/.github/workflows/publish-docker-image-for-zetachain.yml index cdac7272132..c814ada2c90 100644 --- a/.github/workflows/publish-docker-image-for-zetachain.yml +++ b/.github/workflows/publish-docker-image-for-zetachain.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: zetachain steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zilliqa.yml b/.github/workflows/publish-docker-image-for-zilliqa.yml index b2fa1c14524..972b8e0ab1c 100644 --- a/.github/workflows/publish-docker-image-for-zilliqa.yml +++ b/.github/workflows/publish-docker-image-for-zilliqa.yml @@ -14,7 +14,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: zilliqa steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-for-zksync.yml b/.github/workflows/publish-docker-image-for-zksync.yml index 6b6b03498a6..5e71470072e 100644 --- a/.github/workflows/publish-docker-image-for-zksync.yml +++ b/.github/workflows/publish-docker-image-for-zksync.yml @@ -13,7 +13,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 DOCKER_CHAIN_NAME: zksync steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/publish-docker-image-old-ui.yml b/.github/workflows/publish-docker-image-old-ui.yml index 936716fe496..ec555808176 100644 --- a/.github/workflows/publish-docker-image-old-ui.yml +++ b/.github/workflows/publish-docker-image-old-ui.yml @@ -16,7 +16,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-arbitrum.yml b/.github/workflows/release-arbitrum.yml index 36a47383139..88bebcb20e3 100644 --- a/.github/workflows/release-arbitrum.yml +++ b/.github/workflows/release-arbitrum.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-celo.yml b/.github/workflows/release-celo.yml index f1ec3b3e261..5a78fc5081c 100644 --- a/.github/workflows/release-celo.yml +++ b/.github/workflows/release-celo.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 API_GRAPHQL_MAX_COMPLEXITY: 10400 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/release-default.yml b/.github/workflows/release-default.yml index d19d5cd9b19..1e5c6f37e21 100644 --- a/.github/workflows/release-default.yml +++ b/.github/workflows/release-default.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-eden.yml b/.github/workflows/release-eden.yml index ee832c06205..009feb84887 100644 --- a/.github/workflows/release-eden.yml +++ b/.github/workflows/release-eden.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-eth.yml b/.github/workflows/release-eth.yml index f7678ab9963..4e3126f59a2 100644 --- a/.github/workflows/release-eth.yml +++ b/.github/workflows/release-eth.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-filecoin.yml b/.github/workflows/release-filecoin.yml index 0839eb590a4..86a8eefe4a8 100644 --- a/.github/workflows/release-filecoin.yml +++ b/.github/workflows/release-filecoin.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-fuse.yml b/.github/workflows/release-fuse.yml index c5077ed3a0e..c7dfc722b64 100644 --- a/.github/workflows/release-fuse.yml +++ b/.github/workflows/release-fuse.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-gnosis.yml b/.github/workflows/release-gnosis.yml index 161e81b2f44..b6982ddc2de 100644 --- a/.github/workflows/release-gnosis.yml +++ b/.github/workflows/release-gnosis.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-optimism.yml b/.github/workflows/release-optimism.yml index 44dabf542f2..92ca52a73a5 100644 --- a/.github/workflows/release-optimism.yml +++ b/.github/workflows/release-optimism.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-rootstock.yml b/.github/workflows/release-rootstock.yml index 0497c1239b8..0d6501924e7 100644 --- a/.github/workflows/release-rootstock.yml +++ b/.github/workflows/release-rootstock.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-scroll.yml b/.github/workflows/release-scroll.yml index 76d67bca855..0e9c43ab61c 100644 --- a/.github/workflows/release-scroll.yml +++ b/.github/workflows/release-scroll.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zetachain.yml b/.github/workflows/release-zetachain.yml index fca42686e85..3c4cdef5ccd 100644 --- a/.github/workflows/release-zetachain.yml +++ b/.github/workflows/release-zetachain.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zilliqa.yml b/.github/workflows/release-zilliqa.yml index caef5f2b4f6..e18644025dc 100644 --- a/.github/workflows/release-zilliqa.yml +++ b/.github/workflows/release-zilliqa.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/.github/workflows/release-zksync.yml b/.github/workflows/release-zksync.yml index f375ba5e254..467ded817c4 100644 --- a/.github/workflows/release-zksync.yml +++ b/.github/workflows/release-zksync.yml @@ -18,7 +18,7 @@ jobs: name: Push Docker image to GitHub Container Registry runs-on: build env: - RELEASE_VERSION: 11.3.0 + RELEASE_VERSION: 11.3.1 steps: - uses: actions/checkout@v5 - name: Setup repo diff --git a/CHANGELOG.md b/CHANGELOG.md index 15dfe241e25..ebf6f3beade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## 11.3.1 + +### 🚀 Features + +- Support OP Stack Upgrade 20 (Super Root games, SystemConfig v4) ([#14818](https://github.com/blockscout/blockscout/issues/14818)) + +### 📚 Documentation + +- Document OP Stack transaction types in Transaction schema ([#14835](https://github.com/blockscout/blockscout/issues/14835)) + +### ⚡ Performance + +- Cache public address tags and index-driven tags lookup ([#14833](https://github.com/blockscout/blockscout/issues/14833)) +- Bound and deduplicate on-demand token total supply fetcher ([#14811](https://github.com/blockscout/blockscout/issues/14811)) +- Preload signed_authorizations only for EIP-7702 transactions ([#14828](https://github.com/blockscout/blockscout/issues/14828)) +- Reuse preloaded proxy and ABI associations in input decoding ([#14827](https://github.com/blockscout/blockscout/issues/14827)) +- Use Endpoint.local_broadcast instead of Endpoint.broadcast ([#14819](https://github.com/blockscout/blockscout/issues/14819)) +- Optimize realtime events broadcast for legacy topics ([#14810](https://github.com/blockscout/blockscout/issues/14810)) + +### New ENV variables +| Variable | Description | Parameters | +|-----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------| +| `INDEXER_DISABLE_TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER` | If `true`, `Indexer.Fetcher.OnDemand.TokenTotalSupply` fetcher is disabled at runtime. Implemented in [#14811](https://github.com/blockscout/blockscout/pull/14811). | Version: v11.3.1\+
Default: `false`
Applications: Indexer | +| `TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_CONCURRENCY` | Concurrency for `Indexer.Fetcher.OnDemand.TokenTotalSupply`. Implemented in [#14811](https://github.com/blockscout/blockscout/pull/14811). | Version: v11.3.1\+
Default: `5`
Applications: Indexer | +| `TOKEN_TOTAL_SUPPLY_ON_DEMAND_FETCHER_THRESHOLD` | Backoff threshold after a failed token total supply fetch. [Time format](/setup/env-variables/backend-env-variables#time-format). Implemented in [#14811](https://github.com/blockscout/blockscout/pull/14811). | Version: v11.3.1\+
Default: `5m`
Applications: Indexer | + + ## 11.3.0 ### 🚀 Features diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index 8e84ff28c10..214dbc9cd09 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -20,7 +20,7 @@ defmodule BlockScoutWeb.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.3.0", + version: "11.3.1", xref: [ exclude: [ Explorer.Chain.Beacon.Reader, diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index ed0e59b226e..6aa7cab4dc9 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -20,7 +20,7 @@ defmodule EthereumJSONRPC.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.3.0" + version: "11.3.1" ] end diff --git a/apps/explorer/lib/explorer/token/metadata_retriever.ex b/apps/explorer/lib/explorer/token/metadata_retriever.ex index 4f1a250fb5c..6a712ee4f3b 100644 --- a/apps/explorer/lib/explorer/token/metadata_retriever.ex +++ b/apps/explorer/lib/explorer/token/metadata_retriever.ex @@ -15,7 +15,7 @@ defmodule Explorer.Token.MetadataRetriever do @vm_execution_error "VM execution error" @invalid_base64_data "invalid data:application/json;base64" @invalid_ipfs_path "invalid ipfs path" - @default_headers [{"User-Agent", "blockscout-11.3.0"}] + @default_headers [{"User-Agent", "blockscout-11.3.1"}] # https://eips.ethereum.org/EIPS/eip-1155#metadata @erc1155_token_id_placeholder "{id}" diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index eef800239da..578ee08ade3 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -21,7 +21,7 @@ defmodule Explorer.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.3.0", + version: "11.3.1", xref: [exclude: [BlockScoutWeb.Routers.WebRouter.Helpers, Indexer.Helper, Indexer.Fetcher.InternalTransaction]] ] end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index 8d2fb44980a..25ac2b804b0 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -15,7 +15,7 @@ defmodule Indexer.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.3.0", + version: "11.3.1", xref: [ exclude: [ Explorer.Chain.Optimism.Deposit, diff --git a/apps/nft_media_handler/mix.exs b/apps/nft_media_handler/mix.exs index 2ca0a375f6c..aa5528b6382 100644 --- a/apps/nft_media_handler/mix.exs +++ b/apps/nft_media_handler/mix.exs @@ -5,7 +5,7 @@ defmodule NFTMediaHandler.MixProject do def project do [ app: :nft_media_handler, - version: "11.3.0", + version: "11.3.1", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/utils/mix.exs b/apps/utils/mix.exs index 35bc091e48c..e1d6d7bd2f7 100644 --- a/apps/utils/mix.exs +++ b/apps/utils/mix.exs @@ -5,7 +5,7 @@ defmodule Utils.MixProject do def project do [ app: :utils, - version: "11.3.0", + version: "11.3.1", build_path: "../../_build", # config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/docker/Makefile b/docker/Makefile index 01c60d54b12..b3c81e4a26e 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,7 +10,7 @@ STATS_CONTAINER_NAME := stats STATS_DB_CONTAINER_NAME := stats-db PROXY_CONTAINER_NAME := proxy PG_CONTAINER_NAME := postgres -RELEASE_VERSION ?= '11.3.0' +RELEASE_VERSION ?= '11.3.1' TAG := $(RELEASE_VERSION)-commit-$(shell git log -1 --pretty=format:"%h") STABLE_TAG := $(RELEASE_VERSION) diff --git a/mix.exs b/mix.exs index 628ab10b711..e15a85ff597 100644 --- a/mix.exs +++ b/mix.exs @@ -8,7 +8,7 @@ defmodule BlockScout.Mixfile do [ # app: :block_scout, # aliases: aliases(config_env()), - version: "11.3.0", + version: "11.3.1", apps_path: "apps", deps: deps(), dialyzer: dialyzer(), diff --git a/rel/config.exs b/rel/config.exs index 5446ef721ed..43d438d9cb7 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -72,7 +72,7 @@ end # will be used by default release :blockscout do - set version: "11.3.0" + set version: "11.3.1" set applications: [ :runtime_tools, block_scout_web: :permanent,