From 1adf3f4aa3f8f34b8da76449d57fc0f786f6532c Mon Sep 17 00:00:00 2001 From: Kirill Kaiumov Date: Mon, 2 Nov 2026 12:35:26 +0100 Subject: [PATCH 1/2] Filter modules before loading them and load the kept ones in one call The retriever ran filter_modules only after a module had been loaded and fully processed (abstract code, specs, callbacks, markdown), so on a large project an allow-list of a few modules cost as much as documenting everything. Autolink then checked every module reference against the list of filtered module nodes with Enum.any?, which made a filtered build slower than an unfiltered one. Now the retriever applies filter_modules to the docs chunk alone, before Code.ensure_loaded and module_data, and returns the filtered modules as names. Autolink keeps them in a MapSet. The kept modules are loaded with one Code.ensure_all_loaded/1 call instead of a Code.ensure_loaded/1 round trip per module from parallel tasks, which serializes on the code server. Measured on a project with ~50k modules (6 schedulers): a 100-module allow-list went from 55-62 s to 22 s in the retriever+formatter phase with the modules preloaded, and from 468 s to 20 s without a preload; the full unfiltered build went from 473 s to 102 s without a preload. Output is unchanged. Co-Authored-By: Claude Fable 5.1 --- lib/ex_doc/autolink.ex | 6 +- lib/ex_doc/formatter.ex | 8 +-- lib/ex_doc/retriever.ex | 94 +++++++++++++++++++--------- test/ex_doc/language/elixir_test.exs | 8 +-- test/ex_doc/language/erlang_test.exs | 8 +-- test/ex_doc/retriever_test.exs | 7 +-- 6 files changed, 83 insertions(+), 48 deletions(-) diff --git a/lib/ex_doc/autolink.ex b/lib/ex_doc/autolink.ex index f8b46c1c0..bee52f95c 100644 --- a/lib/ex_doc/autolink.ex +++ b/lib/ex_doc/autolink.ex @@ -32,7 +32,7 @@ defmodule ExDoc.Autolink do # * `:skip_code_autolink_to` - function that will be called with a term and return a boolean # whether to skip autolinking to it. # - # * `:filtered_modules` - A list of module nodes that were filtered by the retriever + # * `:filtered_modules` - A set of the module names the retriever filtered out # # * `:warnings` - one of: # @@ -61,7 +61,7 @@ defmodule ExDoc.Autolink do &ExDoc.Formatter.Config.skip_undefined_reference_warnings_on/1, skip_code_autolink_to: &ExDoc.Formatter.Config.skip_code_autolink_to/1, force_module_prefix: nil, - filtered_modules: [], + filtered_modules: MapSet.new(), assets: %{}, warnings: :emit ] @@ -78,7 +78,7 @@ defmodule ExDoc.Autolink do end defp string_app_module_url(tool, module, anchor, config) do - if Enum.any?(config.filtered_modules, &(&1.module == module)) do + if MapSet.member?(config.filtered_modules, module) do warn(config, "reference to a filtered module") nil else diff --git a/lib/ex_doc/formatter.ex b/lib/ex_doc/formatter.ex index f8e2dca5e..236bb4a0a 100644 --- a/lib/ex_doc/formatter.ex +++ b/lib/ex_doc/formatter.ex @@ -24,7 +24,7 @@ defmodule ExDoc.Formatter do @optional_callbacks autolink_options: 0 @doc false - def run(formatter, formatter_config, module_nodes, filtered_nodes, extras) do + def run(formatter, formatter_config, module_nodes, filtered_modules, extras) do if not Code.ensure_loaded?(formatter) do raise "formatter module #{inspect(formatter)} not found" end @@ -35,7 +35,7 @@ defmodule ExDoc.Formatter do {module_nodes, extras} = if function_exported?(formatter, :autolink_options, 0) do autolink_opts = formatter.autolink_options() - autolink(formatter_config, module_nodes, filtered_nodes, extras, autolink_opts) + autolink(formatter_config, module_nodes, filtered_modules, extras, autolink_opts) else {module_nodes, extras} end @@ -180,7 +180,7 @@ defmodule ExDoc.Formatter do ## Autolinking @doc false - def autolink(config, nodes, filtered_nodes, extras, opts) do + def autolink(config, nodes, filtered_modules, extras, opts) do {ext, highlight_opts} = Keyword.pop!(opts, :extension) language = @@ -196,7 +196,7 @@ defmodule ExDoc.Formatter do extras: extra_paths(extras), skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on, skip_code_autolink_to: config.skip_code_autolink_to, - filtered_modules: filtered_nodes, + filtered_modules: MapSet.new(filtered_modules), assets: config.assets } diff --git a/lib/ex_doc/retriever.ex b/lib/ex_doc/retriever.ex index 75dc342aa..ac573a70c 100644 --- a/lib/ex_doc/retriever.ex +++ b/lib/ex_doc/retriever.ex @@ -13,11 +13,11 @@ defmodule ExDoc.Retriever do @doc """ Extract documentation from all modules in the specified directories. - Returns a tuple containing `{modules, filtered}`, using `config.filter_modules` - as a filter criteria. + Returns a tuple containing `{modules, filtered}`: the module nodes that + passed `config.filter_modules` and the names of the modules it rejected. """ @spec docs_from_dir([Path.t()], ExDoc.Config.t()) :: - {[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]} + {[ExDoc.ModuleNode.t()], [module()]} def docs_from_dir(dirs, config) when is_list(dirs) do dirs |> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config)) @@ -34,33 +34,65 @@ defmodule ExDoc.Retriever do @doc """ Extract documentation from all modules and returns a tuple containing - `{modules, filtered}`, two lists of modules that were extracted and filtered - by `config.filter_modules`, respectively. + `{modules, filtered}`: the module nodes that passed `config.filter_modules` + and the names of the modules it rejected. """ @spec docs_from_modules([atom], ExDoc.Config.t()) :: - {[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]} + {[ExDoc.ModuleNode.t()], [module()]} def docs_from_modules(modules, config) when is_list(modules) do modules |> docs_from_modules({[], []}, config) |> sort_modules(config) end - defp docs_from_modules(modules, acc, config) do - modules - |> Task.async_stream(&get_module(&1, config), timeout: :infinity) - |> Enum.reduce(acc, fn {:ok, result}, {modules, filtered} = acc -> - case result do - {:error, _module} -> + # Filtering happens on the docs chunk alone, before the module is loaded and + # its specs, callbacks and docs are processed, so filtered modules cost only a + # chunk read. The kept modules are then loaded in one call: loading them one + # by one from several tasks serializes on the code server, which on large + # projects takes an order of magnitude longer than the rest of the retrieval. + defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do + {kept, filtered} = + modules + |> Task.async_stream(&filter_module(&1, config), timeout: :infinity) + |> Enum.reduce({[], acc_filtered}, fn + {:ok, {:kept, module, docs_chunk}}, {kept, filtered} -> + {[{module, docs_chunk} | kept], filtered} + + {:ok, {:filtered, module}}, {kept, filtered} -> + {kept, [module | filtered]} + + {:ok, :skip}, acc -> acc + end) - {:ok, module_node} -> - if config.filter_modules.(module_node.module, module_node.metadata), - do: {[module_node | modules], filtered}, - else: {modules, [module_node | filtered]} - end - end) + _ = Code.ensure_all_loaded(Enum.map(kept, &elem(&1, 0))) + + modules = + kept + |> Task.async_stream( + fn {module, docs_chunk} -> get_module(module, docs_chunk, config) end, + timeout: :infinity + ) + |> Enum.reduce(acc_modules, fn + {:ok, {:ok, module_node}}, modules -> [module_node | modules] + {:ok, {:error, _module}}, modules -> modules + end) + + {modules, filtered} + end + + defp filter_module(module, config) do + case docs_chunk(module) do + {:docs_v1, _, _, _, _, metadata, _} = docs_chunk -> + if config.filter_modules.(module, metadata), + do: {:kept, module, docs_chunk}, + else: {:filtered, module} + + _ -> + :skip + end end defp sort_modules({modules, filtered}, config) do - {sort_modules(modules, config), sort_modules(filtered, config)} + {sort_modules(modules, config), Enum.sort(filtered)} end defp sort_modules(modules, config) when is_list(modules) do @@ -75,8 +107,8 @@ defmodule ExDoc.Retriever do String.to_atom(name) end - defp get_module(module, config) do - with {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk <- docs_chunk(module), + defp get_module(module, {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk, config) do + with {:module, _} <- ensure_loaded(module), {:ok, language} <- ExDoc.Language.get(language, module), %{} = module_data <- language.module_data(module, docs_chunk, config) do {:ok, generate_node(module, module_data, config)} @@ -86,6 +118,17 @@ defmodule ExDoc.Retriever do end end + defp ensure_loaded(module) do + case Code.ensure_loaded(module) do + {:module, _} = loaded -> + loaded + + {:error, reason} -> + ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", []) + false + end + end + defp docs_chunk(module) do result = Code.fetch_docs(module) Refs.insert_from_chunk(module, result) @@ -95,14 +138,7 @@ defmodule ExDoc.Retriever do false {:docs_v1, _, _, _, _, _, _} = docs -> - case Code.ensure_loaded(module) do - {:module, _} -> - docs - - {:error, reason} -> - ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", []) - false - end + docs {:error, :chunk_not_found} -> false diff --git a/test/ex_doc/language/elixir_test.exs b/test/ex_doc/language/elixir_test.exs index b222412d6..fbadf55ba 100644 --- a/test/ex_doc/language/elixir_test.exs +++ b/test/ex_doc/language/elixir_test.exs @@ -606,28 +606,28 @@ defmodule ExDoc.Language.ElixirTest do ## filtered_modules assert warn(fn -> - opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]] + opts = opts ++ [filtered_modules: MapSet.new([String])] assert autolink_doc("`String`", opts) == ~s|String| end) =~ "reference to a filtered module" assert warn(fn -> - opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]] + opts = opts ++ [filtered_modules: MapSet.new([String])] assert autolink_doc("`String.upcase/1`", opts) == ~s|String.upcase/1| end) =~ "reference to a filtered module" assert warn(fn -> - opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]] + opts = opts ++ [filtered_modules: MapSet.new([String])] assert autolink_doc("`t:String.t/0`", opts) == ~s|t:String.t/0| end) =~ "reference to a filtered module" assert warn(fn -> - opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]] + opts = opts ++ [filtered_modules: MapSet.new([String])] autolink_spec(quote(do: t() :: String.t()), opts) end) =~ "reference to a filtered module" diff --git a/test/ex_doc/language/erlang_test.exs b/test/ex_doc/language/erlang_test.exs index 6dad99cd5..d15665e28 100644 --- a/test/ex_doc/language/erlang_test.exs +++ b/test/ex_doc/language/erlang_test.exs @@ -230,7 +230,7 @@ defmodule ExDoc.Language.ErlangTest do end test "filtered module", c do - opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]] + opts = [filtered_modules: MapSet.new([:lists])] assert warn(fn -> assert autolink_edoc("{@link lists}", c, opts) == @@ -239,7 +239,7 @@ defmodule ExDoc.Language.ErlangTest do end test "filtered module function", c do - opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]] + opts = [filtered_modules: MapSet.new([:lists])] assert warn(fn -> assert autolink_edoc("{@link lists:all/2}", c, opts) == @@ -548,7 +548,7 @@ defmodule ExDoc.Language.ErlangTest do end test "filtered module", c do - opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]] + opts = [filtered_modules: MapSet.new([:lists])] assert warn(fn -> assert autolink_doc("`m:lists`", c, opts) == @@ -557,7 +557,7 @@ defmodule ExDoc.Language.ErlangTest do end test "filtered module callback", c do - opts = [filtered_modules: [%ExDoc.ModuleNode{module: :gen_server, id: "gen_server"}]] + opts = [filtered_modules: MapSet.new([:gen_server])] assert warn(fn -> assert autolink_doc("`c:gen_server:handle_call/3`", c, opts) == diff --git a/test/ex_doc/retriever_test.exs b/test/ex_doc/retriever_test.exs index 89d221cc8..b78490be0 100644 --- a/test/ex_doc/retriever_test.exs +++ b/test/ex_doc/retriever_test.exs @@ -345,10 +345,9 @@ defmodule ExDoc.RetrieverTest do ebin_dir = Path.join(c.tmp_dir, "ebin") config = %ExDoc.Config{filter_modules: fn module, _ -> Atom.to_string(module) =~ "A" end} - assert { - [%{id: "A"}, %{id: "A.A"}], - [%{id: "B"}] - } = Retriever.docs_from_dir([ebin_dir], config) + assert {[%{id: "A"}, %{id: "A.A"}], [B]} = Retriever.docs_from_dir([ebin_dir], config) + assert :erlang.module_loaded(A) + refute :erlang.module_loaded(B) end test "natural sorting", c do From 98fb4c83a42e48455c430a6a56bbbb1fd89540c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Mon, 2 Nov 2026 19:20:15 +0100 Subject: [PATCH 2/2] Apply suggestion from @josevalim --- test/ex_doc/retriever_test.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/ex_doc/retriever_test.exs b/test/ex_doc/retriever_test.exs index b78490be0..964d8f688 100644 --- a/test/ex_doc/retriever_test.exs +++ b/test/ex_doc/retriever_test.exs @@ -346,8 +346,6 @@ defmodule ExDoc.RetrieverTest do config = %ExDoc.Config{filter_modules: fn module, _ -> Atom.to_string(module) =~ "A" end} assert {[%{id: "A"}, %{id: "A.A"}], [B]} = Retriever.docs_from_dir([ebin_dir], config) - assert :erlang.module_loaded(A) - refute :erlang.module_loaded(B) end test "natural sorting", c do