From abc938418e2a47d2b1f4fc116ab5c8ecacbfbcf6 Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Thu, 12 Nov 2026 16:52:19 +0100 Subject: [PATCH 1/5] Cache the code path entries of the documented applications Without the cache, every module lookup in the code server tries each code path entry in turn. The CLI prepends one entry per documented application, so on projects with hundreds of applications lookups get slower as the project grows, and they all queue up on the code server. --- lib/ex_doc/cli.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ex_doc/cli.ex b/lib/ex_doc/cli.ex index eb25b4dac..291b6cb05 100644 --- a/lib/ex_doc/cli.ex +++ b/lib/ex_doc/cli.ex @@ -51,11 +51,11 @@ defmodule ExDoc.CLI do defp generate(args, opts, generator) do [project, version | source_beams] = parse_args(args) - Code.prepend_paths(source_beams) + Code.prepend_paths(source_beams, cache: true) for path <- Keyword.get_values(opts, :paths), path <- Path.wildcard(path) do - Code.prepend_path(path) + Code.prepend_path(path, cache: true) app(path) |> Application.load() end From b85b1162669beaf994cd35c2bed01c62da2b1644 Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Thu, 12 Nov 2026 16:52:20 +0100 Subject: [PATCH 2/5] Retrieve docs from all source directories in one batch Each directory used to be processed on its own, waiting for its slowest module before moving on. Projects made of many small applications now get the same parallelism as a single large one. --- lib/ex_doc/retriever.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/ex_doc/retriever.ex b/lib/ex_doc/retriever.ex index ac573a70c..0a3fdc061 100644 --- a/lib/ex_doc/retriever.ex +++ b/lib/ex_doc/retriever.ex @@ -19,17 +19,14 @@ defmodule ExDoc.Retriever do @spec docs_from_dir([Path.t()], ExDoc.Config.t()) :: {[ExDoc.ModuleNode.t()], [module()]} def docs_from_dir(dirs, config) when is_list(dirs) do + # All directories are retrieved as a single batch, so projects made of + # many small applications are processed in parallel rather than one + # directory at a time. dirs - |> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config)) - |> sort_modules(config) - end - - defp docs_from_dir(dir, acc, config) do - files = Path.wildcard(Path.expand("*.beam", dir)) - - files + |> Enum.flat_map(&Path.wildcard(Path.expand("*.beam", &1))) |> Enum.map(&filename_to_module/1) - |> docs_from_modules(acc, config) + |> docs_from_modules({[], []}, config) + |> sort_modules(config) end @doc """ From 6cdf5233d25c3f28ca4f6a4b4f5645b1e3b63f45 Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Thu, 12 Nov 2026 16:52:20 +0100 Subject: [PATCH 3/5] Load modules without a docs chunk in one call Their references are computed from their exports, callbacks and types, which requires loading them. Loading them one by one from the filtering tasks serializes on the code server, like it did for the kept modules. --- lib/ex_doc/retriever.ex | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/ex_doc/retriever.ex b/lib/ex_doc/retriever.ex index 0a3fdc061..0c92f8b90 100644 --- a/lib/ex_doc/retriever.ex +++ b/lib/ex_doc/retriever.ex @@ -45,9 +45,20 @@ defmodule ExDoc.Retriever do # chunk read. The kept modules are then loaded in one call: loading them one # by one from several tasks serializes on the code server, which on large # projects takes an order of magnitude longer than the rest of the retrieval. + # + # Modules without a docs chunk are loaded too, so their references can be + # computed from their exports, callbacks and types. They are also loaded in + # one call, after all chunks have been fetched, for the same reason. defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do - {kept, filtered} = + chunks = modules + |> Task.async_stream(&{&1, Code.fetch_docs(&1)}, timeout: :infinity) + |> Enum.map(fn {:ok, chunk} -> chunk end) + + _ = Code.ensure_all_loaded(for {module, {:error, _}} <- chunks, do: module) + + {kept, filtered} = + chunks |> Task.async_stream(&filter_module(&1, config), timeout: :infinity) |> Enum.reduce({[], acc_filtered}, fn {:ok, {:kept, module, docs_chunk}}, {kept, filtered} -> @@ -76,8 +87,8 @@ defmodule ExDoc.Retriever do {modules, filtered} end - defp filter_module(module, config) do - case docs_chunk(module) do + defp filter_module({module, result}, config) do + case docs_chunk(module, result) do {:docs_v1, _, _, _, _, metadata, _} = docs_chunk -> if config.filter_modules.(module, metadata), do: {:kept, module, docs_chunk}, @@ -126,8 +137,7 @@ defmodule ExDoc.Retriever do end end - defp docs_chunk(module) do - result = Code.fetch_docs(module) + defp docs_chunk(module, result) do Refs.insert_from_chunk(module, result) case result do From 804d0e3b94c86d5302207ad2ff6c34babbbdd33b Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Thu, 12 Nov 2026 16:52:31 +0100 Subject: [PATCH 4/5] Memoize the application of modules when autolinking Finding the application of a module and whether it is an OTP one goes through the application controller and the code server for every reference, and both lookups get slower with the number of loaded applications and code path entries. --- lib/ex_doc/autolink.ex | 10 ++++++++-- lib/ex_doc/refs.ex | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/ex_doc/autolink.ex b/lib/ex_doc/autolink.ex index bee52f95c..8b18f9587 100644 --- a/lib/ex_doc/autolink.ex +++ b/lib/ex_doc/autolink.ex @@ -114,6 +114,10 @@ defmodule ExDoc.Autolink do defp app_base_url(:otp, app), do: @otpappdocs <> "#{app}" defp app(module) do + ExDoc.Refs.memoize({:app, module}, fn -> find_app(module) end) + end + + defp find_app(module) do case :code.which(module) do :preloaded -> :erts @@ -150,8 +154,10 @@ defmodule ExDoc.Autolink do end defp is_app_otp(app) do - maybe_lib_dir_path = :code.lib_dir(app) - is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir()) + ExDoc.Refs.memoize({:otp_app?, app}, fn -> + maybe_lib_dir_path = :code.lib_dir(app) + is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir()) + end) end def maybe_warn(config, ref, visibility, metadata) do diff --git a/lib/ex_doc/refs.ex b/lib/ex_doc/refs.ex index ddf9173fb..526c7a46b 100644 --- a/lib/ex_doc/refs.ex +++ b/lib/ex_doc/refs.ex @@ -17,6 +17,7 @@ defmodule ExDoc.Refs do @typep visibility() :: :hidden | :public | :undefined | :partial @name __MODULE__ + @memo Module.concat(__MODULE__, Memo) use GenServer @@ -28,15 +29,52 @@ defmodule ExDoc.Refs do @spec init(any()) :: {:ok, nil} def init(_) do :ets.new(@name, [:named_table, :public, :set]) + :ets.new(@memo, [:named_table, :public, :set, read_concurrency: true]) {:ok, nil} end @spec clear() :: :ok def clear() do :ets.delete_all_objects(@name) + :ets.delete_all_objects(@memo) :ok end + @doc """ + Returns the value of `fun`, computed once per `key` until the cache is cleared. + + Used for lookups that go through the code server or the application + controller, which are single processes, and whose cost grows with the + number of entries in the code path and of loaded applications. + """ + @spec memoize(term(), (-> value)) :: value when value: term() + def memoize(key, fun) do + case memo_lookup(key) do + {:ok, value} -> + value + + :error -> + value = fun.() + memo_insert(key, value) + value + end + end + + defp memo_lookup(key) do + case :ets.lookup(@memo, key) do + [{^key, value}] -> {:ok, value} + [] -> :error + end + rescue + _ -> :error + end + + defp memo_insert(key, value) do + :ets.insert(@memo, {key, value}) + rescue + _ -> true + end + @spec get_visibility(ref()) :: visibility() def get_visibility(ref) do case lookup(ref) do From c6134e5da3272c72b6e6b58b34981b5adb50309b Mon Sep 17 00:00:00 2001 From: Roberto Aloi Date: Thu, 12 Nov 2026 16:52:31 +0100 Subject: [PATCH 5/5] Read the beam files of loaded modules directly :code.get_object_code/1 searches the code path and reads the file through the code server and the primitive loader. Abstract code and types are fetched for most modules, concurrently, so these single processes become a bottleneck on large projects. --- lib/ex_doc/language/source.ex | 24 +++++++++++++++++++++++- lib/ex_doc/refs.ex | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/ex_doc/language/source.ex b/lib/ex_doc/language/source.ex index de297028a..353a4533a 100644 --- a/lib/ex_doc/language/source.ex +++ b/lib/ex_doc/language/source.ex @@ -22,7 +22,7 @@ defmodule ExDoc.Language.Source do have had their fields in-lined. """ def get_abstract_code(module) do - with {^module, binary, _file} <- :code.get_object_code(module), + with binary when is_binary(binary) <- get_object_code(module), {:ok, {_, [{:abstract_code, {_vsn, abst_code}}]}} <- :beam_lib.chunks(binary, [:abstract_code]) do expand_records_in_types(abst_code) @@ -31,6 +31,28 @@ defmodule ExDoc.Language.Source do end end + @doc """ + Get the object code of a module, or `nil` if it cannot be found. + + The beam file of a loaded module is read directly from the path it was + loaded from. `:code.get_object_code/1` would instead search the code path + and read the file through the code server and the primitive loader, which + are single processes, so concurrent callers queue up behind each other on + projects with many applications. + """ + def get_object_code(module) do + with path when is_list(path) and path != [] <- :code.which(module), + {:ok, binary} <- File.read(path) do + binary + else + _ -> + case :code.get_object_code(module) do + {^module, binary, _file} -> binary + :error -> nil + end + end + end + defp expand_records_in_types(abst_code) do # Find all records in ast and collect any fields with type declarations records = diff --git a/lib/ex_doc/refs.ex b/lib/ex_doc/refs.ex index 526c7a46b..ac7a0b3b6 100644 --- a/lib/ex_doc/refs.ex +++ b/lib/ex_doc/refs.ex @@ -209,7 +209,7 @@ defmodule ExDoc.Refs do end defp types(module, kind_list) do - case Code.Typespec.fetch_types(module) do + case Code.Typespec.fetch_types(ExDoc.Language.Source.get_object_code(module) || module) do {:ok, list} -> for {kind, {name, _, args}} <- list, kind in kind_list do