diff --git a/lib/ex_doc/autolink.ex b/lib/ex_doc/autolink.ex
index f8b46c1c0..bee52f95c 100644
--- a/lib/ex_doc/autolink.ex
+++ b/lib/ex_doc/autolink.ex
@@ -32,7 +32,7 @@ defmodule ExDoc.Autolink do
# * `:skip_code_autolink_to` - function that will be called with a term and return a boolean
# whether to skip autolinking to it.
#
- # * `:filtered_modules` - A list of module nodes that were filtered by the retriever
+ # * `:filtered_modules` - A set of the module names the retriever filtered out
#
# * `:warnings` - one of:
#
@@ -61,7 +61,7 @@ defmodule ExDoc.Autolink do
&ExDoc.Formatter.Config.skip_undefined_reference_warnings_on/1,
skip_code_autolink_to: &ExDoc.Formatter.Config.skip_code_autolink_to/1,
force_module_prefix: nil,
- filtered_modules: [],
+ filtered_modules: MapSet.new(),
assets: %{},
warnings: :emit
]
@@ -78,7 +78,7 @@ defmodule ExDoc.Autolink do
end
defp string_app_module_url(tool, module, anchor, config) do
- if Enum.any?(config.filtered_modules, &(&1.module == module)) do
+ if MapSet.member?(config.filtered_modules, module) do
warn(config, "reference to a filtered module")
nil
else
diff --git a/lib/ex_doc/formatter.ex b/lib/ex_doc/formatter.ex
index f8e2dca5e..236bb4a0a 100644
--- a/lib/ex_doc/formatter.ex
+++ b/lib/ex_doc/formatter.ex
@@ -24,7 +24,7 @@ defmodule ExDoc.Formatter do
@optional_callbacks autolink_options: 0
@doc false
- def run(formatter, formatter_config, module_nodes, filtered_nodes, extras) do
+ def run(formatter, formatter_config, module_nodes, filtered_modules, extras) do
if not Code.ensure_loaded?(formatter) do
raise "formatter module #{inspect(formatter)} not found"
end
@@ -35,7 +35,7 @@ defmodule ExDoc.Formatter do
{module_nodes, extras} =
if function_exported?(formatter, :autolink_options, 0) do
autolink_opts = formatter.autolink_options()
- autolink(formatter_config, module_nodes, filtered_nodes, extras, autolink_opts)
+ autolink(formatter_config, module_nodes, filtered_modules, extras, autolink_opts)
else
{module_nodes, extras}
end
@@ -180,7 +180,7 @@ defmodule ExDoc.Formatter do
## Autolinking
@doc false
- def autolink(config, nodes, filtered_nodes, extras, opts) do
+ def autolink(config, nodes, filtered_modules, extras, opts) do
{ext, highlight_opts} = Keyword.pop!(opts, :extension)
language =
@@ -196,7 +196,7 @@ defmodule ExDoc.Formatter do
extras: extra_paths(extras),
skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on,
skip_code_autolink_to: config.skip_code_autolink_to,
- filtered_modules: filtered_nodes,
+ filtered_modules: MapSet.new(filtered_modules),
assets: config.assets
}
diff --git a/lib/ex_doc/retriever.ex b/lib/ex_doc/retriever.ex
index 75dc342aa..ac573a70c 100644
--- a/lib/ex_doc/retriever.ex
+++ b/lib/ex_doc/retriever.ex
@@ -13,11 +13,11 @@ defmodule ExDoc.Retriever do
@doc """
Extract documentation from all modules in the specified directories.
- Returns a tuple containing `{modules, filtered}`, using `config.filter_modules`
- as a filter criteria.
+ Returns a tuple containing `{modules, filtered}`: the module nodes that
+ passed `config.filter_modules` and the names of the modules it rejected.
"""
@spec docs_from_dir([Path.t()], ExDoc.Config.t()) ::
- {[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]}
+ {[ExDoc.ModuleNode.t()], [module()]}
def docs_from_dir(dirs, config) when is_list(dirs) do
dirs
|> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config))
@@ -34,33 +34,65 @@ defmodule ExDoc.Retriever do
@doc """
Extract documentation from all modules and returns a tuple containing
- `{modules, filtered}`, two lists of modules that were extracted and filtered
- by `config.filter_modules`, respectively.
+ `{modules, filtered}`: the module nodes that passed `config.filter_modules`
+ and the names of the modules it rejected.
"""
@spec docs_from_modules([atom], ExDoc.Config.t()) ::
- {[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]}
+ {[ExDoc.ModuleNode.t()], [module()]}
def docs_from_modules(modules, config) when is_list(modules) do
modules |> docs_from_modules({[], []}, config) |> sort_modules(config)
end
- defp docs_from_modules(modules, acc, config) do
- modules
- |> Task.async_stream(&get_module(&1, config), timeout: :infinity)
- |> Enum.reduce(acc, fn {:ok, result}, {modules, filtered} = acc ->
- case result do
- {:error, _module} ->
+ # Filtering happens on the docs chunk alone, before the module is loaded and
+ # its specs, callbacks and docs are processed, so filtered modules cost only a
+ # chunk read. The kept modules are then loaded in one call: loading them one
+ # by one from several tasks serializes on the code server, which on large
+ # projects takes an order of magnitude longer than the rest of the retrieval.
+ defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do
+ {kept, filtered} =
+ modules
+ |> Task.async_stream(&filter_module(&1, config), timeout: :infinity)
+ |> Enum.reduce({[], acc_filtered}, fn
+ {:ok, {:kept, module, docs_chunk}}, {kept, filtered} ->
+ {[{module, docs_chunk} | kept], filtered}
+
+ {:ok, {:filtered, module}}, {kept, filtered} ->
+ {kept, [module | filtered]}
+
+ {:ok, :skip}, acc ->
acc
+ end)
- {:ok, module_node} ->
- if config.filter_modules.(module_node.module, module_node.metadata),
- do: {[module_node | modules], filtered},
- else: {modules, [module_node | filtered]}
- end
- end)
+ _ = Code.ensure_all_loaded(Enum.map(kept, &elem(&1, 0)))
+
+ modules =
+ kept
+ |> Task.async_stream(
+ fn {module, docs_chunk} -> get_module(module, docs_chunk, config) end,
+ timeout: :infinity
+ )
+ |> Enum.reduce(acc_modules, fn
+ {:ok, {:ok, module_node}}, modules -> [module_node | modules]
+ {:ok, {:error, _module}}, modules -> modules
+ end)
+
+ {modules, filtered}
+ end
+
+ defp filter_module(module, config) do
+ case docs_chunk(module) do
+ {:docs_v1, _, _, _, _, metadata, _} = docs_chunk ->
+ if config.filter_modules.(module, metadata),
+ do: {:kept, module, docs_chunk},
+ else: {:filtered, module}
+
+ _ ->
+ :skip
+ end
end
defp sort_modules({modules, filtered}, config) do
- {sort_modules(modules, config), sort_modules(filtered, config)}
+ {sort_modules(modules, config), Enum.sort(filtered)}
end
defp sort_modules(modules, config) when is_list(modules) do
@@ -75,8 +107,8 @@ defmodule ExDoc.Retriever do
String.to_atom(name)
end
- defp get_module(module, config) do
- with {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk <- docs_chunk(module),
+ defp get_module(module, {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk, config) do
+ with {:module, _} <- ensure_loaded(module),
{:ok, language} <- ExDoc.Language.get(language, module),
%{} = module_data <- language.module_data(module, docs_chunk, config) do
{:ok, generate_node(module, module_data, config)}
@@ -86,6 +118,17 @@ defmodule ExDoc.Retriever do
end
end
+ defp ensure_loaded(module) do
+ case Code.ensure_loaded(module) do
+ {:module, _} = loaded ->
+ loaded
+
+ {:error, reason} ->
+ ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", [])
+ false
+ end
+ end
+
defp docs_chunk(module) do
result = Code.fetch_docs(module)
Refs.insert_from_chunk(module, result)
@@ -95,14 +138,7 @@ defmodule ExDoc.Retriever do
false
{:docs_v1, _, _, _, _, _, _} = docs ->
- case Code.ensure_loaded(module) do
- {:module, _} ->
- docs
-
- {:error, reason} ->
- ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", [])
- false
- end
+ docs
{:error, :chunk_not_found} ->
false
diff --git a/test/ex_doc/language/elixir_test.exs b/test/ex_doc/language/elixir_test.exs
index b222412d6..fbadf55ba 100644
--- a/test/ex_doc/language/elixir_test.exs
+++ b/test/ex_doc/language/elixir_test.exs
@@ -606,28 +606,28 @@ defmodule ExDoc.Language.ElixirTest do
## filtered_modules
assert warn(fn ->
- opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
+ opts = opts ++ [filtered_modules: MapSet.new([String])]
assert autolink_doc("`String`", opts) ==
~s|String|
end) =~ "reference to a filtered module"
assert warn(fn ->
- opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
+ opts = opts ++ [filtered_modules: MapSet.new([String])]
assert autolink_doc("`String.upcase/1`", opts) ==
~s|String.upcase/1|
end) =~ "reference to a filtered module"
assert warn(fn ->
- opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
+ opts = opts ++ [filtered_modules: MapSet.new([String])]
assert autolink_doc("`t:String.t/0`", opts) ==
~s|t:String.t/0|
end) =~ "reference to a filtered module"
assert warn(fn ->
- opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
+ opts = opts ++ [filtered_modules: MapSet.new([String])]
autolink_spec(quote(do: t() :: String.t()), opts)
end) =~ "reference to a filtered module"
diff --git a/test/ex_doc/language/erlang_test.exs b/test/ex_doc/language/erlang_test.exs
index 6dad99cd5..d15665e28 100644
--- a/test/ex_doc/language/erlang_test.exs
+++ b/test/ex_doc/language/erlang_test.exs
@@ -230,7 +230,7 @@ defmodule ExDoc.Language.ErlangTest do
end
test "filtered module", c do
- opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
+ opts = [filtered_modules: MapSet.new([:lists])]
assert warn(fn ->
assert autolink_edoc("{@link lists}", c, opts) ==
@@ -239,7 +239,7 @@ defmodule ExDoc.Language.ErlangTest do
end
test "filtered module function", c do
- opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
+ opts = [filtered_modules: MapSet.new([:lists])]
assert warn(fn ->
assert autolink_edoc("{@link lists:all/2}", c, opts) ==
@@ -548,7 +548,7 @@ defmodule ExDoc.Language.ErlangTest do
end
test "filtered module", c do
- opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
+ opts = [filtered_modules: MapSet.new([:lists])]
assert warn(fn ->
assert autolink_doc("`m:lists`", c, opts) ==
@@ -557,7 +557,7 @@ defmodule ExDoc.Language.ErlangTest do
end
test "filtered module callback", c do
- opts = [filtered_modules: [%ExDoc.ModuleNode{module: :gen_server, id: "gen_server"}]]
+ opts = [filtered_modules: MapSet.new([:gen_server])]
assert warn(fn ->
assert autolink_doc("`c:gen_server:handle_call/3`", c, opts) ==
diff --git a/test/ex_doc/retriever_test.exs b/test/ex_doc/retriever_test.exs
index 89d221cc8..964d8f688 100644
--- a/test/ex_doc/retriever_test.exs
+++ b/test/ex_doc/retriever_test.exs
@@ -345,10 +345,7 @@ defmodule ExDoc.RetrieverTest do
ebin_dir = Path.join(c.tmp_dir, "ebin")
config = %ExDoc.Config{filter_modules: fn module, _ -> Atom.to_string(module) =~ "A" end}
- assert {
- [%{id: "A"}, %{id: "A.A"}],
- [%{id: "B"}]
- } = Retriever.docs_from_dir([ebin_dir], config)
+ assert {[%{id: "A"}, %{id: "A.A"}], [B]} = Retriever.docs_from_dir([ebin_dir], config)
end
test "natural sorting", c do