Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions lib/ex_doc/autolink.ex
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ defmodule ExDoc.Autolink do
# * `:skip_code_autolink_to` - function that will be called with a term and return a boolean
# whether to skip autolinking to it.
#
# * `:filtered_modules` - A list of module nodes that were filtered by the retriever
# * `:filtered_modules` - A set of the module names the retriever filtered out
#
# * `:warnings` - one of:
#
Expand Down Expand Up @@ -61,7 +61,7 @@ defmodule ExDoc.Autolink do
&ExDoc.Formatter.Config.skip_undefined_reference_warnings_on/1,
skip_code_autolink_to: &ExDoc.Formatter.Config.skip_code_autolink_to/1,
force_module_prefix: nil,
filtered_modules: [],
filtered_modules: MapSet.new(),
assets: %{},
warnings: :emit
]
Expand All @@ -78,7 +78,7 @@ defmodule ExDoc.Autolink do
end

defp string_app_module_url(tool, module, anchor, config) do
if Enum.any?(config.filtered_modules, &(&1.module == module)) do
if MapSet.member?(config.filtered_modules, module) do
warn(config, "reference to a filtered module")
nil
else
Expand Down
8 changes: 4 additions & 4 deletions lib/ex_doc/formatter.ex
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ defmodule ExDoc.Formatter do
@optional_callbacks autolink_options: 0

@doc false
def run(formatter, formatter_config, module_nodes, filtered_nodes, extras) do
def run(formatter, formatter_config, module_nodes, filtered_modules, extras) do
if not Code.ensure_loaded?(formatter) do
raise "formatter module #{inspect(formatter)} not found"
end
Expand All @@ -35,7 +35,7 @@ defmodule ExDoc.Formatter do
{module_nodes, extras} =
if function_exported?(formatter, :autolink_options, 0) do
autolink_opts = formatter.autolink_options()
autolink(formatter_config, module_nodes, filtered_nodes, extras, autolink_opts)
autolink(formatter_config, module_nodes, filtered_modules, extras, autolink_opts)
else
{module_nodes, extras}
end
Expand Down Expand Up @@ -180,7 +180,7 @@ defmodule ExDoc.Formatter do
## Autolinking

@doc false
def autolink(config, nodes, filtered_nodes, extras, opts) do
def autolink(config, nodes, filtered_modules, extras, opts) do
{ext, highlight_opts} = Keyword.pop!(opts, :extension)

language =
Expand All @@ -196,7 +196,7 @@ defmodule ExDoc.Formatter do
extras: extra_paths(extras),
skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on,
skip_code_autolink_to: config.skip_code_autolink_to,
filtered_modules: filtered_nodes,
filtered_modules: MapSet.new(filtered_modules),
assets: config.assets
}

Expand Down
94 changes: 65 additions & 29 deletions lib/ex_doc/retriever.ex
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ defmodule ExDoc.Retriever do
@doc """
Extract documentation from all modules in the specified directories.

Returns a tuple containing `{modules, filtered}`, using `config.filter_modules`
as a filter criteria.
Returns a tuple containing `{modules, filtered}`: the module nodes that
passed `config.filter_modules` and the names of the modules it rejected.
"""
@spec docs_from_dir([Path.t()], ExDoc.Config.t()) ::
{[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]}
{[ExDoc.ModuleNode.t()], [module()]}
def docs_from_dir(dirs, config) when is_list(dirs) do
dirs
|> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config))
Expand All @@ -34,33 +34,65 @@ defmodule ExDoc.Retriever do

@doc """
Extract documentation from all modules and returns a tuple containing
`{modules, filtered}`, two lists of modules that were extracted and filtered
by `config.filter_modules`, respectively.
`{modules, filtered}`: the module nodes that passed `config.filter_modules`
and the names of the modules it rejected.
"""
@spec docs_from_modules([atom], ExDoc.Config.t()) ::
{[ExDoc.ModuleNode.t()], [ExDoc.ModuleNode.t()]}
{[ExDoc.ModuleNode.t()], [module()]}
def docs_from_modules(modules, config) when is_list(modules) do
modules |> docs_from_modules({[], []}, config) |> sort_modules(config)
end

defp docs_from_modules(modules, acc, config) do
modules
|> Task.async_stream(&get_module(&1, config), timeout: :infinity)
|> Enum.reduce(acc, fn {:ok, result}, {modules, filtered} = acc ->
case result do
{:error, _module} ->
# Filtering happens on the docs chunk alone, before the module is loaded and
# its specs, callbacks and docs are processed, so filtered modules cost only a
# chunk read. The kept modules are then loaded in one call: loading them one
# by one from several tasks serializes on the code server, which on large
# projects takes an order of magnitude longer than the rest of the retrieval.
defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do
{kept, filtered} =
modules
|> Task.async_stream(&filter_module(&1, config), timeout: :infinity)
|> Enum.reduce({[], acc_filtered}, fn
{:ok, {:kept, module, docs_chunk}}, {kept, filtered} ->
{[{module, docs_chunk} | kept], filtered}

{:ok, {:filtered, module}}, {kept, filtered} ->
{kept, [module | filtered]}

{:ok, :skip}, acc ->
acc
end)

{:ok, module_node} ->
if config.filter_modules.(module_node.module, module_node.metadata),
do: {[module_node | modules], filtered},
else: {modules, [module_node | filtered]}
end
end)
_ = Code.ensure_all_loaded(Enum.map(kept, &elem(&1, 0)))

modules =
kept
|> Task.async_stream(
fn {module, docs_chunk} -> get_module(module, docs_chunk, config) end,
timeout: :infinity
)
|> Enum.reduce(acc_modules, fn
{:ok, {:ok, module_node}}, modules -> [module_node | modules]
{:ok, {:error, _module}}, modules -> modules
end)

{modules, filtered}
end

defp filter_module(module, config) do
case docs_chunk(module) do
{:docs_v1, _, _, _, _, metadata, _} = docs_chunk ->
if config.filter_modules.(module, metadata),
do: {:kept, module, docs_chunk},
else: {:filtered, module}

_ ->
:skip
end
end

defp sort_modules({modules, filtered}, config) do
{sort_modules(modules, config), sort_modules(filtered, config)}
{sort_modules(modules, config), Enum.sort(filtered)}
end

defp sort_modules(modules, config) when is_list(modules) do
Expand All @@ -75,8 +107,8 @@ defmodule ExDoc.Retriever do
String.to_atom(name)
end

defp get_module(module, config) do
with {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk <- docs_chunk(module),
defp get_module(module, {:docs_v1, _, language, _, _, _metadata, _} = docs_chunk, config) do
with {:module, _} <- ensure_loaded(module),
{:ok, language} <- ExDoc.Language.get(language, module),
%{} = module_data <- language.module_data(module, docs_chunk, config) do
{:ok, generate_node(module, module_data, config)}
Expand All @@ -86,6 +118,17 @@ defmodule ExDoc.Retriever do
end
end

defp ensure_loaded(module) do
case Code.ensure_loaded(module) do
{:module, _} = loaded ->
loaded

{:error, reason} ->
ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", [])
false
end
end

defp docs_chunk(module) do
result = Code.fetch_docs(module)
Refs.insert_from_chunk(module, result)
Expand All @@ -95,14 +138,7 @@ defmodule ExDoc.Retriever do
false

{:docs_v1, _, _, _, _, _, _} = docs ->
case Code.ensure_loaded(module) do
{:module, _} ->
docs

{:error, reason} ->
ExDoc.warn("skipping docs for module #{inspect(module)}, reason: #{reason}", [])
false
end
docs

{:error, :chunk_not_found} ->
false
Expand Down
8 changes: 4 additions & 4 deletions test/ex_doc/language/elixir_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -606,28 +606,28 @@ defmodule ExDoc.Language.ElixirTest do
## filtered_modules

assert warn(fn ->
opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
opts = opts ++ [filtered_modules: MapSet.new([String])]

assert autolink_doc("`String`", opts) ==
~s|<code class="inline">String</code>|
end) =~ "reference to a filtered module"

assert warn(fn ->
opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
opts = opts ++ [filtered_modules: MapSet.new([String])]

assert autolink_doc("`String.upcase/1`", opts) ==
~s|<code class="inline">String.upcase/1</code>|
end) =~ "reference to a filtered module"

assert warn(fn ->
opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
opts = opts ++ [filtered_modules: MapSet.new([String])]

assert autolink_doc("`t:String.t/0`", opts) ==
~s|<code class="inline">t:String.t/0</code>|
end) =~ "reference to a filtered module"

assert warn(fn ->
opts = opts ++ [filtered_modules: [%ExDoc.ModuleNode{module: String}]]
opts = opts ++ [filtered_modules: MapSet.new([String])]
autolink_spec(quote(do: t() :: String.t()), opts)
end) =~ "reference to a filtered module"

Expand Down
8 changes: 4 additions & 4 deletions test/ex_doc/language/erlang_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ defmodule ExDoc.Language.ErlangTest do
end

test "filtered module", c do
opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
opts = [filtered_modules: MapSet.new([:lists])]

assert warn(fn ->
assert autolink_edoc("{@link lists}", c, opts) ==
Expand All @@ -239,7 +239,7 @@ defmodule ExDoc.Language.ErlangTest do
end

test "filtered module function", c do
opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
opts = [filtered_modules: MapSet.new([:lists])]

assert warn(fn ->
assert autolink_edoc("{@link lists:all/2}", c, opts) ==
Expand Down Expand Up @@ -548,7 +548,7 @@ defmodule ExDoc.Language.ErlangTest do
end

test "filtered module", c do
opts = [filtered_modules: [%ExDoc.ModuleNode{module: :lists, id: "lists"}]]
opts = [filtered_modules: MapSet.new([:lists])]

assert warn(fn ->
assert autolink_doc("`m:lists`", c, opts) ==
Expand All @@ -557,7 +557,7 @@ defmodule ExDoc.Language.ErlangTest do
end

test "filtered module callback", c do
opts = [filtered_modules: [%ExDoc.ModuleNode{module: :gen_server, id: "gen_server"}]]
opts = [filtered_modules: MapSet.new([:gen_server])]

assert warn(fn ->
assert autolink_doc("`c:gen_server:handle_call/3`", c, opts) ==
Expand Down
5 changes: 1 addition & 4 deletions test/ex_doc/retriever_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -345,10 +345,7 @@ defmodule ExDoc.RetrieverTest do
ebin_dir = Path.join(c.tmp_dir, "ebin")
config = %ExDoc.Config{filter_modules: fn module, _ -> Atom.to_string(module) =~ "A" end}

assert {
[%{id: "A"}, %{id: "A.A"}],
[%{id: "B"}]
} = Retriever.docs_from_dir([ebin_dir], config)
assert {[%{id: "A"}, %{id: "A.A"}], [B]} = Retriever.docs_from_dir([ebin_dir], config)
end

test "natural sorting", c do
Expand Down
Loading