Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions lib/ex_doc/autolink.ex
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ defmodule ExDoc.Autolink do
defp app_base_url(:otp, app), do: @otpappdocs <> "#{app}"

defp app(module) do
ExDoc.Refs.memoize({:app, module}, fn -> find_app(module) end)
end

defp find_app(module) do
case :code.which(module) do
:preloaded ->
:erts
Expand Down Expand Up @@ -150,8 +154,10 @@ defmodule ExDoc.Autolink do
end

defp is_app_otp(app) do
maybe_lib_dir_path = :code.lib_dir(app)
is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir())
ExDoc.Refs.memoize({:otp_app?, app}, fn ->
maybe_lib_dir_path = :code.lib_dir(app)
is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir())
end)
end

def maybe_warn(config, ref, visibility, metadata) do
Expand Down
4 changes: 2 additions & 2 deletions lib/ex_doc/cli.ex
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@ defmodule ExDoc.CLI do

defp generate(args, opts, generator) do
[project, version | source_beams] = parse_args(args)
Code.prepend_paths(source_beams)
Code.prepend_paths(source_beams, cache: true)

for path <- Keyword.get_values(opts, :paths),
path <- Path.wildcard(path) do
Code.prepend_path(path)
Code.prepend_path(path, cache: true)
app(path) |> Application.load()
end

Expand Down
24 changes: 23 additions & 1 deletion lib/ex_doc/language/source.ex
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ defmodule ExDoc.Language.Source do
have had their fields in-lined.
"""
def get_abstract_code(module) do
with {^module, binary, _file} <- :code.get_object_code(module),
with binary when is_binary(binary) <- get_object_code(module),
{:ok, {_, [{:abstract_code, {_vsn, abst_code}}]}} <-
:beam_lib.chunks(binary, [:abstract_code]) do
expand_records_in_types(abst_code)
Expand All @@ -31,6 +31,28 @@ defmodule ExDoc.Language.Source do
end
end

@doc """
Get the object code of a module, or `nil` if it cannot be found.

The beam file of a loaded module is read directly from the path it was
loaded from. `:code.get_object_code/1` would instead search the code path
and read the file through the code server and the primitive loader, which
are single processes, so concurrent callers queue up behind each other on
projects with many applications.
"""
def get_object_code(module) do
with path when is_list(path) and path != [] <- :code.which(module),
{:ok, binary} <- File.read(path) do
binary
else
_ ->
case :code.get_object_code(module) do
{^module, binary, _file} -> binary
:error -> nil
end
end
end

defp expand_records_in_types(abst_code) do
# Find all records in ast and collect any fields with type declarations
records =
Expand Down
40 changes: 39 additions & 1 deletion lib/ex_doc/refs.ex
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ defmodule ExDoc.Refs do
@typep visibility() :: :hidden | :public | :undefined | :partial

@name __MODULE__
@memo Module.concat(__MODULE__, Memo)

use GenServer

Expand All @@ -28,15 +29,52 @@ defmodule ExDoc.Refs do
@spec init(any()) :: {:ok, nil}
def init(_) do
:ets.new(@name, [:named_table, :public, :set])
:ets.new(@memo, [:named_table, :public, :set, read_concurrency: true])
{:ok, nil}
end

@spec clear() :: :ok
def clear() do
:ets.delete_all_objects(@name)
:ets.delete_all_objects(@memo)
:ok
end

@doc """
Returns the value of `fun`, computed once per `key` until the cache is cleared.

Used for lookups that go through the code server or the application
controller, which are single processes, and whose cost grows with the
number of entries in the code path and of loaded applications.
"""
@spec memoize(term(), (-> value)) :: value when value: term()
def memoize(key, fun) do
case memo_lookup(key) do
{:ok, value} ->
value

:error ->
value = fun.()
memo_insert(key, value)
value
end
end

defp memo_lookup(key) do
case :ets.lookup(@memo, key) do
[{^key, value}] -> {:ok, value}
[] -> :error
end
rescue
_ -> :error
end

defp memo_insert(key, value) do
:ets.insert(@memo, {key, value})
rescue
_ -> true
end

@spec get_visibility(ref()) :: visibility()
def get_visibility(ref) do
case lookup(ref) do
Expand Down Expand Up @@ -171,7 +209,7 @@ defmodule ExDoc.Refs do
end

defp types(module, kind_list) do
case Code.Typespec.fetch_types(module) do
case Code.Typespec.fetch_types(ExDoc.Language.Source.get_object_code(module) || module) do
{:ok, list} ->
for {kind, {name, _, args}} <- list,
kind in kind_list do
Expand Down
35 changes: 21 additions & 14 deletions lib/ex_doc/retriever.ex
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,14 @@ defmodule ExDoc.Retriever do
@spec docs_from_dir([Path.t()], ExDoc.Config.t()) ::
{[ExDoc.ModuleNode.t()], [module()]}
def docs_from_dir(dirs, config) when is_list(dirs) do
# All directories are retrieved as a single batch, so projects made of
# many small applications are processed in parallel rather than one
# directory at a time.
dirs
|> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config))
|> sort_modules(config)
end

defp docs_from_dir(dir, acc, config) do
files = Path.wildcard(Path.expand("*.beam", dir))

files
|> Enum.flat_map(&Path.wildcard(Path.expand("*.beam", &1)))
|> Enum.map(&filename_to_module/1)
|> docs_from_modules(acc, config)
|> docs_from_modules({[], []}, config)
|> sort_modules(config)
end

@doc """
Expand All @@ -48,9 +45,20 @@ defmodule ExDoc.Retriever do
# chunk read. The kept modules are then loaded in one call: loading them one
# by one from several tasks serializes on the code server, which on large
# projects takes an order of magnitude longer than the rest of the retrieval.
#
# Modules without a docs chunk are loaded too, so their references can be
# computed from their exports, callbacks and types. They are also loaded in
# one call, after all chunks have been fetched, for the same reason.
defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do
{kept, filtered} =
chunks =
modules
|> Task.async_stream(&{&1, Code.fetch_docs(&1)}, timeout: :infinity)
|> Enum.map(fn {:ok, chunk} -> chunk end)

_ = Code.ensure_all_loaded(for {module, {:error, _}} <- chunks, do: module)

{kept, filtered} =
chunks
|> Task.async_stream(&filter_module(&1, config), timeout: :infinity)
|> Enum.reduce({[], acc_filtered}, fn
{:ok, {:kept, module, docs_chunk}}, {kept, filtered} ->
Expand Down Expand Up @@ -79,8 +87,8 @@ defmodule ExDoc.Retriever do
{modules, filtered}
end

defp filter_module(module, config) do
case docs_chunk(module) do
defp filter_module({module, result}, config) do
case docs_chunk(module, result) do
{:docs_v1, _, _, _, _, metadata, _} = docs_chunk ->
if config.filter_modules.(module, metadata),
do: {:kept, module, docs_chunk},
Expand Down Expand Up @@ -129,8 +137,7 @@ defmodule ExDoc.Retriever do
end
end

defp docs_chunk(module) do
result = Code.fetch_docs(module)
defp docs_chunk(module, result) do
Refs.insert_from_chunk(module, result)

case result do
Expand Down
Loading