diff --git a/lib/ex_doc/autolink.ex b/lib/ex_doc/autolink.ex index bee52f95c..8b18f9587 100644 --- a/lib/ex_doc/autolink.ex +++ b/lib/ex_doc/autolink.ex @@ -114,6 +114,10 @@ defmodule ExDoc.Autolink do defp app_base_url(:otp, app), do: @otpappdocs <> "#{app}" defp app(module) do + ExDoc.Refs.memoize({:app, module}, fn -> find_app(module) end) + end + + defp find_app(module) do case :code.which(module) do :preloaded -> :erts @@ -150,8 +154,10 @@ defmodule ExDoc.Autolink do end defp is_app_otp(app) do - maybe_lib_dir_path = :code.lib_dir(app) - is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir()) + ExDoc.Refs.memoize({:otp_app?, app}, fn -> + maybe_lib_dir_path = :code.lib_dir(app) + is_list(maybe_lib_dir_path) and List.starts_with?(maybe_lib_dir_path, :code.root_dir()) + end) end def maybe_warn(config, ref, visibility, metadata) do diff --git a/lib/ex_doc/cli.ex b/lib/ex_doc/cli.ex index eb25b4dac..291b6cb05 100644 --- a/lib/ex_doc/cli.ex +++ b/lib/ex_doc/cli.ex @@ -51,11 +51,11 @@ defmodule ExDoc.CLI do defp generate(args, opts, generator) do [project, version | source_beams] = parse_args(args) - Code.prepend_paths(source_beams) + Code.prepend_paths(source_beams, cache: true) for path <- Keyword.get_values(opts, :paths), path <- Path.wildcard(path) do - Code.prepend_path(path) + Code.prepend_path(path, cache: true) app(path) |> Application.load() end diff --git a/lib/ex_doc/language/source.ex b/lib/ex_doc/language/source.ex index de297028a..353a4533a 100644 --- a/lib/ex_doc/language/source.ex +++ b/lib/ex_doc/language/source.ex @@ -22,7 +22,7 @@ defmodule ExDoc.Language.Source do have had their fields in-lined. """ def get_abstract_code(module) do - with {^module, binary, _file} <- :code.get_object_code(module), + with binary when is_binary(binary) <- get_object_code(module), {:ok, {_, [{:abstract_code, {_vsn, abst_code}}]}} <- :beam_lib.chunks(binary, [:abstract_code]) do expand_records_in_types(abst_code) @@ -31,6 +31,28 @@ defmodule ExDoc.Language.Source do end end + @doc """ + Get the object code of a module, or `nil` if it cannot be found. + + The beam file of a loaded module is read directly from the path it was + loaded from. `:code.get_object_code/1` would instead search the code path + and read the file through the code server and the primitive loader, which + are single processes, so concurrent callers queue up behind each other on + projects with many applications. + """ + def get_object_code(module) do + with path when is_list(path) and path != [] <- :code.which(module), + {:ok, binary} <- File.read(path) do + binary + else + _ -> + case :code.get_object_code(module) do + {^module, binary, _file} -> binary + :error -> nil + end + end + end + defp expand_records_in_types(abst_code) do # Find all records in ast and collect any fields with type declarations records = diff --git a/lib/ex_doc/refs.ex b/lib/ex_doc/refs.ex index ddf9173fb..ac7a0b3b6 100644 --- a/lib/ex_doc/refs.ex +++ b/lib/ex_doc/refs.ex @@ -17,6 +17,7 @@ defmodule ExDoc.Refs do @typep visibility() :: :hidden | :public | :undefined | :partial @name __MODULE__ + @memo Module.concat(__MODULE__, Memo) use GenServer @@ -28,15 +29,52 @@ defmodule ExDoc.Refs do @spec init(any()) :: {:ok, nil} def init(_) do :ets.new(@name, [:named_table, :public, :set]) + :ets.new(@memo, [:named_table, :public, :set, read_concurrency: true]) {:ok, nil} end @spec clear() :: :ok def clear() do :ets.delete_all_objects(@name) + :ets.delete_all_objects(@memo) :ok end + @doc """ + Returns the value of `fun`, computed once per `key` until the cache is cleared. + + Used for lookups that go through the code server or the application + controller, which are single processes, and whose cost grows with the + number of entries in the code path and of loaded applications. + """ + @spec memoize(term(), (-> value)) :: value when value: term() + def memoize(key, fun) do + case memo_lookup(key) do + {:ok, value} -> + value + + :error -> + value = fun.() + memo_insert(key, value) + value + end + end + + defp memo_lookup(key) do + case :ets.lookup(@memo, key) do + [{^key, value}] -> {:ok, value} + [] -> :error + end + rescue + _ -> :error + end + + defp memo_insert(key, value) do + :ets.insert(@memo, {key, value}) + rescue + _ -> true + end + @spec get_visibility(ref()) :: visibility() def get_visibility(ref) do case lookup(ref) do @@ -171,7 +209,7 @@ defmodule ExDoc.Refs do end defp types(module, kind_list) do - case Code.Typespec.fetch_types(module) do + case Code.Typespec.fetch_types(ExDoc.Language.Source.get_object_code(module) || module) do {:ok, list} -> for {kind, {name, _, args}} <- list, kind in kind_list do diff --git a/lib/ex_doc/retriever.ex b/lib/ex_doc/retriever.ex index ac573a70c..0c92f8b90 100644 --- a/lib/ex_doc/retriever.ex +++ b/lib/ex_doc/retriever.ex @@ -19,17 +19,14 @@ defmodule ExDoc.Retriever do @spec docs_from_dir([Path.t()], ExDoc.Config.t()) :: {[ExDoc.ModuleNode.t()], [module()]} def docs_from_dir(dirs, config) when is_list(dirs) do + # All directories are retrieved as a single batch, so projects made of + # many small applications are processed in parallel rather than one + # directory at a time. dirs - |> Enum.reduce({[], []}, &docs_from_dir(&1, &2, config)) - |> sort_modules(config) - end - - defp docs_from_dir(dir, acc, config) do - files = Path.wildcard(Path.expand("*.beam", dir)) - - files + |> Enum.flat_map(&Path.wildcard(Path.expand("*.beam", &1))) |> Enum.map(&filename_to_module/1) - |> docs_from_modules(acc, config) + |> docs_from_modules({[], []}, config) + |> sort_modules(config) end @doc """ @@ -48,9 +45,20 @@ defmodule ExDoc.Retriever do # chunk read. The kept modules are then loaded in one call: loading them one # by one from several tasks serializes on the code server, which on large # projects takes an order of magnitude longer than the rest of the retrieval. + # + # Modules without a docs chunk are loaded too, so their references can be + # computed from their exports, callbacks and types. They are also loaded in + # one call, after all chunks have been fetched, for the same reason. defp docs_from_modules(modules, {acc_modules, acc_filtered}, config) do - {kept, filtered} = + chunks = modules + |> Task.async_stream(&{&1, Code.fetch_docs(&1)}, timeout: :infinity) + |> Enum.map(fn {:ok, chunk} -> chunk end) + + _ = Code.ensure_all_loaded(for {module, {:error, _}} <- chunks, do: module) + + {kept, filtered} = + chunks |> Task.async_stream(&filter_module(&1, config), timeout: :infinity) |> Enum.reduce({[], acc_filtered}, fn {:ok, {:kept, module, docs_chunk}}, {kept, filtered} -> @@ -79,8 +87,8 @@ defmodule ExDoc.Retriever do {modules, filtered} end - defp filter_module(module, config) do - case docs_chunk(module) do + defp filter_module({module, result}, config) do + case docs_chunk(module, result) do {:docs_v1, _, _, _, _, metadata, _} = docs_chunk -> if config.filter_modules.(module, metadata), do: {:kept, module, docs_chunk}, @@ -129,8 +137,7 @@ defmodule ExDoc.Retriever do end end - defp docs_chunk(module) do - result = Code.fetch_docs(module) + defp docs_chunk(module, result) do Refs.insert_from_chunk(module, result) case result do