feat(builtin-dep): Add Builtin Source

benches/benchsuite/benches/resolve.rs

@@ -25,7 +25,7 @@ struct ResolveInfo<'gctx> {
 fn do_resolve<'gctx>(gctx: &'gctx GlobalContext, ws_root: &Path) -> ResolveInfo<'gctx> {
     let requested_kinds = [CompileKind::Host];
     let ws = Workspace::new(&ws_root.join("Cargo.toml"), gctx).unwrap();
-    let mut target_data = RustcTargetData::new(&ws, &requested_kinds).unwrap();
+    let mut target_data = RustcTargetData::new(gctx, Some(&ws), &requested_kinds).unwrap();
     let cli_features = CliFeatures::from_command_line(&[], false, true).unwrap();
     let pkgs = cargo::ops::Packages::Default;
     let specs = pkgs.to_package_id_specs(&ws).unwrap();

src/compiler/build_context/target_info.rs

@@ -989,11 +989,11 @@ pub struct RustcTargetData<'gctx> {
 impl<'gctx> RustcTargetData<'gctx> {
     #[tracing::instrument(skip_all)]
     pub fn new(
-        ws: &Workspace<'gctx>,
+        gctx: &'gctx GlobalContext,
+        ws: Option<&Workspace<'gctx>>,
         requested_kinds: &[CompileKind],
     ) -> CargoResult<RustcTargetData<'gctx>> {
-        let gctx = ws.gctx();
-        let rustc = gctx.load_global_rustc(Some(ws))?;
+        let rustc = gctx.load_global_rustc(ws)?;
         let mut target_config = HashMap::default();
         let mut target_info = HashMap::default();
         let target_applies_to_host = gctx.target_applies_to_host()?;
@@ -1054,16 +1054,14 @@ impl<'gctx> RustcTargetData<'gctx> {
                 .iter()
                 .filter_map(|d| d.artifact()?.target()?.to_compile_kind())
         }
-        let all_kinds = requested_kinds
-            .iter()
-            .copied()
-            .chain(ws.members().flat_map(|p| {
-                p.manifest()
-                    .default_kind()
-                    .into_iter()
-                    .chain(p.manifest().forced_kind())
-                    .chain(artifact_targets(p))
-            }));
+        let members = ws.into_iter().flat_map(|ws| ws.members());
+        let all_kinds = requested_kinds.iter().copied().chain(members.flat_map(|p| {
+            p.manifest()
+                .default_kind()
+                .into_iter()
+                .chain(p.manifest().forced_kind())
+                .chain(artifact_targets(p))
+        }));
         for kind in all_kinds {
             res.merge_compile_kind(kind)?;
         }

src/compiler/standard_lib.rs

@@ -218,7 +218,7 @@ fn generate_roots(
     Ok(())
 }
 
-fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
+pub(crate) fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {
     if let Some(s) = target_data.gctx.get_env_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
         return Ok(s.into());
     }

src/compiler/unit_dependencies.rs

@@ -1138,6 +1138,7 @@ impl<'a, 'gctx> State<'a, 'gctx> {
         let kind = unit.kind;
         self.resolve()
             .deps(pkg_id)
+            .filter(|(id, _)| !id.source_id().is_builtin())
             .filter_map(|(id, deps)| {
                 assert!(!deps.is_empty());
                 let deps: Vec<_> = deps

src/ops/cargo_clean.rs

@@ -195,7 +195,7 @@ fn clean_specs(
 ) -> CargoResult<()> {
     // Clean specific packages.
     let requested_kinds = CompileKind::from_requested_targets(clean_ctx.gctx, targets)?;
-    let target_data = RustcTargetData::new(ws, &requested_kinds)?;
+    let target_data = RustcTargetData::new(ws.gctx(), Some(ws), &requested_kinds)?;
     let (pkg_set, resolve) = ops::resolve_ws(ws, dry_run)?;
     let prof_dir_name = profiles.get_dir_name();
     let host_layout = Layout::new(ws, None, &prof_dir_name, true, true)?;

src/ops/cargo_compile/mod.rs

@@ -294,7 +294,7 @@ pub fn create_bcx<'a, 'gctx>(
     }
     gctx.validate_term_config()?;
 
-    let mut target_data = RustcTargetData::new(ws, &build_config.requested_kinds)?;
+    let mut target_data = RustcTargetData::new(gctx, Some(ws), &build_config.requested_kinds)?;
 
     let specs = spec.to_package_id_specs(ws)?;
     let has_dev_units = {

src/ops/cargo_fetch.rs

@@ -34,7 +34,7 @@ pub fn fetch<'a>(
     let gctx = ws.gctx();
     let build_config =
         BuildConfig::new(gctx, jobs, keep_going, &options.targets, UserIntent::Build)?;
-    let mut data = RustcTargetData::new(ws, &build_config.requested_kinds)?;
+    let mut data = RustcTargetData::new(ws.gctx(), Some(ws), &build_config.requested_kinds)?;
     let mut fetched_packages = HashSet::default();
     let mut deps_to_fetch = ws.members().map(|p| p.package_id()).collect::<Vec<_>>();
     let mut to_download = Vec::new();

src/ops/cargo_fix/mod.rs

@@ -150,8 +150,11 @@ pub fn fix(
 ) -> CargoResult<()> {
     check_version_control(gctx, opts)?;
 
-    let mut target_data =
-        RustcTargetData::new(original_ws, &opts.compile_opts.build_config.requested_kinds)?;
+    let mut target_data = RustcTargetData::new(
+        gctx,
+        Some(original_ws),
+        &opts.compile_opts.build_config.requested_kinds,
+    )?;
 
     let specs = opts.compile_opts.spec.to_package_id_specs(&original_ws)?;
     let members: Vec<&Package> = original_ws

src/ops/cargo_metadata.rs

@@ -144,7 +144,7 @@ fn build_resolve_graph(
         &metadata_opts.filter_platforms,
         CompileKindFallback::JustHost,
     )?;
-    let mut target_data = RustcTargetData::new(ws, &requested_kinds)?;
+    let mut target_data = RustcTargetData::new(ws.gctx(), Some(ws), &requested_kinds)?;
     // Resolve entire workspace.
     let specs = Packages::All(Vec::new()).to_package_id_specs(ws)?;
     let force_all = if metadata_opts.filter_platforms.is_empty() {

src/ops/cargo_tree/mod.rs

@@ -148,7 +148,7 @@ pub fn build_and_print(ws: &Workspace<'_>, opts: &TreeOptions) -> CargoResult<()
     // TODO: Target::All is broken with -Zfeatures=itarget. To handle that properly,
     // `FeatureResolver` will need to be taught what "all" means.
     let requested_kinds = CompileKind::from_requested_targets(ws.gctx(), &requested_targets)?;
-    let mut target_data = RustcTargetData::new(ws, &requested_kinds)?;
+    let mut target_data = RustcTargetData::new(ws.gctx(), Some(ws), &requested_kinds)?;
     let specs = opts.packages.to_package_id_specs(ws)?;
     let has_dev = if opts
         .edge_kinds

src/resolver/encode.rs

@@ -661,7 +661,8 @@ pub fn encodable_package_id(
 }
 
 fn encodable_source_id(id: SourceId, version: ResolveVersion) -> Option<TomlLockfileSourceId> {
-    if id.is_path() {
+    // Lockfile support for builtins isn't yet implemented.
+    if id.is_path() || id.is_builtin() {
         None
     } else {
         Some(

src/sources/builtin.rs

@@ -0,0 +1,128 @@
+use crate::{
+    CargoResult, GlobalContext,
+    compiler::{CompileKind, RustcTargetData, standard_lib::detect_sysroot_src_path},
+    sources::{
+        IndexSummary, RecursivePathSource,
+        source::{MaybePackage, QueryKind, Source},
+    },
+    workspace::{Dependency, Package, PackageId, SourceId},
+};
+
+/// A builtin source represents standard library packages used in build-std, which are "built into"
+/// the toolchain.
+///
+/// It's a thin wrapper around a [`RecursivePathSource`] located in the sysroot's library source
+/// path.
+pub struct BuiltinSource<'gctx> {
+    /// The unique identifier for this source
+    source_id: SourceId,
+    /// The underlying path source which discovers packages
+    path_source: RecursivePathSource<'gctx>,
+}
+
+impl<'gctx> BuiltinSource<'gctx> {
+    pub fn new(source_id: SourceId, gctx: &'gctx GlobalContext) -> CargoResult<Self> {
+        let target_data = RustcTargetData::new(gctx, None, &[CompileKind::Host])?;
+        let path = detect_sysroot_src_path(&target_data)?;
+        let path_source = RecursivePathSource::new(&path, source_id, gctx);
+        Ok(Self {
+            source_id,
+            path_source,
+        })
+    }
+}
+
+#[async_trait::async_trait(?Send)]
+impl<'gctx> Source for BuiltinSource<'gctx> {
+    /// All builtin dependencies are opaque, so this will return a summary without any dependencies when queried
+    async fn query(
+        &self,
+        dep: &Dependency,
+        kind: QueryKind,
+        f: &mut dyn FnMut(IndexSummary),
+    ) -> CargoResult<()> {
+        if !dep.source_id().is_builtin() {
+            // Avoid loading packages in the path source if it's not needed
+            return Ok(());
+        }
+        self.path_source.query(dep, kind, f).await
+    }
+
+    fn supports_checksums(&self) -> bool {
+        self.path_source.supports_checksums()
+    }
+
+    fn requires_precise(&self) -> bool {
+        self.path_source.requires_precise()
+    }
+
+    fn source_id(&self) -> SourceId {
+        self.source_id
+    }
+
+    async fn download(&self, id: PackageId) -> CargoResult<MaybePackage> {
+        self.path_source.download(id).await
+    }
+
+    async fn finish_download(&self, id: PackageId, data: Vec<u8>) -> CargoResult<Package> {
+        self.path_source.finish_download(id, data).await
+    }
+
+    fn fingerprint(&self, pkg: &Package) -> CargoResult<String> {
+        self.path_source.fingerprint(pkg)
+    }
+
+    fn describe(&self) -> String {
+        self.source_id.to_string()
+    }
+
+    fn invalidate_cache(&self) {
+        self.path_source.invalidate_cache();
+    }
+
+    fn set_quiet(&mut self, quiet: bool) {
+        self.path_source.set_quiet(quiet);
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::path::Path;
+
+    use crate::{
+        GlobalContext,
+        sources::IndexSummary,
+        util::{VersionReqMatchMode, data_structures::HashMap},
+        workspace::{Dependency, SourceId},
+    };
+
+    #[test]
+    fn builtin_source() {
+        let mut gctx = GlobalContext::default().unwrap();
+        let mock_std_root =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/testsuite/mock-std/library");
+        let env = HashMap::from_iter([(
+            "__CARGO_TESTS_ONLY_SRC_ROOT".to_string(),
+            mock_std_root.display().to_string(),
+        )]);
+        gctx.set_env(env);
+
+        let source_id = SourceId::for_builtin().unwrap();
+        let dep = Dependency::new_override("core".into(), source_id);
+
+        let source = dep.source_id().load(&gctx).unwrap();
+        let results =
+            crate::util::block_on(source.query_vec(&dep, crate::sources::source::QueryKind::Exact))
+                .unwrap();
+
+        assert_eq!(results.len(), 1);
+        let result = results[0].clone();
+        if let IndexSummary::Candidate(s) = result {
+            assert!(dep.matches(&s, VersionReqMatchMode::Default));
+            assert!(s.dependencies().is_empty());
+            assert!(s.source_id().is_builtin())
+        } else {
+            panic!("no candidate found");
+        };
+    }
+}

src/sources/mod.rs

@@ -26,6 +26,7 @@
 //!
 //! [source replacement]: https://doc.rust-lang.org/nightly/cargo/reference/source-replacement.html
 
+pub use self::builtin::BuiltinSource;
 pub use self::config::SourceConfigMap;
 pub use self::directory::DirectorySource;
 pub use self::git::GitSource;
@@ -37,6 +38,7 @@ pub use self::registry::{
 };
 pub use self::replaced::ReplacedSource;
 
+pub mod builtin;
 pub mod config;
 pub mod directory;
 pub mod git;

src/sources/path.rs

@@ -1189,12 +1189,15 @@ fn read_nested_packages(
     // Registry sources are not allowed to have `path=` dependencies because
     // they're all translated to actual registry dependencies.
     //
+    // The standard library source intentionally does not include some test crates leading to broken
+    // dev-dependencies links. The main directory walk already gets all the packages we need.
+    //
     // We normalize the path here ensure that we don't infinitely walk around
     // looking for crates. By normalizing we ensure that we visit this crate at
     // most once.
     //
     // TODO: filesystem/symlink implications?
-    if !source_id.is_registry() {
+    if !source_id.is_registry() && !source_id.is_builtin() {
         for p in nested.iter() {
             let path = paths::normalize_path(&path.join(p));
             let result =

src/util/command_prelude.rs

@@ -1416,7 +1416,7 @@ fn get_packages() -> CargoResult<Vec<Package>> {
     let ws = Workspace::new(&find_root_manifest_for_wd(gctx.cwd())?, &gctx)?;
 
     let requested_kinds = CompileKind::from_requested_targets(ws.gctx(), &[])?;
-    let mut target_data = RustcTargetData::new(&ws, &requested_kinds)?;
+    let mut target_data = RustcTargetData::new(&gctx, Some(&ws), &requested_kinds)?;
     // `cli_features.all_features` must be true in case that `specs` is empty.
     let cli_features = CliFeatures::new_all(true);
     let has_dev_units = HasDevUnits::Yes;

src/workspace/source_id.rs

@@ -1,7 +1,9 @@
 use crate::context;
 use crate::sources::registry::CRATES_IO_HTTP_INDEX;
 use crate::sources::source::Source;
-use crate::sources::{CRATES_IO_DOMAIN, CRATES_IO_INDEX, CRATES_IO_REGISTRY, DirectorySource};
+use crate::sources::{
+    BuiltinSource, CRATES_IO_DOMAIN, CRATES_IO_INDEX, CRATES_IO_REGISTRY, DirectorySource,
+};
 use crate::sources::{GitSource, PathSource, RegistrySource};
 use crate::util::data_structures::HashSet;
 use crate::util::interning::InternedString;
@@ -395,6 +397,11 @@ impl SourceId {
         matches!(self.inner.kind, SourceKind::Git(_))
     }
 
+    /// Returns `true` if this source is builtin.
+    pub fn is_builtin(self) -> bool {
+        matches!(self.inner.kind, SourceKind::Builtin)
+    }
+
     /// Creates an implementation of `Source` corresponding to this ID.
     pub fn load<'a>(self, gctx: &'a GlobalContext) -> CargoResult<Box<dyn Source + 'a>> {
         trace!("loading SourceId; {}", self);
@@ -411,7 +418,7 @@ impl SourceId {
                 }
                 Ok(Box::new(PathSource::new(&path, self, gctx)))
             }
-            SourceKind::Builtin => todo!("builtin source"),
+            SourceKind::Builtin => Ok(Box::new(BuiltinSource::new(self, gctx)?)),
             SourceKind::Registry | SourceKind::SparseRegistry => {
                 Ok(Box::new(RegistrySource::remote(self, gctx)?))
             }