comptime read files experiments

compiler/rustc_const_eval/src/const_eval/comptime_fs.rs

@@ -0,0 +1,57 @@
+use std::io::ErrorKind;
+
+use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_middle::comptime::ReadFileError;
+use rustc_middle::query::LocalCrate;
+use rustc_middle::ty::TyCtxt;
+use rustc_span::Symbol;
+use tracing::instrument;
+
+#[instrument(skip(tcx), level = "debug")]
+pub fn experimental_comptime_list_files_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> &[Symbol] {
+    let filelist = tcx
+        .sess
+        .opts
+        .unstable_opts
+        .experimental_comptime_files
+        .iter()
+        .map(|p| Symbol::intern(p))
+        .collect::<Vec<_>>();
+
+    tcx.arena.alloc_slice(&filelist)
+}
+
+#[instrument(skip(tcx), level = "debug")]
+pub fn experimental_comptime_fetch_file_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    key: Symbol,
+) -> Result<Option<&'tcx [u8]>, ReadFileError<'tcx>> {
+    // In case we are in a context without a crate root, just return that
+    // every file is whitelisted but does not exist. Is this code path
+    // taken by rustdoc when checking a comment code block?
+    let Some(crate_root_path) = tcx.sess.io.input.opt_path() else {
+        return Ok(None);
+    };
+
+    let requested_path = key.as_str();
+
+    // Always resolve path from the crate root to provide consistent behavior.
+    let resolved_path = crate_root_path.with_file_name(requested_path);
+
+    // Very simple implementation where we whitelist paths directly. We don't even attempt
+    // to canonicalize for now.
+    if tcx
+        .experimental_comptime_list_files(LOCAL_CRATE)
+        .iter()
+        .find(|s| requested_path == s.as_str())
+        .is_some()
+    {
+        match std::fs::read(resolved_path) {
+            Ok(content) => Ok(Some(tcx.arena.alloc_slice(&content))),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
+            Err(e) => Err(ReadFileError::IO(tcx.arena.alloc_str(&e.to_string()))),
+        }
+    } else {
+        Err(ReadFileError::NotWhitelisted)
+    }
+}

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -10,6 +10,7 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexMap, IndexEntry};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{CRATE_HIR_ID, HirId};
 use rustc_lint_defs::builtin::LONG_RUNNING_CONST_EVAL;
+use rustc_middle::comptime::ReadFileError;
 use rustc_middle::mir::AssertMessage;
 use rustc_middle::mir::interpret::ReportedErrorInfo;
 use rustc_middle::query::TyCtxtAt;
@@ -24,10 +25,10 @@ use super::error::*;
 use crate::diagnostics::{LongRunning, LongRunningWarn};
 use crate::interpret::{
     self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
-    GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet,
-    RetagMode, Scalar, compile_time_machine, ensure_monomorphic_enough, err_inval, interp_ok,
-    throw_exhaust, throw_inval, throw_ub, throw_ub_format, throw_unsup, throw_unsup_format,
-    type_implements_dyn_trait,
+    GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, Projectable,
+    RangeSet, RetagMode, Scalar, compile_time_machine, ensure_monomorphic_enough, err_inval,
+    interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_format, throw_unsup,
+    throw_unsup_format, type_implements_dyn_trait,
 };
 
 /// When hitting this many interpreted terminators we emit a deny by default lint
@@ -598,6 +599,112 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                 }
             }
 
+            sym::comptime_fetch_file => {
+                let arg_place = ecx.deref_pointer(&args[0])?;
+                let path = ecx.read_str(&arg_place)?;
+                let path_sym = Symbol::intern(path);
+
+                let krate = ecx
+                    .stack()
+                    .first()
+                    .expect("should have a call frame if we are here")
+                    .instance()
+                    .def_id()
+                    .krate;
+
+                let content_maybe = match ecx
+                    .tcx
+                    .experimental_comptime_fetch_file((krate, path_sym))
+                {
+                    Ok(content) => content,
+                    Err(ReadFileError::IO(err)) => {
+                        let guard = ecx
+                            .tcx
+                            .dcx()
+                            .struct_err("error while reading file in constant evaluation")
+                            .with_span(ecx.cur_span())
+                            .with_span_help(ecx.tcx.span, "while evaluating this constant")
+                            .with_note(format!("filename: {path_sym}"))
+                            .with_note(format!("error: {err}"))
+                            .emit_err();
+                        throw_inval!(AlreadyReported(ReportedErrorInfo::allowed_in_infallible(
+                            guard
+                        )));
+                    }
+                    Err(ReadFileError::NotWhitelisted) => {
+                        let guard = ecx
+                            .tcx
+                            .dcx()
+                            .struct_err("this file cannot be accessed during const-evaluation")
+                            .with_span(ecx.cur_span())
+                            .with_span_help(ecx.tcx.span, "while evaluating this constant")
+                            .with_note(format!("filename: {path_sym}"))
+                            .with_help("pass -Zexperimental-comptime-files=<path> to the compiler to allow reading this file")
+                            .emit_err();
+                        throw_inval!(AlreadyReported(ReportedErrorInfo::allowed_in_infallible(
+                            guard
+                        )));
+                    }
+                };
+
+                let variant_index = if let Some(value) = content_maybe {
+                    let ptr = ecx.allocate_bytes_dedup(value)?;
+
+                    let (variant_idx, variant_place) =
+                        ecx.project_downcast_named(dest, sym::Some)?;
+                    let field_place = ecx.project_field(&variant_place, FieldIdx::ZERO)?;
+                    ecx.write_immediate(
+                        Immediate::ScalarPair(
+                            Scalar::from_pointer(ptr, ecx),
+                            Scalar::from_target_usize(value.len() as u64, ecx),
+                        ),
+                        &field_place,
+                    )?;
+
+                    variant_idx
+                } else {
+                    ecx.project_downcast_named(dest, sym::None)?.0
+                };
+
+                ecx.write_discriminant(variant_index, dest)?;
+            }
+
+            sym::comptime_list_files => {
+                let krate = ecx
+                    .stack()
+                    .first()
+                    .expect("should have a call frame if we are here")
+                    .instance()
+                    .def_id()
+                    .krate;
+
+                let filelist = ecx.tcx().experimental_comptime_list_files(krate);
+
+                let element_type =
+                    dest.layout().ty.builtin_deref(false).unwrap().sequence_element_type(ecx.tcx());
+
+                // type and layout of [&str, filelist.len()];
+                let array_ty =
+                    ecx.layout_of(Ty::new_array(ecx.tcx(), element_type, filelist.len() as _))?;
+
+                // actual allocated array
+                let array_place = ecx.allocate(array_ty, interpret::MemoryKind::Stack)?;
+
+                let mut array_iter = ecx.project_array_fields(&array_place)?;
+                while let Some((idx, entry_place)) = array_iter.next(ecx)? {
+                    let file = filelist[idx as usize];
+                    let file_place = ecx.allocate_str_dedup(file.as_str())?;
+                    let imm = file_place.to_ref(ecx);
+                    ecx.write_immediate(imm, &entry_place)?;
+                }
+
+                let array_place = array_place.map_provenance(CtfeProvenance::as_immutable);
+                ecx.write_immediate(
+                    Immediate::new_slice(array_place.ptr(), filelist.len() as _, ecx),
+                    dest,
+                )?;
+            }
+
             sym::type_id_vtable => {
                 let tp_ty = ecx.read_type_id(&args[0])?;
                 let result_ty = ecx.read_type_id(&args[1])?;

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -8,6 +8,7 @@ use tracing::instrument;
 
 use crate::interpret::InterpCx;
 
+pub(crate) mod comptime_fs;
 mod dummy_machine;
 mod dyn_trait;
 mod error;

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -714,10 +714,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         self.read_scalar(op)?.to_target_isize(self)
     }
 
-    /// Turn the wide MPlace into a string (must already be dereferenced!)
-    pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
+    /// Turn the wide MPlace into a slice of bytes (must already be dereferenced!)
+    pub fn read_bytes(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &[u8]> {
         let len = mplace.len(self)?;
         let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
+        interp_ok(bytes)
+    }
+
+    /// Turn the wide MPlace into a string (must already be dereferenced!)
+    pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
+        let bytes = self.read_bytes(mplace)?;
         let s = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
         interp_ok(s)
     }

compiler/rustc_const_eval/src/lib.rs

@@ -61,6 +61,10 @@ pub fn provide(providers: &mut Providers) {
     };
     providers.hooks.validate_scalar_in_layout =
         |tcx, scalar, layout| util::validate_scalar_in_layout(tcx, scalar, layout);
+    providers.queries.experimental_comptime_fetch_file =
+        const_eval::comptime_fs::experimental_comptime_fetch_file_provider;
+    providers.queries.experimental_comptime_list_files =
+        const_eval::comptime_fs::experimental_comptime_list_files_provider;
 }
 
 /// `rustc_driver::main` installs a handler that will set this to `true` if

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -85,6 +85,8 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
         | sym::ceil
         | sym::codeview_annotation
         | sym::cold_path
+        | sym::comptime_fetch_file
+        | sym::comptime_list_files
         | sym::const_eval_select
         | sym::contract_check_ensures
         | sym::contract_check_requires
@@ -288,6 +290,23 @@ pub(crate) fn check_intrinsic_type(
         }
         sym::needs_drop => (1, 0, vec![], tcx.types.bool),
 
+        sym::comptime_fetch_file => {
+            let slice_u8 = Ty::new_slice(tcx, tcx.types.u8);
+            let slice_u8_static: Ty<'_> = Ty::new_imm_ref(tcx, tcx.lifetimes.re_static, slice_u8);
+
+            let br = ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon };
+            let var_name =
+                Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), tcx.types.str_);
+
+            (0, 0, vec![var_name], Ty::new_option(tcx, slice_u8_static))
+        }
+
+        sym::comptime_list_files => {
+            let slice = Ty::new_slice(tcx, Ty::new_static_str(tcx));
+            let slice_static: Ty<'_> = Ty::new_imm_ref(tcx, tcx.lifetimes.re_static, slice);
+            (0, 0, vec![], slice_static)
+        }
+
         sym::type_name => (1, 0, vec![], Ty::new_static_str(tcx)),
         sym::type_id => (1, 0, vec![], type_id_ty()),
         sym::type_id_array_len => (0, 0, vec![type_id_ty()], tcx.types.usize),

compiler/rustc_interface/src/passes.rs

@@ -657,6 +657,15 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
         );
         files.extend(extra_tracked_files);
 
+        let comptime_tracked_files = hash_iter_files(
+            tcx.experimental_comptime_list_files(LOCAL_CRATE)
+                .iter()
+                .map(|s| PathBuf::from(s.as_str()))
+                .map(normalize_path),
+            checksum_hash_algo,
+        );
+        files.extend(comptime_tracked_files);
+
         // We also need to track used PGO profile files
         if let Some(ref profile_instr) = sess.opts.cg.profile_use {
             files.extend(hash_iter_files(

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -26,6 +26,7 @@ use rustc_span::{Span, Symbol, bug, kw};
 use super::{Decodable, DecodeIterator};
 use crate::creader::{CStore, LoadedMacro};
 use crate::rmeta::AttrFlags;
+use crate::rmeta::decoder::MetaBlob;
 use crate::rmeta::table::IsDefault;
 use crate::{eii, foreign_modules, native_libs};
 
@@ -624,6 +625,28 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
         ..providers.queries
     };
     provide_extern(&mut providers.extern_queries);
+    provide_extern_comptime(&mut providers.extern_queries);
+}
+
+fn provide_extern_comptime(extern_queries: &mut ExternProviders) {
+    extern_queries.experimental_comptime_list_files = |tcx, cnum| {
+        let cstore = CStore::from_tcx(tcx);
+        let crate_data = cstore.get_crate_data(cnum);
+        let filelist = crate_data.root.comptime_files.keys().copied().collect::<Vec<_>>();
+        tcx.arena.alloc_slice(&filelist)
+    };
+
+    extern_queries.experimental_comptime_fetch_file = |tcx, (cnum, path)| {
+        let cstore = CStore::from_tcx(tcx);
+        let crate_data = cstore.get_crate_data(cnum);
+        let data = crate_data
+            .root
+            .comptime_files
+            .get(&path)
+            .map(|lv| lv.decode(crate_data.blob()))
+            .and_then(|v| v.map(|d| &*tcx.arena.alloc_slice(&d)));
+        Ok(data)
+    };
 }
 
 impl CStore {

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -771,6 +771,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let denied_partial_mitigations = stat!("denied-partial-mitigations", || self
             .encode_enabled_denied_partial_mitigations());
 
+        let comptime_files = stat!("comptime-files", || self.encode_comptime_files(tcx));
+
         let root = stat!("crate-root", || {
             let attrs = tcx.hir_krate_attrs();
             self.lazy(CrateRoot {
@@ -829,6 +831,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 expn_hashes,
                 def_path_hash_map,
                 specialization_enabled_in: tcx.specialization_enabled_in(LOCAL_CRATE),
+                comptime_files,
             })
         });
 
@@ -2465,6 +2468,24 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
         LazyArray::default()
     }
+
+    fn encode_comptime_files(
+        &mut self,
+        tcx: TyCtxt<'_>,
+    ) -> IndexMap<Symbol, LazyValue<Option<Vec<u8>>>> {
+        let mut comptime_files = IndexMap::default();
+        for &path in tcx.experimental_comptime_list_files(LOCAL_CRATE) {
+            // Must encode None too, as we depend on the keys of the IndexMap to store the list
+            // of files that we export from this crate.
+            if let Ok(file_data) = tcx.experimental_comptime_fetch_file((LOCAL_CRATE, path)) {
+                comptime_files.insert_full(path, self.lazy(file_data.map(|d| d.to_vec())));
+            } else {
+                bug!("a comptile time file read error should fail crate compilation")
+            }
+        }
+
+        comptime_files
+    }
 }
 
 /// Used to prefetch queries which will be needed later by metadata encoding.

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -41,6 +41,7 @@ use rustc_middle::middle::resolve::{AmbigModChild, DocLinkResMap, ModChild};
 use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir;
 use rustc_middle::mir::ConstValue;
+use rustc_middle::ty::data_structures::IndexMap;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util::Providers;
@@ -331,6 +332,8 @@ pub(crate) struct CrateRoot {
     symbol_mangling_version: SymbolManglingVersion,
 
     specialization_enabled_in: bool,
+
+    comptime_files: IndexMap<Symbol, LazyValue<Option<Vec<u8>>>>,
 }
 
 /// A separate struct for extra metadata that must be encoded *after*
@@ -653,6 +656,7 @@ define_tables! {
     defaultness: Table<hir::Defaultness>,
     impl_is_fully_generic_for_reflection: Table<bool>,
 
+
 - optional:
     opt_rpitit_info: Table<LazyValue<ty::ImplTraitInTraitData>>,
     attributes: Table<LazyArray<Attribute>>,

compiler/rustc_metadata/src/rmeta/parameterized.rs

@@ -70,6 +70,7 @@ macro_rules! trivially_parameterized_over_tcx {
 
 trivially_parameterized_over_tcx! {
     bool,
+    u8,
     u32,
     u64,
     usize,