Ensure nested allocations in statics neither get deduplicated nor duplicated

compiler/rustc_ast_lowering/src/asm.rs

@@ -196,7 +196,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             .get_partial_res(sym.id)
                             .and_then(|res| res.full_res())
                             .and_then(|res| match res {
-                                Res::Def(DefKind::Static(_), def_id) => Some(def_id),
+                                Res::Def(DefKind::Static { .. }, def_id) => Some(def_id),
                                 _ => None,
                             });
 

compiler/rustc_codegen_gcc/src/consts.rs

@@ -63,7 +63,7 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
         global_value
     }
 
-    fn codegen_static(&self, def_id: DefId, is_mutable: bool) {
+    fn codegen_static(&self, def_id: DefId) {
         let attrs = self.tcx.codegen_fn_attrs(def_id);
 
         let value = match codegen_static_initializer(&self, def_id) {
@@ -92,7 +92,7 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
 
         // As an optimization, all shared statics which do not have interior
         // mutability are placed into read-only memory.
-        if !is_mutable && self.type_is_freeze(ty) {
+        if !self.tcx.static_mutability(def_id).unwrap().is_mut() && self.type_is_freeze(ty) {
             #[cfg(feature = "master")]
             global.global_set_readonly();
         }
@@ -349,7 +349,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(
     cx.const_struct(&llvals, true)
 }
 
-pub fn codegen_static_initializer<'gcc, 'tcx>(
+fn codegen_static_initializer<'gcc, 'tcx>(
     cx: &CodegenCx<'gcc, 'tcx>,
     def_id: DefId,
 ) -> Result<(RValue<'gcc>, ConstAllocation<'tcx>), ErrorHandled> {

compiler/rustc_codegen_gcc/src/mono_item.rs

@@ -1,7 +1,9 @@
 #[cfg(feature = "master")]
 use gccjit::{FnAttribute, VarAttribute};
 use rustc_codegen_ssa::traits::PreDefineMethods;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
+use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
@@ -23,7 +25,14 @@ impl<'gcc, 'tcx> PreDefineMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     ) {
         let attrs = self.tcx.codegen_fn_attrs(def_id);
         let instance = Instance::mono(self.tcx, def_id);
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+        let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
+        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
+        // the gcc type from the actual evaluated initializer.
+        let ty = if nested {
+            self.tcx.types.unit
+        } else {
+            instance.ty(self.tcx, ty::ParamEnv::reveal_all())
+        };
         let gcc_type = self.layout_of(ty).gcc_type(self);
 
         let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);

compiler/rustc_codegen_llvm/src/consts.rs

@@ -9,6 +9,7 @@ use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 use rustc_codegen_ssa::traits::*;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
@@ -17,7 +18,7 @@ use rustc_middle::mir::interpret::{
 };
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::layout::LayoutOf;
-use rustc_middle::ty::{self, Instance, Ty};
+use rustc_middle::ty::{self, Instance};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::Lto;
 use rustc_target::abi::{
@@ -114,7 +115,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
     cx.const_struct(&llvals, true)
 }
 
-pub fn codegen_static_initializer<'ll, 'tcx>(
+fn codegen_static_initializer<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
 ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
@@ -147,11 +148,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align:
 fn check_and_apply_linkage<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     attrs: &CodegenFnAttrs,
-    ty: Ty<'tcx>,
+    llty: &'ll Type,
     sym: &str,
     def_id: DefId,
 ) -> &'ll Value {
-    let llty = cx.layout_of(ty).llvm_type(cx);
     if let Some(linkage) = attrs.import_linkage {
         debug!("get_static: sym={} linkage={:?}", sym, linkage);
 
@@ -226,9 +226,28 @@ impl<'ll> CodegenCx<'ll, '_> {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value {
         let instance = Instance::mono(self.tcx, def_id);
-        if let Some(&g) = self.instances.borrow().get(&instance) {
+        trace!(?instance);
+
+        let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
+        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
+        // the llvm type from the actual evaluated initializer.
+        let llty = if nested {
+            self.type_i8()
+        } else {
+            let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+            trace!(?ty);
+            self.layout_of(ty).llvm_type(self)
+        };
+        self.get_static_inner(def_id, llty)
+    }
+
+    #[instrument(level = "debug", skip(self, llty))]
+    pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
+        if let Some(&g) = self.instances.borrow().get(&Instance::mono(self.tcx, def_id)) {
+            trace!("used cached value");
             return g;
         }
 
@@ -240,14 +259,12 @@ impl<'ll> CodegenCx<'ll, '_> {
                  statics defined in the same CGU, but did not for `{def_id:?}`"
         );
 
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
-        let sym = self.tcx.symbol_name(instance).name;
+        let sym = self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name;
         let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
 
-        debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs);
+        debug!(?sym, ?fn_attrs);
 
         let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) {
-            let llty = self.layout_of(ty).llvm_type(self);
             if let Some(g) = self.get_declared_value(sym) {
                 if self.val_ty(g) != self.type_ptr() {
                     span_bug!(self.tcx.def_span(def_id), "Conflicting types for static");
@@ -264,7 +281,7 @@ impl<'ll> CodegenCx<'ll, '_> {
 
             g
         } else {
-            check_and_apply_linkage(self, fn_attrs, ty, sym, def_id)
+            check_and_apply_linkage(self, fn_attrs, llty, sym, def_id)
         };
 
         // Thread-local statics in some other crate need to *always* be linked
@@ -332,34 +349,18 @@ impl<'ll> CodegenCx<'ll, '_> {
             }
         }
 
-        self.instances.borrow_mut().insert(instance, g);
+        self.instances.borrow_mut().insert(Instance::mono(self.tcx, def_id), g);
         g
     }
-}
-
-impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
-    fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
-        if let Some(&gv) = self.const_globals.borrow().get(&cv) {
-            unsafe {
-                // Upgrade the alignment in cases where the same constant is used with different
-                // alignment requirements
-                let llalign = align.bytes() as u32;
-                if llalign > llvm::LLVMGetAlignment(gv) {
-                    llvm::LLVMSetAlignment(gv, llalign);
-                }
-            }
-            return gv;
-        }
-        let gv = self.static_addr_of_mut(cv, align, kind);
-        unsafe {
-            llvm::LLVMSetGlobalConstant(gv, True);
-        }
-        self.const_globals.borrow_mut().insert(cv, gv);
-        gv
-    }
 
-    fn codegen_static(&self, def_id: DefId, is_mutable: bool) {
+    fn codegen_static_item(&self, def_id: DefId) {
         unsafe {
+            assert!(
+                llvm::LLVMGetInitializer(
+                    self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap()
+                )
+                .is_none()
+            );
             let attrs = self.tcx.codegen_fn_attrs(def_id);
 
             let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else {
@@ -368,13 +369,11 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
             };
             let alloc = alloc.inner();
 
-            let g = self.get_static(def_id);
-
             let val_llty = self.val_ty(v);
 
-            let instance = Instance::mono(self.tcx, def_id);
-            let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
-            let llty = self.layout_of(ty).llvm_type(self);
+            let g = self.get_static_inner(def_id, val_llty);
+            let llty = self.val_ty(g);
+
             let g = if val_llty == llty {
                 g
             } else {
@@ -409,16 +408,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                 self.statics_to_rauw.borrow_mut().push((g, new_g));
                 new_g
             };
-            set_global_alignment(self, g, self.align_of(ty));
+            set_global_alignment(self, g, alloc.align);
             llvm::LLVMSetInitializer(g, v);
 
             if self.should_assume_dso_local(g, true) {
                 llvm::LLVMRustSetDSOLocal(g, true);
             }
 
-            // As an optimization, all shared statics which do not have interior
-            // mutability are placed into read-only memory.
-            if !is_mutable && self.type_is_freeze(ty) {
+            // Forward the allocation's mutability (picked by the const interner) to LLVM.
+            if alloc.mutability.is_not() {
                 llvm::LLVMSetGlobalConstant(g, llvm::True);
             }
 
@@ -541,6 +539,32 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
             }
         }
     }
+}
+
+impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
+    fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
+        if let Some(&gv) = self.const_globals.borrow().get(&cv) {
+            unsafe {
+                // Upgrade the alignment in cases where the same constant is used with different
+                // alignment requirements
+                let llalign = align.bytes() as u32;
+                if llalign > llvm::LLVMGetAlignment(gv) {
+                    llvm::LLVMSetAlignment(gv, llalign);
+                }
+            }
+            return gv;
+        }
+        let gv = self.static_addr_of_mut(cv, align, kind);
+        unsafe {
+            llvm::LLVMSetGlobalConstant(gv, True);
+        }
+        self.const_globals.borrow_mut().insert(cv, gv);
+        gv
+    }
+
+    fn codegen_static(&self, def_id: DefId) {
+        self.codegen_static_item(def_id)
+    }
 
     /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
     fn add_used_global(&self, global: &'ll Value) {

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -26,6 +26,7 @@ use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind;
 use rustc_codegen_ssa::traits::*;
 use rustc_fs_util::path_to_c_string;
 use rustc_hir::def::CtorKind;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
@@ -1309,6 +1310,11 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo
     };
 
     let is_local_to_unit = is_node_local_to_unit(cx, def_id);
+
+    let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { bug!() };
+    if nested {
+        return;
+    }
     let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all());
     let type_di_node = type_di_node(cx, variable_type);
     let var_name = tcx.item_name(def_id);

compiler/rustc_codegen_llvm/src/mono_item.rs

@@ -5,7 +5,9 @@ use crate::errors::SymbolAlreadyDefined;
 use crate::llvm;
 use crate::type_of::LayoutLlvmExt;
 use rustc_codegen_ssa::traits::*;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
+use rustc_middle::bug;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
 use rustc_middle::ty::{self, Instance, TypeVisitableExt};
@@ -21,7 +23,14 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
         symbol_name: &str,
     ) {
         let instance = Instance::mono(self.tcx, def_id);
-        let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
+        let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
+        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
+        // the llvm type from the actual evaluated initializer.
+        let ty = if nested {
+            self.tcx.types.unit
+        } else {
+            instance.ty(self.tcx, ty::ParamEnv::reveal_all())
+        };
         let llty = self.layout_of(ty).llvm_type(self);
 
         let g = self.define_global(symbol_name, llty).unwrap_or_else(|| {

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -87,7 +87,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<S
 
             // Only consider nodes that actually have exported symbols.
             match tcx.def_kind(def_id) {
-                DefKind::Fn | DefKind::Static(_) => {}
+                DefKind::Fn | DefKind::Static { .. } => {}
                 DefKind::AssocFn if tcx.impl_of_method(def_id.to_def_id()).is_some() => {}
                 _ => return None,
             };
@@ -483,7 +483,7 @@ fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel
         let target = &tcx.sess.target.llvm_target;
         // WebAssembly cannot export data symbols, so reduce their export level
         if target.contains("emscripten") {
-            if let DefKind::Static(_) = tcx.def_kind(sym_def_id) {
+            if let DefKind::Static { .. } = tcx.def_kind(sym_def_id) {
                 return SymbolExportLevel::Rust;
             }
         }

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -30,7 +30,7 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
 
         match *self {
             MonoItem::Static(def_id) => {
-                cx.codegen_static(def_id, cx.tcx().is_mutable_static(def_id));
+                cx.codegen_static(def_id);
             }
             MonoItem::GlobalAsm(item_id) => {
                 let item = cx.tcx().hir().item(item_id);

compiler/rustc_codegen_ssa/src/traits/statics.rs

@@ -4,7 +4,7 @@ use rustc_target::abi::Align;
 
 pub trait StaticMethods: BackendTypes {
     fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value;
-    fn codegen_static(&self, def_id: DefId, is_mutable: bool);
+    fn codegen_static(&self, def_id: DefId);
 
     /// Mark the given global value as "used", to prevent the compiler and linker from potentially
     /// removing a static variable that may otherwise appear unused.

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -37,7 +37,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
             || matches!(
                 ecx.tcx.def_kind(cid.instance.def_id()),
                 DefKind::Const
-                    | DefKind::Static(_)
+                    | DefKind::Static { .. }
                     | DefKind::ConstParam
                     | DefKind::AnonConst
                     | DefKind::InlineConst
@@ -59,7 +59,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
     };
 
     let ret = if let InternKind::Static(_) = intern_kind {
-        create_static_alloc(ecx, cid.instance.def_id(), layout)?
+        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
     } else {
         ecx.allocate(layout, MemoryKind::Stack)?
     };
@@ -380,7 +380,11 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
         }
         Ok(mplace) => {
             // Since evaluation had no errors, validate the resulting constant.
+
+            // Temporarily allow access to the static_root_ids for the purpose of validation.
+            let static_root_ids = ecx.machine.static_root_ids.take();
             let res = const_validate_mplace(&ecx, &mplace, cid);
+            ecx.machine.static_root_ids = static_root_ids;
 
             let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();