Rollup merge of #69646 - RalfJung:layout-visitor, r=eddyb

Miri visitor: detect primitive types based on type, not layout (also, more tests)

I also converted the union-based transmutes to use `mem::transmute` for increased readability.

r? @eddyb @oli-obk

src/librustc_target/abi/mod.rs

@@ -872,7 +872,7 @@ impl Niche {
 #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub struct LayoutDetails {
     /// Says where the fields are located within the layout.
-    /// Primitives and fieldless enums appear as unions without fields.
+    /// Primitives and uninhabited enums appear as unions without fields.
     pub fields: FieldPlacement,
 
     /// Encodes information about multi-variant layouts.

src/test/ui/consts/const-eval/ub-enum.rs

@@ -1,74 +1,95 @@
+#![feature(const_transmute, never_type)]
 #![allow(const_err)] // make sure we cannot allow away the errors tested here
 
+use std::mem;
 
 #[repr(transparent)]
 #[derive(Copy, Clone)]
 struct Wrap<T>(T);
 
+#[derive(Copy, Clone)]
+enum Never {}
+
+// # simple enum with discriminant 0
+
 #[repr(usize)]
 #[derive(Copy, Clone)]
 enum Enum {
     A = 0,
 }
-#[repr(C)]
-union TransmuteEnum {
-    in1: &'static u8,
-    in2: usize,
-    out1: Enum,
-    out2: Wrap<Enum>,
-}
 
-const GOOD_ENUM: Enum = unsafe { TransmuteEnum { in2: 0 }.out1 };
+const GOOD_ENUM: Enum = unsafe { mem::transmute(0usize) };
 
-const BAD_ENUM: Enum = unsafe { TransmuteEnum { in2: 1 }.out1 };
+const BAD_ENUM: Enum = unsafe { mem::transmute(1usize) };
 //~^ ERROR is undefined behavior
 
-const BAD_ENUM_PTR: Enum = unsafe { TransmuteEnum { in1: &1 }.out1 };
+const BAD_ENUM_PTR: Enum = unsafe { mem::transmute(&1) };
 //~^ ERROR is undefined behavior
 
-const BAD_ENUM_WRAPPED: Wrap<Enum> = unsafe { TransmuteEnum { in1: &1 }.out2 };
+const BAD_ENUM_WRAPPED: Wrap<Enum> = unsafe { mem::transmute(&1) };
 //~^ ERROR is undefined behavior
 
+// # simple enum with discriminant 2
+
 // (Potentially) invalid enum discriminant
 #[repr(usize)]
 #[derive(Copy, Clone)]
 enum Enum2 {
     A = 2,
 }
 
-#[repr(C)]
-union TransmuteEnum2 {
-    in1: usize,
-    in2: &'static u8,
-    in3: (),
-    out1: Enum2,
-    out2: Wrap<Enum2>, // something wrapping the enum so that we test layout first, not enum
-    out3: Option<Enum2>,
-}
-const BAD_ENUM2: Enum2 = unsafe { TransmuteEnum2 { in1: 0 }.out1 };
+const BAD_ENUM2: Enum2 = unsafe { mem::transmute(0usize) };
 //~^ ERROR is undefined behavior
-const BAD_ENUM2_PTR: Enum2 = unsafe { TransmuteEnum2 { in2: &0 }.out1 };
+const BAD_ENUM2_PTR: Enum2 = unsafe { mem::transmute(&0) };
 //~^ ERROR is undefined behavior
-const BAD_ENUM2_WRAPPED: Wrap<Enum2> = unsafe { TransmuteEnum2 { in2: &0 }.out2 };
+// something wrapping the enum so that we test layout first, not enum
+const BAD_ENUM2_WRAPPED: Wrap<Enum2> = unsafe { mem::transmute(&0) };
 //~^ ERROR is undefined behavior
 
 // Undef enum discriminant.
-const BAD_ENUM2_UNDEF : Enum2 = unsafe { TransmuteEnum2 { in3: () }.out1 };
+#[repr(C)]
+union MaybeUninit<T: Copy> {
+    uninit: (),
+    init: T,
+}
+const BAD_ENUM2_UNDEF : Enum2 = unsafe { MaybeUninit { uninit: () }.init };
 //~^ ERROR is undefined behavior
 
 // Pointer value in an enum with a niche that is not just 0.
-const BAD_ENUM2_OPTION_PTR: Option<Enum2> = unsafe { TransmuteEnum2 { in2: &0 }.out3 };
+const BAD_ENUM2_OPTION_PTR: Option<Enum2> = unsafe { mem::transmute(&0) };
 //~^ ERROR is undefined behavior
 
+// # valid discriminant for uninhabited variant
+
+// An enum with 3 variants of which some are uninhabited -- so the uninhabited variants *do*
+// have a discriminant.
+enum UninhDiscriminant {
+    A,
+    B(!),
+    C,
+    D(Never),
+}
+
+const GOOD_INHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(0u8) }; // variant A
+const GOOD_INHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(2u8) }; // variant C
+
+const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) };
+//~^ ERROR is undefined behavior
+const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) };
+//~^ ERROR is undefined behavior
+
+// # other
+
 // Invalid enum field content (mostly to test printing of paths for enum tuple
 // variants and tuples).
-#[repr(C)]
-union TransmuteChar {
-    a: u32,
-    b: char,
-}
 // Need to create something which does not clash with enum layout optimizations.
-const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { TransmuteChar { a: !0 }.b }));
+const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::transmute(!0u32) }));
+//~^ ERROR is undefined behavior
+
+// All variants are uninhabited but also have data.
+const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(1u64) };
+//~^ ERROR is undefined behavior
+const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(1u64) };
 //~^ ERROR is undefined behavior
 
 fn main() {

src/test/ui/consts/const-eval/ub-enum.stderr

@@ -1,75 +1,107 @@
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-enum.rs:23:1
    |
-LL | const BAD_ENUM: Enum = unsafe { TransmuteEnum { in2: 1 }.out1 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 1, but expected a valid enum discriminant
+LL | const BAD_ENUM: Enum = unsafe { mem::transmute(1usize) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 1, but expected a valid enum discriminant
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-enum.rs:26:1
    |
-LL | const BAD_ENUM_PTR: Enum = unsafe { TransmuteEnum { in1: &1 }.out1 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
+LL | const BAD_ENUM_PTR: Enum = unsafe { mem::transmute(&1) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-enum.rs:29:1
    |
-LL | const BAD_ENUM_WRAPPED: Wrap<Enum> = unsafe { TransmuteEnum { in1: &1 }.out2 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .0.<enum-tag>, but expected initialized plain (non-pointer) bytes
+LL | const BAD_ENUM_WRAPPED: Wrap<Enum> = unsafe { mem::transmute(&1) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .0.<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:48:1
+  --> $DIR/ub-enum.rs:41:1
    |
-LL | const BAD_ENUM2: Enum2 = unsafe { TransmuteEnum2 { in1: 0 }.out1 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 0, but expected a valid enum discriminant
+LL | const BAD_ENUM2: Enum2 = unsafe { mem::transmute(0usize) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 0, but expected a valid enum discriminant
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:50:1
+  --> $DIR/ub-enum.rs:43:1
    |
-LL | const BAD_ENUM2_PTR: Enum2 = unsafe { TransmuteEnum2 { in2: &0 }.out1 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
+LL | const BAD_ENUM2_PTR: Enum2 = unsafe { mem::transmute(&0) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:52:1
+  --> $DIR/ub-enum.rs:46:1
    |
-LL | const BAD_ENUM2_WRAPPED: Wrap<Enum2> = unsafe { TransmuteEnum2 { in2: &0 }.out2 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .0.<enum-tag>, but expected initialized plain (non-pointer) bytes
+LL | const BAD_ENUM2_WRAPPED: Wrap<Enum2> = unsafe { mem::transmute(&0) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .0.<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:56:1
+  --> $DIR/ub-enum.rs:55:1
    |
-LL | const BAD_ENUM2_UNDEF : Enum2 = unsafe { TransmuteEnum2 { in3: () }.out1 };
+LL | const BAD_ENUM2_UNDEF : Enum2 = unsafe { MaybeUninit { uninit: () }.init };
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes at .<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:60:1
+  --> $DIR/ub-enum.rs:59:1
    |
-LL | const BAD_ENUM2_OPTION_PTR: Option<Enum2> = unsafe { TransmuteEnum2 { in2: &0 }.out3 };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
+LL | const BAD_ENUM2_OPTION_PTR: Option<Enum2> = unsafe { mem::transmute(&0) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer at .<enum-tag>, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
 error[E0080]: it is undefined behavior to use this value
-  --> $DIR/ub-enum.rs:71:1
+  --> $DIR/ub-enum.rs:76:1
    |
-LL | const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { TransmuteChar { a: !0 }.b }));
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 4294967295 at .<enum-variant(Some)>.0.1, but expected a valid unicode codepoint
+LL | const BAD_UNINHABITED_VARIANT1: UninhDiscriminant = unsafe { mem::transmute(1u8) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of the never type `!` at .<enum-variant(B)>.0
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 
-error: aborting due to 9 previous errors
+error[E0080]: it is undefined behavior to use this value
+  --> $DIR/ub-enum.rs:78:1
+   |
+LL | const BAD_UNINHABITED_VARIANT2: UninhDiscriminant = unsafe { mem::transmute(3u8) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of uninhabited type Never at .<enum-variant(D)>.0
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
+error[E0080]: it is undefined behavior to use this value
+  --> $DIR/ub-enum.rs:86:1
+   |
+LL | const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::transmute(!0u32) }));
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered 4294967295 at .<enum-variant(Some)>.0.1, but expected a valid unicode codepoint
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
+error[E0080]: it is undefined behavior to use this value
+  --> $DIR/ub-enum.rs:90:1
+   |
+LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(1u64) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of the never type `!` at .<enum-variant(Err)>.0.1
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
+error[E0080]: it is undefined behavior to use this value
+  --> $DIR/ub-enum.rs:92:1
+   |
+LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(1u64) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of uninhabited type Never at .<enum-variant(Err)>.0.1
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
+error: aborting due to 13 previous errors
 
 For more information about this error, try `rustc --explain E0080`.

src/test/ui/consts/const-eval/ub-nonnull.rs

@@ -25,11 +25,11 @@ const NULL_USIZE: NonZeroUsize = unsafe { mem::transmute(0usize) };
 //~^ ERROR it is undefined behavior to use this value
 
 #[repr(C)]
-union Transmute {
+union MaybeUninit<T: Copy> {
     uninit: (),
-    out: NonZeroU8,
+    init: T,
 }
-const UNINIT: NonZeroU8 = unsafe { Transmute { uninit: () }.out };
+const UNINIT: NonZeroU8 = unsafe { MaybeUninit { uninit: () }.init };
 //~^ ERROR it is undefined behavior to use this value
 
 // Also test other uses of rustc_layout_scalar_valid_range_start

src/test/ui/consts/const-eval/ub-nonnull.stderr

@@ -43,8 +43,8 @@ LL | const NULL_USIZE: NonZeroUsize = unsafe { mem::transmute(0usize) };
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-nonnull.rs:32:1
    |
-LL | const UNINIT: NonZeroU8 = unsafe { Transmute { uninit: () }.out };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes at .0, but expected initialized plain (non-pointer) bytes
+LL | const UNINIT: NonZeroU8 = unsafe { MaybeUninit { uninit: () }.init };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes at .0, but expected initialized plain (non-pointer) bytes
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
 

src/test/ui/consts/const-eval/ub-ref.rs

@@ -8,9 +8,16 @@ const UNALIGNED: &u16 = unsafe { mem::transmute(&[0u8; 4]) };
 //~^ ERROR it is undefined behavior to use this value
 //~^^ type validation failed: encountered an unaligned reference (required 2 byte alignment but found 1)
 
+const UNALIGNED_BOX: Box<u16> = unsafe { mem::transmute(&[0u8; 4]) };
+//~^ ERROR it is undefined behavior to use this value
+//~^^ type validation failed: encountered an unaligned box (required 2 byte alignment but found 1)
+
 const NULL: &u16 = unsafe { mem::transmute(0usize) };
 //~^ ERROR it is undefined behavior to use this value
 
+const NULL_BOX: Box<u16> = unsafe { mem::transmute(0usize) };
+//~^ ERROR it is undefined behavior to use this value
+
 // It is very important that we reject this: We do promote `&(4 * REF_AS_USIZE)`,
 // but that would fail to compile; so we ended up breaking user code that would
 // have worked fine had we not promoted.
@@ -20,7 +27,13 @@ const REF_AS_USIZE: usize = unsafe { mem::transmute(&0) };
 const REF_AS_USIZE_SLICE: &[usize] = &[unsafe { mem::transmute(&0) }];
 //~^ ERROR it is undefined behavior to use this value
 
+const REF_AS_USIZE_BOX_SLICE: Box<[usize]> = unsafe { mem::transmute::<&[usize], _>(&[mem::transmute(&0)]) };
+//~^ ERROR it is undefined behavior to use this value
+
 const USIZE_AS_REF: &'static u8 = unsafe { mem::transmute(1337usize) };
 //~^ ERROR it is undefined behavior to use this value
 
+const USIZE_AS_BOX: Box<u8> = unsafe { mem::transmute(1337usize) };
+//~^ ERROR it is undefined behavior to use this value
+
 fn main() {}