From 11c3b93d039eb7efff39c8e3fafd50ccaba5198d Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 8 Jul 2025 10:20:54 +0200 Subject: [PATCH 01/14] add extra log messages to track setup --- crates/intrinsic-test/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/intrinsic-test/src/main.rs b/crates/intrinsic-test/src/main.rs index 054138a0db..538f317a29 100644 --- a/crates/intrinsic-test/src/main.rs +++ b/crates/intrinsic-test/src/main.rs @@ -30,12 +30,15 @@ fn main() { let test_environment = test_environment_result.unwrap(); + info!("building C binaries"); if !test_environment.build_c_file() { std::process::exit(2); } + info!("building Rust binaries"); if !test_environment.build_rust_file() { std::process::exit(3); } + info!("comaparing outputs"); if !test_environment.compare_outputs() { std::process::exit(1); } From 3babc4b8fbb4f582339fcdfb3ec6b1effc4a8f6f Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 8 Jul 2025 09:59:03 +0200 Subject: [PATCH 02/14] run rust programs with the runner --- Cargo.lock | 42 ++++----- crates/intrinsic-test/src/arm/config.rs | 1 - crates/intrinsic-test/src/arm/mod.rs | 5 +- crates/intrinsic-test/src/common/cli.rs | 4 +- crates/intrinsic-test/src/common/compare.rs | 32 +++---- crates/intrinsic-test/src/common/gen_rust.rs | 99 ++++++++++---------- 6 files changed, 89 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80f424dfdd..7d9cd1bf8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,20 +73,20 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.102", + "syn 2.0.104", ] [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "cc" -version = "1.2.26" +version = "1.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" +checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" dependencies = [ "shlex", ] @@ -128,7 +128,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.102", + "syn 2.0.104", ] [[package]] @@ -338,9 +338,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown 0.15.4", @@ -403,9 +403,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "linked-hash-map" @@ -624,7 +624,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.102", + "syn 2.0.104", ] [[package]] @@ -685,7 +685,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.102", + "syn 2.0.104", ] [[package]] @@ -742,7 +742,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.102", + "syn 2.0.104", ] [[package]] @@ -780,9 +780,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.102" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -838,7 +838,7 @@ version = "0.113.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "286049849b5a5bd09a8773171be96824afabffc7cc3df6caaf33a38db6cd07ae" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "semver", ] @@ -945,20 +945,20 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.102", + "syn 2.0.104", ] diff --git a/crates/intrinsic-test/src/arm/config.rs b/crates/intrinsic-test/src/arm/config.rs index cee80374ae..9a7b37253d 100644 --- a/crates/intrinsic-test/src/arm/config.rs +++ b/crates/intrinsic-test/src/arm/config.rs @@ -114,7 +114,6 @@ pub const AARCH_CONFIGURATIONS: &str = r#" #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_fcma))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_dotprod))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_i8mm))] -#![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_sha3))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_sm4))] #![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_ftts))] #![feature(fmt_helpers_for_derive)] diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 6aaa49ff97..1a66ed4268 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -104,7 +104,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { } fn compare_outputs(&self) -> bool { - if let Some(ref toolchain) = self.cli_options.toolchain { + if self.cli_options.toolchain.is_some() { let intrinsics_name_list = self .intrinsics .iter() @@ -113,8 +113,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { compare_outputs( &intrinsics_name_list, - toolchain, - &self.cli_options.c_runner, + &self.cli_options.runner, &self.cli_options.target, ) } else { diff --git a/crates/intrinsic-test/src/common/cli.rs b/crates/intrinsic-test/src/common/cli.rs index 1d57272300..a5277c2677 100644 --- a/crates/intrinsic-test/src/common/cli.rs +++ b/crates/intrinsic-test/src/common/cli.rs @@ -60,7 +60,7 @@ pub struct ProcessedCli { pub filename: PathBuf, pub toolchain: Option, pub cpp_compiler: Option, - pub c_runner: String, + pub runner: String, pub target: String, pub linker: Option, pub cxx_toolchain_dir: Option, @@ -102,7 +102,7 @@ impl ProcessedCli { Self { toolchain, cpp_compiler, - c_runner, + runner: c_runner, target, linker, cxx_toolchain_dir, diff --git a/crates/intrinsic-test/src/common/compare.rs b/crates/intrinsic-test/src/common/compare.rs index 9e0cbe8cd6..f16d83b026 100644 --- a/crates/intrinsic-test/src/common/compare.rs +++ b/crates/intrinsic-test/src/common/compare.rs @@ -2,26 +2,24 @@ use super::cli::FailureReason; use rayon::prelude::*; use std::process::Command; -pub fn compare_outputs( - intrinsic_name_list: &Vec, - toolchain: &str, - runner: &str, - target: &str, -) -> bool { +pub fn compare_outputs(intrinsic_name_list: &Vec, runner: &str, target: &str) -> bool { + fn runner_command(runner: &str) -> Command { + let mut it = runner.split_whitespace(); + let mut cmd = Command::new(it.next().unwrap()); + cmd.args(it); + + cmd + } + let intrinsics = intrinsic_name_list .par_iter() .filter_map(|intrinsic_name| { - let c = Command::new("sh") - .arg("-c") - .arg(format!("{runner} ./c_programs/{intrinsic_name}")) + let c = runner_command(runner) + .arg(format!("./c_programs/{intrinsic_name}")) .output(); - let rust = Command::new("sh") - .current_dir("rust_programs") - .arg("-c") - .arg(format!( - "cargo {toolchain} run --target {target} --bin {intrinsic_name} --release", - )) + let rust = runner_command(runner) + .arg(format!("target/{target}/release/{intrinsic_name}")) .env("RUSTFLAGS", "-Cdebuginfo=0") .output(); @@ -42,8 +40,8 @@ pub fn compare_outputs( if !rust.status.success() { error!( "Failed to run Rust program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}", - stdout = std::str::from_utf8(&rust.stdout).unwrap_or(""), - stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""), + stdout = String::from_utf8_lossy(&rust.stdout), + stderr = String::from_utf8_lossy(&rust.stderr), ); return Some(FailureReason::RunRust(intrinsic_name.clone())); } diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index 52bccaf905..0e4a95ab52 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -2,7 +2,6 @@ use itertools::Itertools; use rayon::prelude::*; use std::collections::BTreeMap; use std::fs::File; -use std::io::Write; use std::process::Command; use super::argument::Argument; @@ -23,8 +22,8 @@ pub fn format_rust_main_template( ) -> String { format!( r#"{notices}#![feature(simd_ffi)] -#![feature(link_llvm_intrinsics)] #![feature(f16)] +#![allow(unused)] {configurations} {definitions} @@ -38,6 +37,42 @@ fn main() {{ ) } +fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> { + writeln!( + w, + concat!( + "[package]\n", + "name = \"intrinsic-test-programs\"\n", + "version = \"{version}\"\n", + "authors = [{authors}]\n", + "license = \"{license}\"\n", + "edition = \"2018\"\n", + "[workspace]\n", + "[dependencies]\n", + "core_arch = {{ path = \"../crates/core_arch\" }}", + ), + version = env!("CARGO_PKG_VERSION"), + authors = env!("CARGO_PKG_AUTHORS") + .split(":") + .format_with(", ", |author, fmt| fmt(&format_args!("\"{author}\""))), + license = env!("CARGO_PKG_LICENSE"), + )?; + + for binary in binaries { + writeln!( + w, + concat!( + "[[bin]]\n", + "name = \"{binary}\"\n", + "path = \"{binary}/main.rs\"\n", + ), + binary = binary, + )?; + } + + Ok(()) +} + pub fn compile_rust_programs( binaries: Vec, toolchain: Option<&str>, @@ -45,56 +80,20 @@ pub fn compile_rust_programs( linker: Option<&str>, ) -> bool { let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); - cargo - .write_all( - format!( - r#"[package] -name = "intrinsic-test-programs" -version = "{version}" -authors = [{authors}] -license = "{license}" -edition = "2018" -[workspace] -[dependencies] -core_arch = {{ path = "../crates/core_arch" }} -{binaries}"#, - version = env!("CARGO_PKG_VERSION"), - authors = env!("CARGO_PKG_AUTHORS") - .split(":") - .format_with(", ", |author, fmt| fmt(&format_args!("\"{author}\""))), - license = env!("CARGO_PKG_LICENSE"), - binaries = binaries - .iter() - .map(|binary| { - format!( - r#"[[bin]] -name = "{binary}" -path = "{binary}/main.rs""#, - ) - }) - .collect::>() - .join("\n") - ) - .into_bytes() - .as_slice(), - ) - .unwrap(); - - let toolchain = match toolchain { - None => return true, - Some(t) => t, - }; + write_cargo_toml(&mut cargo, &binaries).unwrap(); /* If there has been a linker explicitly set from the command line then * we want to set it via setting it in the RUSTFLAGS*/ - let cargo_command = format!("cargo {toolchain} build --target {target} --release"); + let mut cargo_command = Command::new("cargo"); + cargo_command.current_dir("rust_programs"); - let mut command = Command::new("sh"); - command - .current_dir("rust_programs") - .arg("-c") - .arg(cargo_command); + if let Some(toolchain) = toolchain { + if !toolchain.is_empty() { + cargo_command.arg(toolchain); + } + } + cargo_command.args(["build", "--target", target, "--release"]); let mut rust_flags = "-Cdebuginfo=0".to_string(); if let Some(linker) = linker { @@ -102,11 +101,11 @@ path = "{binary}/main.rs""#, rust_flags.push_str(linker); rust_flags.push_str(" -C link-args=-static"); - command.env("CPPFLAGS", "-fuse-ld=lld"); + cargo_command.env("CPPFLAGS", "-fuse-ld=lld"); } - command.env("RUSTFLAGS", rust_flags); - let output = command.output(); + cargo_command.env("RUSTFLAGS", rust_flags); + let output = cargo_command.output(); if let Ok(output) = output { if output.status.success() { From 345d17a23614ee9c7bc7b60b82cfe7645eb646e2 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 8 Jul 2025 22:41:36 +0200 Subject: [PATCH 03/14] optimize C file creation --- crates/intrinsic-test/src/arm/mod.rs | 2 +- crates/intrinsic-test/src/common/gen_c.rs | 14 ------ .../intrinsic-test/src/common/write_file.rs | 49 ++++++++++--------- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 1a66ed4268..bfd0c4d6c3 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -72,7 +72,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { match compiler { None => true, Some(compiler) => compile_c_arm( - intrinsics_name_list.as_slice(), + intrinsics_name_list.unwrap().as_slice(), compiler, target, cxx_toolchain_dir, diff --git a/crates/intrinsic-test/src/common/gen_c.rs b/crates/intrinsic-test/src/common/gen_c.rs index 1cfb66c39b..66c4ad4d1a 100644 --- a/crates/intrinsic-test/src/common/gen_c.rs +++ b/crates/intrinsic-test/src/common/gen_c.rs @@ -1,6 +1,5 @@ use itertools::Itertools; use rayon::prelude::*; -use std::collections::BTreeMap; use std::process::Command; use super::argument::Argument; @@ -87,19 +86,6 @@ pub fn compile_c_programs(compiler_commands: &[String]) -> bool { .is_none() } -// Creates directory structure and file path mappings -pub fn setup_c_file_paths(identifiers: &Vec) -> BTreeMap<&String, String> { - let _ = std::fs::create_dir("c_programs"); - identifiers - .par_iter() - .map(|identifier| { - let c_filename = format!(r#"c_programs/{identifier}.cpp"#); - - (identifier, c_filename) - }) - .collect::>() -} - pub fn generate_c_test_loop( intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 0ba3e829a6..42a17efe1f 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -1,10 +1,9 @@ -use super::gen_c::create_c_test_program; -use super::gen_c::setup_c_file_paths; +use std::fs::File; +use std::io::Write; + use super::gen_rust::{create_rust_test_program, setup_rust_file_paths}; use super::intrinsic::IntrinsicDefinition; use super::intrinsic_helpers::IntrinsicTypeDefinition; -use std::fs::File; -use std::io::Write; pub fn write_file(filename: &String, code: String) { let mut file = File::create(filename).unwrap(); @@ -12,34 +11,36 @@ pub fn write_file(filename: &String, code: String) { } pub fn write_c_testfiles( - intrinsics: &Vec<&dyn IntrinsicDefinition>, + intrinsics: &[&dyn IntrinsicDefinition], target: &str, c_target: &str, headers: &[&str], notice: &str, arch_specific_definitions: &[&str], -) -> Vec { - let intrinsics_name_list = intrinsics +) -> std::io::Result> { + std::fs::create_dir_all("c_programs")?; + + intrinsics .iter() - .map(|i| i.name().clone()) - .collect::>(); - let filename_mapping = setup_c_file_paths(&intrinsics_name_list); + .map(|intrinsic| { + let identifier = intrinsic.name().to_owned(); + let mut file = File::create(format!("c_programs/{identifier}.cpp")).unwrap(); - intrinsics.iter().for_each(|&i| { - let c_code = create_c_test_program( - i, - headers, - target, - c_target, - notice, - arch_specific_definitions, - ); - if let Some(filename) = filename_mapping.get(&i.name()) { - write_file(filename, c_code) - }; - }); + // write_c_test_program(&mut file, intrinsic)?; + let c_code = crate::common::gen_c::create_c_test_program( + *intrinsic, + headers, + target, + c_target, + notice, + arch_specific_definitions, + ); - intrinsics_name_list + file.write_all(c_code.as_bytes())?; + + Ok(identifier) + }) + .collect() } pub fn write_rust_testfiles( From 2f7b0a60b5b4db5d811888393406434d1a089557 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 8 Jul 2025 22:52:35 +0200 Subject: [PATCH 04/14] optimize Rust file creation --- crates/intrinsic-test/src/arm/mod.rs | 11 ++-- crates/intrinsic-test/src/common/gen_rust.rs | 16 ----- .../intrinsic-test/src/common/write_file.rs | 59 +++++++++++-------- 3 files changed, 37 insertions(+), 49 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index bfd0c4d6c3..97d26f315e 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -57,11 +57,9 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let c_target = "aarch64"; let intrinsics_name_list = write_c_testfiles( - &self - .intrinsics + self.intrinsics .iter() - .map(|i| i as &dyn IntrinsicDefinition<_>) - .collect::>(), + .map(|i| i as &dyn IntrinsicDefinition<_>), target, c_target, &["arm_neon.h", "arm_acle.h", "arm_fp16.h"], @@ -92,15 +90,14 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let intrinsics_name_list = write_rust_testfiles( self.intrinsics .iter() - .map(|i| i as &dyn IntrinsicDefinition<_>) - .collect::>(), + .map(|i| i as &dyn IntrinsicDefinition<_>), rust_target, &build_notices("// "), F16_FORMATTING_DEF, AARCH_CONFIGURATIONS, ); - compile_rust_programs(intrinsics_name_list, toolchain, target, linker) + compile_rust_programs(intrinsics_name_list.unwrap(), toolchain, target, linker) } fn compare_outputs(&self) -> bool { diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index 0e4a95ab52..369fd037a1 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -1,6 +1,4 @@ use itertools::Itertools; -use rayon::prelude::*; -use std::collections::BTreeMap; use std::fs::File; use std::process::Command; @@ -124,20 +122,6 @@ pub fn compile_rust_programs( } } -// Creates directory structure and file path mappings -pub fn setup_rust_file_paths(identifiers: &Vec) -> BTreeMap<&String, String> { - identifiers - .par_iter() - .map(|identifier| { - let rust_dir = format!("rust_programs/{identifier}"); - let _ = std::fs::create_dir_all(&rust_dir); - let rust_filename = format!("{rust_dir}/main.rs"); - - (identifier, rust_filename) - }) - .collect::>() -} - pub fn generate_rust_test_loop( intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 42a17efe1f..a5fbfdfd7a 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -1,34 +1,32 @@ use std::fs::File; use std::io::Write; -use super::gen_rust::{create_rust_test_program, setup_rust_file_paths}; +use super::gen_rust::create_rust_test_program; use super::intrinsic::IntrinsicDefinition; use super::intrinsic_helpers::IntrinsicTypeDefinition; -pub fn write_file(filename: &String, code: String) { - let mut file = File::create(filename).unwrap(); - file.write_all(code.into_bytes().as_slice()).unwrap(); -} - -pub fn write_c_testfiles( - intrinsics: &[&dyn IntrinsicDefinition], +pub fn write_c_testfiles<'a, T, I>( + intrinsics: I, target: &str, c_target: &str, headers: &[&str], notice: &str, arch_specific_definitions: &[&str], -) -> std::io::Result> { +) -> std::io::Result> +where + T: IntrinsicTypeDefinition + Sized + 'a, + I: Iterator>, +{ std::fs::create_dir_all("c_programs")?; intrinsics - .iter() .map(|intrinsic| { let identifier = intrinsic.name().to_owned(); let mut file = File::create(format!("c_programs/{identifier}.cpp")).unwrap(); // write_c_test_program(&mut file, intrinsic)?; let c_code = crate::common::gen_c::create_c_test_program( - *intrinsic, + intrinsic, headers, target, c_target, @@ -43,25 +41,34 @@ pub fn write_c_testfiles( .collect() } -pub fn write_rust_testfiles( - intrinsics: Vec<&dyn IntrinsicDefinition>, +pub fn write_rust_testfiles<'a, T, I>( + intrinsics: I, rust_target: &str, notice: &str, definitions: &str, cfg: &str, -) -> Vec { - let intrinsics_name_list = intrinsics - .iter() - .map(|i| i.name().clone()) - .collect::>(); - let filename_mapping = setup_rust_file_paths(&intrinsics_name_list); +) -> std::io::Result> +where + T: IntrinsicTypeDefinition + Sized + 'a, + I: Iterator>, +{ + std::fs::create_dir_all("rust_programs")?; + + intrinsics + .map(|intrinsic| { + let identifier = intrinsic.name().to_owned(); - intrinsics.iter().for_each(|&i| { - let rust_code = create_rust_test_program(i, rust_target, notice, definitions, cfg); - if let Some(filename) = filename_mapping.get(&i.name()) { - write_file(filename, rust_code) - } - }); + let rust_dir = format!("rust_programs/{identifier}"); + std::fs::create_dir_all(&rust_dir)?; + let rust_filename = format!("{rust_dir}/main.rs"); + let mut file = File::create(rust_filename).unwrap(); - intrinsics_name_list + let rust_code = + create_rust_test_program(intrinsic, rust_target, notice, definitions, cfg); + + file.write_all(rust_code.as_bytes())?; + + Ok(identifier) + }) + .collect() } From 65598e999e641d4829b2963b8057b1eae0afad48 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 00:50:31 +0200 Subject: [PATCH 05/14] wire up the writer into more of the rust emission code --- crates/intrinsic-test/src/common/argument.rs | 34 +++-- crates/intrinsic-test/src/common/gen_rust.rs | 144 ++++++++---------- .../intrinsic-test/src/common/write_file.rs | 5 +- crates/intrinsic-test/src/main.rs | 2 +- 4 files changed, 83 insertions(+), 102 deletions(-) diff --git a/crates/intrinsic-test/src/common/argument.rs b/crates/intrinsic-test/src/common/argument.rs index 443ccb919f..a9efc6adef 100644 --- a/crates/intrinsic-test/src/common/argument.rs +++ b/crates/intrinsic-test/src/common/argument.rs @@ -142,21 +142,25 @@ where /// Creates a line for each argument that initializes an array for Rust from which `loads` argument /// values can be loaded as a sliding window, e.g `const A_VALS: [u32; 20] = [...];` - pub fn gen_arglists_rust(&self, indentation: Indentation, loads: u32) -> String { - self.iter() - .filter(|&arg| !arg.has_constraint()) - .map(|arg| { - format!( - "{indentation}{bind} {name}: [{ty}; {load_size}] = {values};", - bind = arg.rust_vals_array_binding(), - name = arg.rust_vals_array_name(), - ty = arg.ty.rust_scalar_type(), - load_size = arg.ty.num_lanes() * arg.ty.num_vectors() + loads - 1, - values = arg.ty.populate_random(indentation, loads, &Language::Rust) - ) - }) - .collect::>() - .join("\n") + pub fn gen_arglists_rust( + &self, + w: &mut impl std::io::Write, + indentation: Indentation, + loads: u32, + ) -> std::io::Result<()> { + for arg in self.iter().filter(|&arg| !arg.has_constraint()) { + writeln!( + w, + "{indentation}{bind} {name}: [{ty}; {load_size}] = {values};", + bind = arg.rust_vals_array_binding(), + name = arg.rust_vals_array_name(), + ty = arg.ty.rust_scalar_type(), + load_size = arg.ty.num_lanes() * arg.ty.num_vectors() + loads - 1, + values = arg.ty.populate_random(indentation, loads, &Language::Rust) + )? + } + + Ok(()) } /// Creates a line for each argument that initializes the argument from an array `[arg]_vals` at diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index 369fd037a1..b79bd1314f 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -10,31 +10,6 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition; // The number of times each intrinsic will be called. const PASSES: u32 = 20; -pub fn format_rust_main_template( - notices: &str, - definitions: &str, - configurations: &str, - arch_definition: &str, - arglists: &str, - passes: &str, -) -> String { - format!( - r#"{notices}#![feature(simd_ffi)] -#![feature(f16)] -#![allow(unused)] -{configurations} -{definitions} - -use core_arch::arch::{arch_definition}::*; - -fn main() {{ -{arglists} -{passes} -}} -"#, - ) -} - fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> { writeln!( w, @@ -123,11 +98,12 @@ pub fn compile_rust_programs( } pub fn generate_rust_test_loop( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, additional: &str, passes: u32, -) -> String { +) -> std::io::Result<()> { let constraints = intrinsic.arguments().as_constraint_parameters_rust(); let constraints = if !constraints.is_empty() { format!("::<{constraints}>") @@ -138,7 +114,8 @@ pub fn generate_rust_test_loop( let return_value = format_f16_return_value(intrinsic); let indentation2 = indentation.nested(); let indentation3 = indentation2.nested(); - format!( + write!( + w, "{indentation}for i in 0..{passes} {{\n\ {indentation2}unsafe {{\n\ {loaded_args}\ @@ -153,74 +130,77 @@ pub fn generate_rust_test_loop( ) } -pub fn generate_rust_constraint_blocks( +fn generate_rust_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, - constraints: &[&Argument], + constraints: &mut (impl Iterator> + Clone), name: String, -) -> String { - if let Some((current, constraints)) = constraints.split_last() { - let range = current - .constraint - .iter() - .map(|c| c.to_range()) - .flat_map(|r| r.into_iter()); - - let body_indentation = indentation.nested(); - range - .map(|i| { - format!( - "{indentation}{{\n\ - {body_indentation}const {name}: {ty} = {val};\n\ - {pass}\n\ - {indentation}}}", - name = current.name, - ty = current.ty.rust_type(), - val = i, - pass = generate_rust_constraint_blocks( - intrinsic, - body_indentation, - constraints, - format!("{name}-{i}") - ) - ) - }) - .join("\n") - } else { - generate_rust_test_loop(intrinsic, indentation, &name, PASSES) +) -> std::io::Result<()> { + let Some(current) = constraints.next() else { + return generate_rust_test_loop(w, intrinsic, indentation, &name, PASSES); + }; + + let body_indentation = indentation.nested(); + for i in current.constraint.iter().flat_map(|c| c.to_range()) { + let ty = current.ty.rust_type(); + + writeln!(w, "{indentation}{{")?; + + writeln!(w, "{body_indentation}const {}: {ty} = {i};", current.name)?; + + generate_rust_constraint_blocks( + w, + intrinsic, + body_indentation, + &mut constraints.clone(), + format!("{name}-{i}"), + )?; + + writeln!(w, "{indentation}}}")?; } + + Ok(()) } // Top-level function to create complete test program pub fn create_rust_test_program( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, target: &str, notice: &str, definitions: &str, cfg: &str, -) -> String { +) -> std::io::Result<()> { + let indentation = Indentation::default(); + + write!(w, "{notice}")?; + + writeln!(w, "#![feature(simd_ffi)]")?; + writeln!(w, "#![feature(f16)]")?; + writeln!(w, "#![allow(unused)]")?; + + writeln!(w, "{cfg}")?; + writeln!(w, "{definitions}")?; + + writeln!(w, "use core_arch::arch::{target}::*;")?; + + writeln!(w, "fn main() {{")?; + + // Define the arrays of arguments. let arguments = intrinsic.arguments(); - let constraints = arguments - .iter() - .filter(|i| i.has_constraint()) - .collect_vec(); + arguments.gen_arglists_rust(w, indentation.nested(), PASSES)?; - let indentation = Indentation::default(); - format_rust_main_template( - notice, - definitions, - cfg, - target, - intrinsic - .arguments() - .gen_arglists_rust(indentation.nested(), PASSES) - .as_str(), - generate_rust_constraint_blocks( - intrinsic, - indentation.nested(), - &constraints, - Default::default(), - ) - .as_str(), - ) + // Define any const generics as `const` items, then generate the actual test loop. + generate_rust_constraint_blocks( + w, + intrinsic, + indentation.nested(), + &mut arguments.iter().rev().filter(|i| i.has_constraint()), + Default::default(), + )?; + + writeln!(w, "}}")?; + + Ok(()) } diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index a5fbfdfd7a..4829c651db 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -63,10 +63,7 @@ where let rust_filename = format!("{rust_dir}/main.rs"); let mut file = File::create(rust_filename).unwrap(); - let rust_code = - create_rust_test_program(intrinsic, rust_target, notice, definitions, cfg); - - file.write_all(rust_code.as_bytes())?; + create_rust_test_program(&mut file, intrinsic, rust_target, notice, definitions, cfg)?; Ok(identifier) }) diff --git a/crates/intrinsic-test/src/main.rs b/crates/intrinsic-test/src/main.rs index 538f317a29..a30397d3a6 100644 --- a/crates/intrinsic-test/src/main.rs +++ b/crates/intrinsic-test/src/main.rs @@ -38,7 +38,7 @@ fn main() { if !test_environment.build_rust_file() { std::process::exit(3); } - info!("comaparing outputs"); + info!("comparing outputs"); if !test_environment.compare_outputs() { std::process::exit(1); } From c45dc7d6af8ded6f6ab8198bb0d2bac23be16c18 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 11:40:41 +0200 Subject: [PATCH 06/14] wire up writer for C file writes --- crates/intrinsic-test/src/common/argument.rs | 30 +-- crates/intrinsic-test/src/common/gen_c.rs | 204 ++++++++---------- .../intrinsic-test/src/common/write_file.rs | 9 +- 3 files changed, 110 insertions(+), 133 deletions(-) diff --git a/crates/intrinsic-test/src/common/argument.rs b/crates/intrinsic-test/src/common/argument.rs index a9efc6adef..b72c954f4a 100644 --- a/crates/intrinsic-test/src/common/argument.rs +++ b/crates/intrinsic-test/src/common/argument.rs @@ -125,19 +125,23 @@ where /// Creates a line for each argument that initializes an array for C from which `loads` argument /// values can be loaded as a sliding window. /// e.g `const int32x2_t a_vals = {0x3effffff, 0x3effffff, 0x3f7fffff}`, if loads=2. - pub fn gen_arglists_c(&self, indentation: Indentation, loads: u32) -> String { - self.iter() - .filter(|&arg| !arg.has_constraint()) - .map(|arg| { - format!( - "{indentation}const {ty} {name}_vals[] = {values};", - ty = arg.ty.c_scalar_type(), - name = arg.name, - values = arg.ty.populate_random(indentation, loads, &Language::C) - ) - }) - .collect::>() - .join("\n") + pub fn gen_arglists_c( + &self, + w: &mut impl std::io::Write, + indentation: Indentation, + loads: u32, + ) -> std::io::Result<()> { + for arg in self.iter().filter(|&arg| !arg.has_constraint()) { + writeln!( + w, + "{indentation}const {ty} {name}_vals[] = {values};", + ty = arg.ty.c_scalar_type(), + name = arg.name, + values = arg.ty.populate_random(indentation, loads, &Language::C) + )? + } + + Ok(()) } /// Creates a line for each argument that initializes an array for Rust from which `loads` argument diff --git a/crates/intrinsic-test/src/common/gen_c.rs b/crates/intrinsic-test/src/common/gen_c.rs index 66c4ad4d1a..f5d7e8b4aa 100644 --- a/crates/intrinsic-test/src/common/gen_c.rs +++ b/crates/intrinsic-test/src/common/gen_c.rs @@ -1,4 +1,3 @@ -use itertools::Itertools; use rayon::prelude::*; use std::process::Command; @@ -10,57 +9,6 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition; // The number of times each intrinsic will be called. const PASSES: u32 = 20; -// Formats the main C program template with placeholders -pub fn format_c_main_template( - notices: &str, - header_files: &[&str], - arch_identifier: &str, - arch_specific_definitions: &[&str], - arglists: &str, - passes: &str, -) -> String { - format!( - r#"{notices}{header_files} -#include -#include -#include -#include - -template T1 cast(T2 x) {{ - static_assert(sizeof(T1) == sizeof(T2), "sizeof T1 and T2 must be the same"); - T1 ret{{}}; - memcpy(&ret, &x, sizeof(T1)); - return ret; -}} - -std::ostream& operator<<(std::ostream& os, float16_t value) {{ - uint16_t temp = 0; - memcpy(&temp, &value, sizeof(float16_t)); - std::stringstream ss; - ss << "0x" << std::setfill('0') << std::setw(4) << std::hex << temp; - os << ss.str(); - return os; -}} - -#ifdef __{arch_identifier}__ -{arch_specific_definitions} -#endif - -{arglists} - -int main(int argc, char **argv) {{ -{passes} - return 0; -}}"#, - header_files = header_files - .iter() - .map(|header| format!("#include <{header}>")) - .collect::>() - .join("\n"), - arch_specific_definitions = arch_specific_definitions.join("\n"), - ) -} - pub fn compile_c_programs(compiler_commands: &[String]) -> bool { compiler_commands .par_iter() @@ -87,14 +35,15 @@ pub fn compile_c_programs(compiler_commands: &[String]) -> bool { } pub fn generate_c_test_loop( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, additional: &str, passes: u32, - _target: &str, -) -> String { +) -> std::io::Result<()> { let body_indentation = indentation.nested(); - format!( + write!( + w, "{indentation}for (int i=0; i<{passes}; i++) {{\n\ {loaded_args}\ {body_indentation}auto __return_value = {intrinsic_call}({args});\n\ @@ -107,78 +56,105 @@ pub fn generate_c_test_loop( ) } -pub fn generate_c_constraint_blocks( +pub fn generate_c_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, indentation: Indentation, - constraints: &[&Argument], + constraints: &mut (impl Iterator> + Clone), name: String, - target: &str, -) -> String { - if let Some((current, constraints)) = constraints.split_last() { - let range = current - .constraint - .iter() - .map(|c| c.to_range()) - .flat_map(|r| r.into_iter()); - - let body_indentation = indentation.nested(); - range - .map(|i| { - format!( - "{indentation}{{\n\ - {body_indentation}{ty} {name} = {val};\n\ - {pass}\n\ - {indentation}}}", - name = current.name, - ty = current.ty.c_type(), - val = i, - pass = generate_c_constraint_blocks( - intrinsic, - body_indentation, - constraints, - format!("{name}-{i}"), - target, - ) - ) - }) - .join("\n") - } else { - generate_c_test_loop(intrinsic, indentation, &name, PASSES, target) +) -> std::io::Result<()> { + let Some(current) = constraints.next() else { + return generate_c_test_loop(w, intrinsic, indentation, &name, PASSES); + }; + + let body_indentation = indentation.nested(); + for i in current.constraint.iter().flat_map(|c| c.to_range()) { + let ty = current.ty.c_type(); + + writeln!(w, "{indentation}{{")?; + writeln!(w, "{body_indentation}{ty} {} = {i};", current.name)?; + + generate_c_constraint_blocks( + w, + intrinsic, + body_indentation, + &mut constraints.clone(), + format!("{name}-{i}"), + )?; + + writeln!(w, "{indentation}}}")?; } + + Ok(()) } // Compiles C test programs using specified compiler pub fn create_c_test_program( + w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, header_files: &[&str], - target: &str, + _target: &str, c_target: &str, notices: &str, arch_specific_definitions: &[&str], -) -> String { +) -> std::io::Result<()> { + let indentation = Indentation::default(); + + write!(w, "{notices}")?; + + for header in header_files { + writeln!(w, "#include <{header}>")?; + } + + writeln!( + w, + r#" +#include +#include +#include +#include + +template T1 cast(T2 x) {{ + static_assert(sizeof(T1) == sizeof(T2), "sizeof T1 and T2 must be the same"); + T1 ret{{}}; + memcpy(&ret, &x, sizeof(T1)); + return ret; +}} + +std::ostream& operator<<(std::ostream& os, float16_t value) {{ + uint16_t temp = 0; + memcpy(&temp, &value, sizeof(float16_t)); + std::stringstream ss; + ss << "0x" << std::setfill('0') << std::setw(4) << std::hex << temp; + os << ss.str(); + return os; +}} +"# + )?; + + let arch_identifier = c_target; + writeln!(w, "#ifdef __{arch_identifier}__")?; + for def in arch_specific_definitions { + writeln!(w, "{def}")?; + } + writeln!(w, "#endif")?; + + // Define the arrays of arguments. let arguments = intrinsic.arguments(); - let constraints = arguments - .iter() - .filter(|&i| i.has_constraint()) - .collect_vec(); + arguments.gen_arglists_c(w, indentation, PASSES)?; - let indentation = Indentation::default(); - format_c_main_template( - notices, - header_files, - c_target, - arch_specific_definitions, - intrinsic - .arguments() - .gen_arglists_c(indentation, PASSES) - .as_str(), - generate_c_constraint_blocks( - intrinsic, - indentation.nested(), - constraints.as_slice(), - Default::default(), - target, - ) - .as_str(), - ) + writeln!(w, "int main(int argc, char **argv) {{")?; + + generate_c_constraint_blocks( + w, + intrinsic, + indentation.nested(), + &mut arguments.iter().rev().filter(|&i| i.has_constraint()), + Default::default(), + )?; + + writeln!(w, " return 0;")?; + writeln!(w, "}}")?; + + Ok(()) } diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 4829c651db..5e858c7d05 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -1,5 +1,4 @@ use std::fs::File; -use std::io::Write; use super::gen_rust::create_rust_test_program; use super::intrinsic::IntrinsicDefinition; @@ -24,17 +23,15 @@ where let identifier = intrinsic.name().to_owned(); let mut file = File::create(format!("c_programs/{identifier}.cpp")).unwrap(); - // write_c_test_program(&mut file, intrinsic)?; - let c_code = crate::common::gen_c::create_c_test_program( + crate::common::gen_c::create_c_test_program( + &mut file, intrinsic, headers, target, c_target, notice, arch_specific_definitions, - ); - - file.write_all(c_code.as_bytes())?; + )?; Ok(identifier) }) From 85b82fae15f5485c7edb02a4a9ab41d06a5ab210 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 11:45:36 +0200 Subject: [PATCH 07/14] get rid of a dyn to use `par_iter` --- crates/intrinsic-test/src/arm/mod.rs | 12 +++++------- crates/intrinsic-test/src/common/write_file.rs | 12 ++++++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 97d26f315e..2fdfa8caa5 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -1,3 +1,5 @@ +use rayon::prelude::*; + mod compile; mod config; mod intrinsic; @@ -8,7 +10,7 @@ use crate::common::SupportedArchitectureTest; use crate::common::cli::ProcessedCli; use crate::common::compare::compare_outputs; use crate::common::gen_rust::compile_rust_programs; -use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition}; +use crate::common::intrinsic::Intrinsic; use crate::common::intrinsic_helpers::TypeKind; use crate::common::write_file::{write_c_testfiles, write_rust_testfiles}; use compile::compile_c_arm; @@ -57,9 +59,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let c_target = "aarch64"; let intrinsics_name_list = write_c_testfiles( - self.intrinsics - .iter() - .map(|i| i as &dyn IntrinsicDefinition<_>), + self.intrinsics.par_iter(), target, c_target, &["arm_neon.h", "arm_acle.h", "arm_fp16.h"], @@ -88,9 +88,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let toolchain = self.cli_options.toolchain.as_deref(); let linker = self.cli_options.linker.as_deref(); let intrinsics_name_list = write_rust_testfiles( - self.intrinsics - .iter() - .map(|i| i as &dyn IntrinsicDefinition<_>), + self.intrinsics.par_iter(), rust_target, &build_notices("// "), F16_FORMATTING_DEF, diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 5e858c7d05..0038b6210c 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -4,7 +4,9 @@ use super::gen_rust::create_rust_test_program; use super::intrinsic::IntrinsicDefinition; use super::intrinsic_helpers::IntrinsicTypeDefinition; -pub fn write_c_testfiles<'a, T, I>( +use rayon::prelude::*; + +pub fn write_c_testfiles<'a, T, I, E>( intrinsics: I, target: &str, c_target: &str, @@ -14,7 +16,8 @@ pub fn write_c_testfiles<'a, T, I>( ) -> std::io::Result> where T: IntrinsicTypeDefinition + Sized + 'a, - I: Iterator>, + I: ParallelIterator, + E: IntrinsicDefinition + 'a, { std::fs::create_dir_all("c_programs")?; @@ -38,7 +41,7 @@ where .collect() } -pub fn write_rust_testfiles<'a, T, I>( +pub fn write_rust_testfiles<'a, T, I, E>( intrinsics: I, rust_target: &str, notice: &str, @@ -47,7 +50,8 @@ pub fn write_rust_testfiles<'a, T, I>( ) -> std::io::Result> where T: IntrinsicTypeDefinition + Sized + 'a, - I: Iterator>, + I: ParallelIterator, + E: IntrinsicDefinition + 'a, { std::fs::create_dir_all("rust_programs")?; From cef530ba30d36b8cb787b31fb2b3e1d6c03d4c2f Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 14:03:31 +0200 Subject: [PATCH 08/14] combine all rust intrinsics in a single program --- crates/intrinsic-test/src/arm/mod.rs | 2 +- crates/intrinsic-test/src/common/compare.rs | 5 ++- crates/intrinsic-test/src/common/gen_rust.rs | 45 ++++++++++++++++++- .../intrinsic-test/src/common/write_file.rs | 6 +-- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 2fdfa8caa5..a5e7be87df 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -1,7 +1,7 @@ use rayon::prelude::*; mod compile; -mod config; +pub mod config; mod intrinsic; mod json_parser; mod types; diff --git a/crates/intrinsic-test/src/common/compare.rs b/crates/intrinsic-test/src/common/compare.rs index f16d83b026..6a76927ddb 100644 --- a/crates/intrinsic-test/src/common/compare.rs +++ b/crates/intrinsic-test/src/common/compare.rs @@ -19,8 +19,9 @@ pub fn compare_outputs(intrinsic_name_list: &Vec, runner: &str, target: .output(); let rust = runner_command(runner) - .arg(format!("target/{target}/release/{intrinsic_name}")) - .env("RUSTFLAGS", "-Cdebuginfo=0") + .arg(format!("target/{target}/release/intrinsic-test-programs")) + .arg(intrinsic_name) + .current_dir("rust_programs") .output(); let (c, rust) = match (c, rust) { diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index b79bd1314f..e33f77ccf6 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -52,8 +52,49 @@ pub fn compile_rust_programs( target: &str, linker: Option<&str>, ) -> bool { + use std::io::Write; + let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); - write_cargo_toml(&mut cargo, &binaries).unwrap(); + write_cargo_toml(&mut cargo, &[]).unwrap(); + let mut main_rs = File::create("rust_programs/src/main.rs").unwrap(); + + writeln!(main_rs, "#![feature(simd_ffi)]").unwrap(); + writeln!(main_rs, "#![feature(f16)]").unwrap(); + writeln!(main_rs, "#![allow(unused)]").unwrap(); + + let definitions = crate::arm::config::F16_FORMATTING_DEF; + let cfg = crate::arm::config::AARCH_CONFIGURATIONS; + + writeln!(main_rs, "{cfg}").unwrap(); + writeln!(main_rs, "{definitions}").unwrap(); + + // TODO hardcodes target + writeln!(main_rs, "use core_arch::arch::aarch64::*;").unwrap(); + + for binary in binaries.iter() { + writeln!(main_rs, "mod {binary};").unwrap(); + } + + writeln!(main_rs, "fn main() {{").unwrap(); + + writeln!( + main_rs, + " match std::env::args().nth(1).unwrap().as_str() {{" + ) + .unwrap(); + + for binary in binaries { + writeln!(main_rs, " \"{binary}\" => {binary}::run(),").unwrap(); + } + + writeln!( + main_rs, + " other => panic!(\"unknown intrinsic `{{}}`\", other)," + ) + .unwrap(); + + writeln!(main_rs, " }}").unwrap(); + writeln!(main_rs, "}}").unwrap(); /* If there has been a linker explicitly set from the command line then * we want to set it via setting it in the RUSTFLAGS*/ @@ -185,7 +226,7 @@ pub fn create_rust_test_program( writeln!(w, "use core_arch::arch::{target}::*;")?; - writeln!(w, "fn main() {{")?; + writeln!(w, "pub fn run() {{")?; // Define the arrays of arguments. let arguments = intrinsic.arguments(); diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 0038b6210c..44538c68e0 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -53,15 +53,13 @@ where I: ParallelIterator, E: IntrinsicDefinition + 'a, { - std::fs::create_dir_all("rust_programs")?; + std::fs::create_dir_all("rust_programs/src")?; intrinsics .map(|intrinsic| { let identifier = intrinsic.name().to_owned(); - let rust_dir = format!("rust_programs/{identifier}"); - std::fs::create_dir_all(&rust_dir)?; - let rust_filename = format!("{rust_dir}/main.rs"); + let rust_filename = format!("rust_programs/src/{identifier}.rs"); let mut file = File::create(rust_filename).unwrap(); create_rust_test_program(&mut file, intrinsic, rust_target, notice, definitions, cfg)?; From bd809f396b19fba24488c756c689244f4c83f5dd Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 14:17:17 +0200 Subject: [PATCH 09/14] only compile formatting logic once --- crates/intrinsic-test/src/arm/config.rs | 6 +- crates/intrinsic-test/src/arm/mod.rs | 34 ++++++-- crates/intrinsic-test/src/common/gen_rust.rs | 83 +++++++------------ .../intrinsic-test/src/common/write_file.rs | 10 +-- 4 files changed, 64 insertions(+), 69 deletions(-) diff --git a/crates/intrinsic-test/src/arm/config.rs b/crates/intrinsic-test/src/arm/config.rs index 9a7b37253d..53c3ba29b4 100644 --- a/crates/intrinsic-test/src/arm/config.rs +++ b/crates/intrinsic-test/src/arm/config.rs @@ -26,7 +26,7 @@ pub const F16_FORMATTING_DEF: &str = r#" /// Used to continue `Debug`ging SIMD types as `MySimd(1, 2, 3, 4)`, as they /// were before moving to array-based simd. #[inline] -fn debug_simd_finish( +pub(crate) fn debug_simd_finish( formatter: &mut core::fmt::Formatter<'_>, type_name: &str, array: &[T; N], @@ -47,11 +47,11 @@ impl core::fmt::Debug for Hex { } } -fn debug_f16(x: T) -> impl core::fmt::Debug { +pub(crate) fn debug_f16(x: T) -> impl core::fmt::Debug { Hex(x) } -trait DebugHexF16 { +pub(crate) trait DebugHexF16 { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result; } diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index a5e7be87df..375aa9c4ba 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -1,7 +1,9 @@ +use std::fs::File; + use rayon::prelude::*; mod compile; -pub mod config; +mod config; mod intrinsic; mod json_parser; mod types; @@ -9,7 +11,7 @@ mod types; use crate::common::SupportedArchitectureTest; use crate::common::cli::ProcessedCli; use crate::common::compare::compare_outputs; -use crate::common::gen_rust::compile_rust_programs; +use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs}; use crate::common::intrinsic::Intrinsic; use crate::common::intrinsic_helpers::TypeKind; use crate::common::write_file::{write_c_testfiles, write_rust_testfiles}; @@ -79,23 +81,37 @@ impl SupportedArchitectureTest for ArmArchitectureTest { } fn build_rust_file(&self) -> bool { - let rust_target = if self.cli_options.target.contains("v7") { + let architecture = if self.cli_options.target.contains("v7") { "arm" } else { "aarch64" }; + + let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); + write_cargo_toml(&mut cargo, &[]).unwrap(); + + let mut main_rs = File::create("rust_programs/src/main.rs").unwrap(); + write_main_rs( + &mut main_rs, + architecture, + AARCH_CONFIGURATIONS, + F16_FORMATTING_DEF, + self.intrinsics.iter().map(|i| i.name.as_str()), + ) + .unwrap(); + let target = &self.cli_options.target; let toolchain = self.cli_options.toolchain.as_deref(); let linker = self.cli_options.linker.as_deref(); - let intrinsics_name_list = write_rust_testfiles( + + write_rust_testfiles( self.intrinsics.par_iter(), - rust_target, + architecture, &build_notices("// "), - F16_FORMATTING_DEF, - AARCH_CONFIGURATIONS, - ); + ) + .unwrap(); - compile_rust_programs(intrinsics_name_list.unwrap(), toolchain, target, linker) + compile_rust_programs(toolchain, target, linker) } fn compare_outputs(&self) -> bool { diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index e33f77ccf6..9f4d9b4d1d 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -1,5 +1,4 @@ use itertools::Itertools; -use std::fs::File; use std::process::Command; use super::argument::Argument; @@ -10,7 +9,7 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition; // The number of times each intrinsic will be called. const PASSES: u32 = 20; -fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> { +pub fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> { writeln!( w, concat!( @@ -46,56 +45,46 @@ fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io Ok(()) } -pub fn compile_rust_programs( - binaries: Vec, - toolchain: Option<&str>, - target: &str, - linker: Option<&str>, -) -> bool { - use std::io::Write; - - let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); - write_cargo_toml(&mut cargo, &[]).unwrap(); - let mut main_rs = File::create("rust_programs/src/main.rs").unwrap(); - - writeln!(main_rs, "#![feature(simd_ffi)]").unwrap(); - writeln!(main_rs, "#![feature(f16)]").unwrap(); - writeln!(main_rs, "#![allow(unused)]").unwrap(); - - let definitions = crate::arm::config::F16_FORMATTING_DEF; - let cfg = crate::arm::config::AARCH_CONFIGURATIONS; +pub fn write_main_rs<'a>( + w: &mut impl std::io::Write, + architecture: &str, + cfg: &str, + definitions: &str, + intrinsics: impl Iterator + Clone, +) -> std::io::Result<()> { + writeln!(w, "#![feature(simd_ffi)]")?; + writeln!(w, "#![feature(f16)]")?; + writeln!(w, "#![allow(unused)]")?; - writeln!(main_rs, "{cfg}").unwrap(); - writeln!(main_rs, "{definitions}").unwrap(); + writeln!(w, "{cfg}")?; + writeln!(w, "{definitions}")?; - // TODO hardcodes target - writeln!(main_rs, "use core_arch::arch::aarch64::*;").unwrap(); + writeln!(w, "use core_arch::arch::{architecture}::*;")?; - for binary in binaries.iter() { - writeln!(main_rs, "mod {binary};").unwrap(); + for binary in intrinsics.clone() { + writeln!(w, "mod {binary};")?; } - writeln!(main_rs, "fn main() {{").unwrap(); + writeln!(w, "fn main() {{")?; - writeln!( - main_rs, - " match std::env::args().nth(1).unwrap().as_str() {{" - ) - .unwrap(); + writeln!(w, " match std::env::args().nth(1).unwrap().as_str() {{")?; - for binary in binaries { - writeln!(main_rs, " \"{binary}\" => {binary}::run(),").unwrap(); + for binary in intrinsics { + writeln!(w, " \"{binary}\" => {binary}::run(),")?; } writeln!( - main_rs, + w, " other => panic!(\"unknown intrinsic `{{}}`\", other)," - ) - .unwrap(); + )?; + + writeln!(w, " }}")?; + writeln!(w, "}}")?; - writeln!(main_rs, " }}").unwrap(); - writeln!(main_rs, "}}").unwrap(); + Ok(()) +} +pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Option<&str>) -> bool { /* If there has been a linker explicitly set from the command line then * we want to set it via setting it in the RUSTFLAGS*/ @@ -155,7 +144,7 @@ pub fn generate_rust_test_loop( let return_value = format_f16_return_value(intrinsic); let indentation2 = indentation.nested(); let indentation3 = indentation2.nested(); - write!( + writeln!( w, "{indentation}for i in 0..{passes} {{\n\ {indentation2}unsafe {{\n\ @@ -208,23 +197,15 @@ fn generate_rust_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>( pub fn create_rust_test_program( w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, - target: &str, + architecture: &str, notice: &str, - definitions: &str, - cfg: &str, ) -> std::io::Result<()> { let indentation = Indentation::default(); write!(w, "{notice}")?; - writeln!(w, "#![feature(simd_ffi)]")?; - writeln!(w, "#![feature(f16)]")?; - writeln!(w, "#![allow(unused)]")?; - - writeln!(w, "{cfg}")?; - writeln!(w, "{definitions}")?; - - writeln!(w, "use core_arch::arch::{target}::*;")?; + writeln!(w, "use core_arch::arch::{architecture}::*;")?; + writeln!(w, "use crate::{{debug_simd_finish, debug_f16}};")?; writeln!(w, "pub fn run() {{")?; diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 44538c68e0..593947a3e0 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -43,11 +43,9 @@ where pub fn write_rust_testfiles<'a, T, I, E>( intrinsics: I, - rust_target: &str, + architecture: &str, notice: &str, - definitions: &str, - cfg: &str, -) -> std::io::Result> +) -> std::io::Result<()> where T: IntrinsicTypeDefinition + Sized + 'a, I: ParallelIterator, @@ -62,9 +60,9 @@ where let rust_filename = format!("rust_programs/src/{identifier}.rs"); let mut file = File::create(rust_filename).unwrap(); - create_rust_test_program(&mut file, intrinsic, rust_target, notice, definitions, cfg)?; + create_rust_test_program(&mut file, intrinsic, architecture, notice)?; - Ok(identifier) + Ok(()) }) .collect() } From 6f2d27f6f446269f602c090d8641444e6e87db21 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 15:53:22 +0200 Subject: [PATCH 10/14] combine rust functions into #`available_parallelism` files --- crates/intrinsic-test/src/arm/mod.rs | 37 +++++++++++++++---- crates/intrinsic-test/src/common/gen_rust.rs | 19 ++++------ .../intrinsic-test/src/common/write_file.rs | 27 -------------- crates/intrinsic-test/src/main.rs | 8 ++-- 4 files changed, 41 insertions(+), 50 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 375aa9c4ba..11dfaa7002 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -14,7 +14,7 @@ use crate::common::compare::compare_outputs; use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs}; use crate::common::intrinsic::Intrinsic; use crate::common::intrinsic_helpers::TypeKind; -use crate::common::write_file::{write_c_testfiles, write_rust_testfiles}; +use crate::common::write_file::write_c_testfiles; use compile::compile_c_arm; use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices}; use intrinsic::ArmIntrinsicType; @@ -81,18 +81,24 @@ impl SupportedArchitectureTest for ArmArchitectureTest { } fn build_rust_file(&self) -> bool { + std::fs::create_dir_all("rust_programs/src").unwrap(); + let architecture = if self.cli_options.target.contains("v7") { "arm" } else { "aarch64" }; + let available_parallelism = std::thread::available_parallelism().unwrap().get(); + let chunk_size = self.intrinsics.len().div_ceil(available_parallelism); + let mut cargo = File::create("rust_programs/Cargo.toml").unwrap(); write_cargo_toml(&mut cargo, &[]).unwrap(); let mut main_rs = File::create("rust_programs/src/main.rs").unwrap(); write_main_rs( &mut main_rs, + available_parallelism, architecture, AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, @@ -104,12 +110,29 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let toolchain = self.cli_options.toolchain.as_deref(); let linker = self.cli_options.linker.as_deref(); - write_rust_testfiles( - self.intrinsics.par_iter(), - architecture, - &build_notices("// "), - ) - .unwrap(); + let notice = &build_notices("// "); + self.intrinsics + .par_chunks(chunk_size) + .enumerate() + .map(|(i, chunk)| { + use std::io::Write; + + let rust_filename = format!("rust_programs/src/mod_{i}.rs"); + let mut file = File::create(rust_filename).unwrap(); + + write!(file, "{notice}")?; + + writeln!(file, "use core_arch::arch::{architecture}::*;")?; + writeln!(file, "use crate::{{debug_simd_finish, debug_f16}};")?; + + for intrinsic in chunk { + crate::common::gen_rust::create_rust_test_module(&mut file, intrinsic)?; + } + + Ok(()) + }) + .collect::>() + .unwrap(); compile_rust_programs(toolchain, target, linker) } diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index 9f4d9b4d1d..142e1d7ace 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -47,6 +47,7 @@ pub fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std pub fn write_main_rs<'a>( w: &mut impl std::io::Write, + available_parallelism: usize, architecture: &str, cfg: &str, definitions: &str, @@ -61,8 +62,9 @@ pub fn write_main_rs<'a>( writeln!(w, "use core_arch::arch::{architecture}::*;")?; - for binary in intrinsics.clone() { - writeln!(w, "mod {binary};")?; + for module in 0..available_parallelism { + writeln!(w, "mod mod_{module};")?; + writeln!(w, "use mod_{module}::*;")?; } writeln!(w, "fn main() {{")?; @@ -70,7 +72,7 @@ pub fn write_main_rs<'a>( writeln!(w, " match std::env::args().nth(1).unwrap().as_str() {{")?; for binary in intrinsics { - writeln!(w, " \"{binary}\" => {binary}::run(),")?; + writeln!(w, " \"{binary}\" => run_{binary}(),")?; } writeln!( @@ -194,20 +196,13 @@ fn generate_rust_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>( } // Top-level function to create complete test program -pub fn create_rust_test_program( +pub fn create_rust_test_module( w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, - architecture: &str, - notice: &str, ) -> std::io::Result<()> { let indentation = Indentation::default(); - write!(w, "{notice}")?; - - writeln!(w, "use core_arch::arch::{architecture}::*;")?; - writeln!(w, "use crate::{{debug_simd_finish, debug_f16}};")?; - - writeln!(w, "pub fn run() {{")?; + writeln!(w, "pub fn run_{}() {{", intrinsic.name())?; // Define the arrays of arguments. let arguments = intrinsic.arguments(); diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index 593947a3e0..a52b1b89dc 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -1,6 +1,5 @@ use std::fs::File; -use super::gen_rust::create_rust_test_program; use super::intrinsic::IntrinsicDefinition; use super::intrinsic_helpers::IntrinsicTypeDefinition; @@ -40,29 +39,3 @@ where }) .collect() } - -pub fn write_rust_testfiles<'a, T, I, E>( - intrinsics: I, - architecture: &str, - notice: &str, -) -> std::io::Result<()> -where - T: IntrinsicTypeDefinition + Sized + 'a, - I: ParallelIterator, - E: IntrinsicDefinition + 'a, -{ - std::fs::create_dir_all("rust_programs/src")?; - - intrinsics - .map(|intrinsic| { - let identifier = intrinsic.name().to_owned(); - - let rust_filename = format!("rust_programs/src/{identifier}.rs"); - let mut file = File::create(rust_filename).unwrap(); - - create_rust_test_program(&mut file, intrinsic, architecture, notice)?; - - Ok(()) - }) - .collect() -} diff --git a/crates/intrinsic-test/src/main.rs b/crates/intrinsic-test/src/main.rs index a30397d3a6..925917be66 100644 --- a/crates/intrinsic-test/src/main.rs +++ b/crates/intrinsic-test/src/main.rs @@ -30,14 +30,14 @@ fn main() { let test_environment = test_environment_result.unwrap(); - info!("building C binaries"); - if !test_environment.build_c_file() { - std::process::exit(2); - } info!("building Rust binaries"); if !test_environment.build_rust_file() { std::process::exit(3); } + info!("building C binaries"); + if !test_environment.build_c_file() { + std::process::exit(2); + } info!("comparing outputs"); if !test_environment.compare_outputs() { std::process::exit(1); From 7026b09cae41e00bfdcc73fad3ffbf3353913e67 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 16:37:27 +0200 Subject: [PATCH 11/14] when generating C, put the arguments within the function --- crates/intrinsic-test/src/common/gen_c.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/intrinsic-test/src/common/gen_c.rs b/crates/intrinsic-test/src/common/gen_c.rs index f5d7e8b4aa..aed99609e5 100644 --- a/crates/intrinsic-test/src/common/gen_c.rs +++ b/crates/intrinsic-test/src/common/gen_c.rs @@ -139,11 +139,11 @@ std::ostream& operator<<(std::ostream& os, float16_t value) {{ } writeln!(w, "#endif")?; + writeln!(w, "int main(int argc, char **argv) {{")?; + // Define the arrays of arguments. let arguments = intrinsic.arguments(); - arguments.gen_arglists_c(w, indentation, PASSES)?; - - writeln!(w, "int main(int argc, char **argv) {{")?; + arguments.gen_arglists_c(w, indentation.nested(), PASSES)?; generate_c_constraint_blocks( w, From 0a593988d6b5002ad2d98050f98cca2f019a5434 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 18:20:56 +0200 Subject: [PATCH 12/14] WIP --- crates/intrinsic-test/src/arm/mod.rs | 95 +++++++++++++--- crates/intrinsic-test/src/common/compare.rs | 4 +- crates/intrinsic-test/src/common/gen_c.rs | 102 ++++++++++++++---- .../intrinsic-test/src/common/write_file.rs | 79 +++++++------- 4 files changed, 199 insertions(+), 81 deletions(-) diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 11dfaa7002..82ec318e12 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -11,11 +11,10 @@ mod types; use crate::common::SupportedArchitectureTest; use crate::common::cli::ProcessedCli; use crate::common::compare::compare_outputs; +use crate::common::gen_c::{write_main_cpp, write_mod_cpp}; use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs}; use crate::common::intrinsic::Intrinsic; use crate::common::intrinsic_helpers::TypeKind; -use crate::common::write_file::write_c_testfiles; -use compile::compile_c_arm; use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices}; use intrinsic::ArmIntrinsicType; use json_parser::get_neon_intrinsics; @@ -60,24 +59,86 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let cxx_toolchain_dir = self.cli_options.cxx_toolchain_dir.as_deref(); let c_target = "aarch64"; - let intrinsics_name_list = write_c_testfiles( - self.intrinsics.par_iter(), - target, + let available_parallelism = std::thread::available_parallelism().unwrap().get(); + let chunk_size = self.intrinsics.len().div_ceil(available_parallelism); + + let notice = &build_notices("// "); + self.intrinsics + .par_chunks(chunk_size) + .enumerate() + .map(|(i, chunk)| { + let c_filename = format!("c_programs/mod_{i}.cpp"); + let mut file = File::create(&c_filename).unwrap(); + write_mod_cpp(&mut file, ¬ice, chunk).unwrap(); + + // compile this cpp file into a .o file + + // clang++ -march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3 -O2 -ffp-contract=off -Wno-narrowing --target=aarch64-unknown-linux-gnu c_file_0.c -c + let mut cmd = std::process::Command::new("clang++"); + cmd.current_dir("c_programs"); + + cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3"); + cmd.arg("-O2"); + cmd.arg("-ffp-contract=off"); + cmd.arg("-Wno-narrowing"); + cmd.arg("--target=aarch64-unknown-linux-gnu"); + cmd.arg("-c"); + cmd.arg(format!("mod_{i}.cpp")); + + let output = cmd.output(); + eprintln!( + "{}", + String::from_utf8_lossy(&output.as_ref().unwrap().stderr) + ); + assert!(output.unwrap().status.success()); + + Ok(()) + }) + .collect::>() + .unwrap(); + + let c_filename = format!("c_programs/main.cpp"); + let mut file = File::create(&c_filename).unwrap(); + write_main_cpp( + &mut file, c_target, - &["arm_neon.h", "arm_acle.h", "arm_fp16.h"], - &build_notices("// "), - &[POLY128_OSTREAM_DEF], - ); + POLY128_OSTREAM_DEF, + self.intrinsics.iter().map(|i| i.name.as_str()), + ) + .unwrap(); - match compiler { - None => true, - Some(compiler) => compile_c_arm( - intrinsics_name_list.unwrap().as_slice(), - compiler, - target, - cxx_toolchain_dir, - ), + let mut cmd = std::process::Command::new("clang++"); + cmd.current_dir("c_programs"); + + cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3"); + cmd.arg("-O2"); + cmd.arg("-ffp-contract=off"); + cmd.arg("-Wno-narrowing"); + cmd.arg("--target=aarch64-unknown-linux-gnu"); + cmd.arg(format!("main.cpp")); + for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) { + cmd.arg(format!("mod_{i}.o")); } + cmd.args(&["-o", "intrinsic-test-programs"]); + + let output = cmd.output(); + eprintln!( + "{}", + String::from_utf8_lossy(&output.as_ref().unwrap().stderr) + ); + assert!(output.unwrap().status.success()); + + // match compiler { + // None => true, + // Some(compiler) => compile_c_arm( + // intrinsics_name_list.unwrap().as_slice(), + // compiler, + // target, + // cxx_toolchain_dir, + // ), + // } + + true } fn build_rust_file(&self) -> bool { diff --git a/crates/intrinsic-test/src/common/compare.rs b/crates/intrinsic-test/src/common/compare.rs index 6a76927ddb..c5a8a7ad87 100644 --- a/crates/intrinsic-test/src/common/compare.rs +++ b/crates/intrinsic-test/src/common/compare.rs @@ -15,7 +15,9 @@ pub fn compare_outputs(intrinsic_name_list: &Vec, runner: &str, target: .par_iter() .filter_map(|intrinsic_name| { let c = runner_command(runner) - .arg(format!("./c_programs/{intrinsic_name}")) + .arg("intrinsic-test-programs") + .arg(intrinsic_name) + .current_dir("c_programs") .output(); let rust = runner_command(runner) diff --git a/crates/intrinsic-test/src/common/gen_c.rs b/crates/intrinsic-test/src/common/gen_c.rs index aed99609e5..80575bafd0 100644 --- a/crates/intrinsic-test/src/common/gen_c.rs +++ b/crates/intrinsic-test/src/common/gen_c.rs @@ -42,7 +42,7 @@ pub fn generate_c_test_loop( passes: u32, ) -> std::io::Result<()> { let body_indentation = indentation.nested(); - write!( + writeln!( w, "{indentation}for (int i=0; i<{passes}; i++) {{\n\ {loaded_args}\ @@ -89,20 +89,40 @@ pub fn generate_c_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>( } // Compiles C test programs using specified compiler -pub fn create_c_test_program( +pub fn create_c_test_function( w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, - header_files: &[&str], - _target: &str, - c_target: &str, - notices: &str, - arch_specific_definitions: &[&str], ) -> std::io::Result<()> { let indentation = Indentation::default(); - write!(w, "{notices}")?; + writeln!(w, "int run_{}() {{", intrinsic.name())?; + + // Define the arrays of arguments. + let arguments = intrinsic.arguments(); + arguments.gen_arglists_c(w, indentation.nested(), PASSES)?; - for header in header_files { + generate_c_constraint_blocks( + w, + intrinsic, + indentation.nested(), + &mut arguments.iter().rev().filter(|&i| i.has_constraint()), + Default::default(), + )?; + + writeln!(w, " return 0;")?; + writeln!(w, "}}")?; + + Ok(()) +} + +pub fn write_mod_cpp( + w: &mut impl std::io::Write, + notice: &str, + intrinsics: &[impl IntrinsicDefinition], +) -> std::io::Result<()> { + write!(w, "{notice}")?; + + for header in ["arm_neon.h", "arm_acle.h", "arm_fp16.h"] { writeln!(w, "#include <{header}>")?; } @@ -121,6 +141,38 @@ template T1 cast(T2 x) {{ return ret; }} +std::ostream& operator<<(std::ostream& os, float16_t value); +std::ostream& operator<<(std::ostream& os, poly128_t value); +"# + )?; + + for intrinsic in intrinsics { + create_c_test_function(w, intrinsic)?; + } + + Ok(()) +} + +pub fn write_main_cpp<'a>( + w: &mut impl std::io::Write, + architecture: &str, + arch_specific_definitions: &str, + intrinsics: impl Iterator + Clone, +) -> std::io::Result<()> { + writeln!(w, "#include ")?; + writeln!(w, "#include ")?; + + for header in ["arm_neon.h", "arm_acle.h", "arm_fp16.h"] { + writeln!(w, "#include <{header}>")?; + } + + writeln!( + w, + r#" +#include +#include +#include + std::ostream& operator<<(std::ostream& os, float16_t value) {{ uint16_t temp = 0; memcpy(&temp, &value, sizeof(float16_t)); @@ -132,28 +184,32 @@ std::ostream& operator<<(std::ostream& os, float16_t value) {{ "# )?; - let arch_identifier = c_target; - writeln!(w, "#ifdef __{arch_identifier}__")?; - for def in arch_specific_definitions { - writeln!(w, "{def}")?; - } + writeln!(w, "#ifdef __{architecture}__")?; + writeln!(w, "{arch_specific_definitions }")?; writeln!(w, "#endif")?; + for intrinsic in intrinsics.clone() { + writeln!(w, "extern int run_{intrinsic}(void);")?; + } + writeln!(w, "int main(int argc, char **argv) {{")?; + writeln!(w, " std::string intrinsic_name = argv[1];")?; - // Define the arrays of arguments. - let arguments = intrinsic.arguments(); - arguments.gen_arglists_c(w, indentation.nested(), PASSES)?; + writeln!(w, " if (false) {{")?; - generate_c_constraint_blocks( + for intrinsic in intrinsics { + writeln!(w, " }} else if (intrinsic_name == \"{intrinsic}\") {{")?; + writeln!(w, " return run_{intrinsic}();")?; + } + + writeln!(w, " }} else {{")?; + writeln!( w, - intrinsic, - indentation.nested(), - &mut arguments.iter().rev().filter(|&i| i.has_constraint()), - Default::default(), + " std::cerr << \"Unknown command: \" << intrinsic_name << \"\\n\";" )?; + writeln!(w, " return -1;")?; + writeln!(w, " }}")?; - writeln!(w, " return 0;")?; writeln!(w, "}}")?; Ok(()) diff --git a/crates/intrinsic-test/src/common/write_file.rs b/crates/intrinsic-test/src/common/write_file.rs index a52b1b89dc..d919534e59 100644 --- a/crates/intrinsic-test/src/common/write_file.rs +++ b/crates/intrinsic-test/src/common/write_file.rs @@ -1,41 +1,40 @@ -use std::fs::File; +// use std::fs::File; +// +// use super::intrinsic::IntrinsicDefinition; +// use super::intrinsic_helpers::IntrinsicTypeDefinition; +// +// use rayon::prelude::*; -use super::intrinsic::IntrinsicDefinition; -use super::intrinsic_helpers::IntrinsicTypeDefinition; - -use rayon::prelude::*; - -pub fn write_c_testfiles<'a, T, I, E>( - intrinsics: I, - target: &str, - c_target: &str, - headers: &[&str], - notice: &str, - arch_specific_definitions: &[&str], -) -> std::io::Result> -where - T: IntrinsicTypeDefinition + Sized + 'a, - I: ParallelIterator, - E: IntrinsicDefinition + 'a, -{ - std::fs::create_dir_all("c_programs")?; - - intrinsics - .map(|intrinsic| { - let identifier = intrinsic.name().to_owned(); - let mut file = File::create(format!("c_programs/{identifier}.cpp")).unwrap(); - - crate::common::gen_c::create_c_test_program( - &mut file, - intrinsic, - headers, - target, - c_target, - notice, - arch_specific_definitions, - )?; - - Ok(identifier) - }) - .collect() -} +// pub fn write_c_testfiles<'a, T, I, E>( +// intrinsics: I, +// target: &str, +// c_target: &str, +// headers: &[&str], +// notice: &str, +// arch_specific_definitions: &[&str], +// ) -> std::io::Result> +// where +// T: IntrinsicTypeDefinition + Sized + 'a, +// I: ParallelIterator, +// E: IntrinsicDefinition + 'a, +// { +// std::fs::create_dir_all("c_programs")?; +// +// intrinsics +// .map(|intrinsic| { +// let identifier = intrinsic.name().to_owned(); +// let mut file = File::create(format!("c_programs/{identifier}.cpp")).unwrap(); +// +// crate::common::gen_c::create_c_test_function( +// &mut file, +// intrinsic, +// +// +// +// +// )?; +// +// Ok(identifier) +// }) +// .collect() +// } From 85aa7afdc9bd7e3f0592e6401328c6af510664ce Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 20:01:34 +0200 Subject: [PATCH 13/14] Combining C files is very effective --- Cargo.lock | 1 - Cargo.toml | 3 +- ci/run.sh | 4 +- crates/intrinsic-test/Cargo.toml | 1 - crates/intrinsic-test/src/arm/compile.rs | 62 ++++++-- crates/intrinsic-test/src/arm/mod.rs | 78 ++++------ crates/intrinsic-test/src/common/cli.rs | 9 +- crates/intrinsic-test/src/common/compare.rs | 9 +- crates/intrinsic-test/src/common/compile_c.rs | 146 +++++++++++------- crates/intrinsic-test/src/common/gen_c.rs | 40 ++--- crates/intrinsic-test/src/common/gen_rust.rs | 27 +++- crates/intrinsic-test/src/main.rs | 12 +- 12 files changed, 221 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d9cd1bf8f..ef24a790b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,7 +354,6 @@ dependencies = [ "csv", "diff", "itertools", - "lazy_static", "log", "pretty_env_logger", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 0db26f31a2..a2a7b57e98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,8 @@ members = [ "examples", ] exclude = [ - "crates/wasm-assert-instr-tests" + "crates/wasm-assert-instr-tests", + "rust_programs" ] [profile.release] diff --git a/ci/run.sh b/ci/run.sh index 8eadb9285c..42c7824c9e 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -169,7 +169,7 @@ esac # Arm specific case "${TARGET}" in aarch64-unknown-linux-gnu*|armv7-unknown-linux-gnueabihf*) - CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=warn \ + CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=trace \ cargo run "${INTRINSIC_TEST}" "${PROFILE}" \ --bin intrinsic-test -- intrinsics_data/arm_intrinsics.json \ --runner "${TEST_RUNNER}" \ @@ -179,7 +179,7 @@ case "${TARGET}" in ;; aarch64_be-unknown-linux-gnu*) - CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=warn \ + CPPFLAGS="${TEST_CPPFLAGS}" RUSTFLAGS="${HOST_RUSTFLAGS}" RUST_LOG=trace \ cargo run "${INTRINSIC_TEST}" "${PROFILE}" \ --bin intrinsic-test -- intrinsics_data/arm_intrinsics.json \ --runner "${TEST_RUNNER}" \ diff --git a/crates/intrinsic-test/Cargo.toml b/crates/intrinsic-test/Cargo.toml index 06051abc8d..2b2df2dacc 100644 --- a/crates/intrinsic-test/Cargo.toml +++ b/crates/intrinsic-test/Cargo.toml @@ -11,7 +11,6 @@ license = "MIT OR Apache-2.0" edition = "2024" [dependencies] -lazy_static = "1.4.0" serde = { version = "1", features = ["derive"] } serde_json = "1.0" csv = "1.1" diff --git a/crates/intrinsic-test/src/arm/compile.rs b/crates/intrinsic-test/src/arm/compile.rs index 8276cd87c1..32c7865f2e 100644 --- a/crates/intrinsic-test/src/arm/compile.rs +++ b/crates/intrinsic-test/src/arm/compile.rs @@ -1,11 +1,11 @@ use crate::common::compile_c::CompilationCommandBuilder; -use crate::common::gen_c::compile_c_programs; pub fn compile_c_arm( - intrinsics_name_list: &[String], compiler: &str, target: &str, cxx_toolchain_dir: Option<&str>, + inputs: &[String], + output: Option<&str>, ) -> bool { // -ffp-contract=off emulates Rust's approach of not fusing separate mul-add operations let mut command = CompilationCommandBuilder::new() @@ -49,16 +49,52 @@ pub fn compile_c_arm( command = command.add_extra_flag("-flax-vector-conversions"); } - let compiler_commands = intrinsics_name_list - .iter() - .map(|intrinsic_name| { - command - .clone() - .set_input_name(intrinsic_name) - .set_output_name(intrinsic_name) - .make_string() - }) - .collect::>(); + let mut command = command.into_command(); + command.command_mut().current_dir("c_programs"); - compile_c_programs(&compiler_commands) + for input in inputs { + assert!( + std::path::Path::new("c_programs").join(input).exists(), + "{}", + input + ); + } + command.command_mut().args(inputs); + + if let Some(output) = output { + trace!("running {compiler} to produce {output}"); + if output.ends_with(".o") { + command.command_mut().arg("-c"); + } + command.command_mut().args(["-o", output]); + } else { + trace!("running {compiler}"); + } + + trace!("running {compiler}\n{:?}", &command); + + if log::log_enabled!(log::Level::Trace) { + command.command_mut().stdout(std::process::Stdio::inherit()); + command.command_mut().stderr(std::process::Stdio::inherit()); + } + + let output = command.output(); + + trace!("{compiler} is done"); + + if let Ok(output) = output { + if output.status.success() { + true + } else { + error!( + "Failed to compile code for intrinsics: \n\nstdout:\n{}\n\nstderr:\n{}", + std::str::from_utf8(&output.stdout).unwrap_or(""), + std::str::from_utf8(&output.stderr).unwrap_or("") + ); + false + } + } else { + error!("Command failed: {output:#?}"); + false + } } diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 82ec318e12..622f57a039 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -8,6 +8,7 @@ mod intrinsic; mod json_parser; mod types; +use crate::arm::compile::compile_c_arm; use crate::common::SupportedArchitectureTest; use crate::common::cli::ProcessedCli; use crate::common::compare::compare_outputs; @@ -54,7 +55,7 @@ impl SupportedArchitectureTest for ArmArchitectureTest { } fn build_c_file(&self) -> bool { - let compiler = self.cli_options.cpp_compiler.as_deref(); + let compiler = self.cli_options.cpp_compiler.as_deref().unwrap(); let target = &self.cli_options.target; let cxx_toolchain_dir = self.cli_options.cxx_toolchain_dir.as_deref(); let c_target = "aarch64"; @@ -69,36 +70,24 @@ impl SupportedArchitectureTest for ArmArchitectureTest { .map(|(i, chunk)| { let c_filename = format!("c_programs/mod_{i}.cpp"); let mut file = File::create(&c_filename).unwrap(); - write_mod_cpp(&mut file, ¬ice, chunk).unwrap(); + write_mod_cpp(&mut file, notice, c_target, chunk).unwrap(); // compile this cpp file into a .o file - // clang++ -march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3 -O2 -ffp-contract=off -Wno-narrowing --target=aarch64-unknown-linux-gnu c_file_0.c -c - let mut cmd = std::process::Command::new("clang++"); - cmd.current_dir("c_programs"); - - cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3"); - cmd.arg("-O2"); - cmd.arg("-ffp-contract=off"); - cmd.arg("-Wno-narrowing"); - cmd.arg("--target=aarch64-unknown-linux-gnu"); - cmd.arg("-c"); - cmd.arg(format!("mod_{i}.cpp")); - - let output = cmd.output(); - eprintln!( - "{}", - String::from_utf8_lossy(&output.as_ref().unwrap().stderr) + compile_c_arm( + compiler, + target, + cxx_toolchain_dir, + &[format!("mod_{i}.cpp")], + Some(&format!("mod_{i}.o")), ); - assert!(output.unwrap().status.success()); Ok(()) }) .collect::>() .unwrap(); - let c_filename = format!("c_programs/main.cpp"); - let mut file = File::create(&c_filename).unwrap(); + let mut file = File::create("c_programs/main.cpp").unwrap(); write_main_cpp( &mut file, c_target, @@ -107,38 +96,18 @@ impl SupportedArchitectureTest for ArmArchitectureTest { ) .unwrap(); - let mut cmd = std::process::Command::new("clang++"); - cmd.current_dir("c_programs"); - - cmd.arg("-march=armv8.6-a+crypto+crc+dotprod+fp16+faminmax+lut+sha3"); - cmd.arg("-O2"); - cmd.arg("-ffp-contract=off"); - cmd.arg("-Wno-narrowing"); - cmd.arg("--target=aarch64-unknown-linux-gnu"); - cmd.arg(format!("main.cpp")); + let mut inputs = vec![format!("main.cpp")]; for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) { - cmd.arg(format!("mod_{i}.o")); + inputs.push(format!("mod_{i}.o")); } - cmd.args(&["-o", "intrinsic-test-programs"]); - let output = cmd.output(); - eprintln!( - "{}", - String::from_utf8_lossy(&output.as_ref().unwrap().stderr) - ); - assert!(output.unwrap().status.success()); - - // match compiler { - // None => true, - // Some(compiler) => compile_c_arm( - // intrinsics_name_list.unwrap().as_slice(), - // compiler, - // target, - // cxx_toolchain_dir, - // ), - // } - - true + compile_c_arm( + compiler, + target, + cxx_toolchain_dir, + &inputs, + Some("intrinsic-test-programs"), + ) } fn build_rust_file(&self) -> bool { @@ -171,6 +140,12 @@ impl SupportedArchitectureTest for ArmArchitectureTest { let toolchain = self.cli_options.toolchain.as_deref(); let linker = self.cli_options.linker.as_deref(); + warn!( + "available parallelism: {:?} {}", + std::thread::available_parallelism(), + rayon::current_num_threads(), + ); + let notice = &build_notices("// "); self.intrinsics .par_chunks(chunk_size) @@ -178,7 +153,10 @@ impl SupportedArchitectureTest for ArmArchitectureTest { .map(|(i, chunk)| { use std::io::Write; + dbg!(chunk_size, chunk.len()); + let rust_filename = format!("rust_programs/src/mod_{i}.rs"); + trace!("generating `{rust_filename}`"); let mut file = File::create(rust_filename).unwrap(); write!(file, "{notice}")?; diff --git a/crates/intrinsic-test/src/common/cli.rs b/crates/intrinsic-test/src/common/cli.rs index a5277c2677..f50034ebf8 100644 --- a/crates/intrinsic-test/src/common/cli.rs +++ b/crates/intrinsic-test/src/common/cli.rs @@ -90,11 +90,10 @@ impl ProcessedCli { (None, None) } else { ( - Some( - cli_options - .toolchain - .map_or_else(String::new, |t| format!("+{t}")), - ), + match cli_options.toolchain { + Some(t) => Some(format!("+{t}")), + None => Some(String::new()), // NOTE this is confusing + }, Some(cli_options.cppcompiler), ) }; diff --git a/crates/intrinsic-test/src/common/compare.rs b/crates/intrinsic-test/src/common/compare.rs index c5a8a7ad87..4099d7420d 100644 --- a/crates/intrinsic-test/src/common/compare.rs +++ b/crates/intrinsic-test/src/common/compare.rs @@ -23,7 +23,6 @@ pub fn compare_outputs(intrinsic_name_list: &Vec, runner: &str, target: let rust = runner_command(runner) .arg(format!("target/{target}/release/intrinsic-test-programs")) .arg(intrinsic_name) - .current_dir("rust_programs") .output(); let (c, rust) = match (c, rust) { @@ -33,19 +32,19 @@ pub fn compare_outputs(intrinsic_name_list: &Vec, runner: &str, target: if !c.status.success() { error!( - "Failed to run C program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}", + "Failed to run C program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}", stdout = std::str::from_utf8(&c.stdout).unwrap_or(""), stderr = std::str::from_utf8(&c.stderr).unwrap_or(""), - ); + ); return Some(FailureReason::RunC(intrinsic_name.clone())); } if !rust.status.success() { error!( - "Failed to run Rust program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}", + "Failed to run Rust program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}", stdout = String::from_utf8_lossy(&rust.stdout), stderr = String::from_utf8_lossy(&rust.stderr), - ); + ); return Some(FailureReason::RunRust(intrinsic_name.clone())); } diff --git a/crates/intrinsic-test/src/common/compile_c.rs b/crates/intrinsic-test/src/common/compile_c.rs index aebb7b111e..769326ef03 100644 --- a/crates/intrinsic-test/src/common/compile_c.rs +++ b/crates/intrinsic-test/src/common/compile_c.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + #[derive(Clone)] pub struct CompilationCommandBuilder { compiler: String, @@ -8,7 +10,6 @@ pub struct CompilationCommandBuilder { include_paths: Vec, project_root: Option, output: String, - input: String, linker: Option, extra_flags: Vec, } @@ -24,7 +25,6 @@ impl CompilationCommandBuilder { include_paths: Vec::new(), project_root: None, output: String::new(), - input: String::new(), linker: None, extra_flags: Vec::new(), } @@ -71,18 +71,6 @@ impl CompilationCommandBuilder { self } - /// The name of the output executable, without any suffixes - pub fn set_output_name(mut self, path: &str) -> Self { - self.output = path.to_string(); - self - } - - /// The name of the input C file, without any suffixes - pub fn set_input_name(mut self, path: &str) -> Self { - self.input = path.to_string(); - self - } - pub fn set_linker(mut self, linker: String) -> Self { self.linker = Some(linker); self @@ -99,56 +87,102 @@ impl CompilationCommandBuilder { } } +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum CompilationCommand { + Simple(std::process::Command), + CustomLinker { + cmd: std::process::Command, + linker: std::process::Command, + cleanup: PathBuf, + }, +} + +impl CompilationCommand { + pub fn command_mut(&mut self) -> &mut std::process::Command { + match self { + CompilationCommand::Simple(command) => command, + CompilationCommand::CustomLinker { cmd, .. } => cmd, + } + } + + pub fn output(self) -> std::io::Result { + match self { + CompilationCommand::Simple(mut cmd) => cmd.output(), + CompilationCommand::CustomLinker { + mut cmd, + mut linker, + cleanup, + } => { + let output = cmd.output()?; + + linker.current_dir("c_programs"); + + if log::log_enabled!(log::Level::Trace) { + linker.stdout(std::process::Stdio::inherit()); + linker.stderr(std::process::Stdio::inherit()); + } + + if let Err(e) = linker.output() { + panic!( + "Failed running custom linker {:?}:\n{e:?}", + linker.get_program(), + ); + } + if cleanup.exists() { + std::fs::remove_file(cleanup)?; + } + + Ok(output) + } + } + } +} + impl CompilationCommandBuilder { - pub fn make_string(self) -> String { - let arch_flags = self.arch_flags.join("+"); - let flags = std::env::var("CPPFLAGS").unwrap_or("".into()); + pub fn into_command(self) -> CompilationCommand { let project_root = self.project_root.unwrap_or_default(); let project_root_str = project_root.as_str(); - let mut output = self.output.clone(); - if self.linker.is_some() { - output += ".o" - }; - let mut command = format!( - "{} {flags} -march={arch_flags} \ - -O{} \ - -o {project_root}/{} \ - {project_root}/{}.cpp", - self.compiler, self.optimization, output, self.input, - ); - - command = command + " " + self.extra_flags.join(" ").as_str(); + + let mut cmd = std::process::Command::new(self.compiler); + + let flags = std::env::var("CPPFLAGS").unwrap_or("".into()); + cmd.args(flags.split_whitespace()); + + cmd.arg(format!("-march={}", self.arch_flags.join("+"))); + + cmd.arg(format!("-O{}", self.optimization)); + + cmd.args(self.extra_flags); if let Some(target) = &self.target { - command = command + " --target=" + target; + cmd.arg(format!("--target={target}")); } if let (Some(linker), Some(cxx_toolchain_dir)) = (&self.linker, &self.cxx_toolchain_dir) { - let include_args = self - .include_paths - .iter() - .map(|path| "--include-directory=".to_string() + cxx_toolchain_dir + path) - .collect::>() - .join(" "); - - command = command - + " -c " - + include_args.as_str() - + " && " - + linker - + " " - + project_root_str - + "/" - + &output - + " -o " - + project_root_str - + "/" - + &self.output - + " && rm " - + project_root_str - + "/" - + &output; + cmd.arg("-c"); + cmd.args( + self.include_paths + .iter() + .map(|path| "--include-directory=".to_string() + cxx_toolchain_dir + path), + ); + + let output = "dummy_value"; + let mut linker_cmd = std::process::Command::new(linker); + linker_cmd.arg(format!("{project_root_str}/{output}")); + + linker_cmd.arg("-o"); + linker_cmd.arg(format!("{project_root_str}/{}", self.output)); + + let remove_path = PathBuf::from(format!("{project_root_str}/{output}")); + + CompilationCommand::CustomLinker { + cmd, + linker: linker_cmd, + cleanup: remove_path, + } + } else { + CompilationCommand::Simple(cmd) } - command } } diff --git a/crates/intrinsic-test/src/common/gen_c.rs b/crates/intrinsic-test/src/common/gen_c.rs index 80575bafd0..aa587f71b3 100644 --- a/crates/intrinsic-test/src/common/gen_c.rs +++ b/crates/intrinsic-test/src/common/gen_c.rs @@ -1,6 +1,3 @@ -use rayon::prelude::*; -use std::process::Command; - use super::argument::Argument; use super::indentation::Indentation; use super::intrinsic::IntrinsicDefinition; @@ -9,31 +6,6 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition; // The number of times each intrinsic will be called. const PASSES: u32 = 20; -pub fn compile_c_programs(compiler_commands: &[String]) -> bool { - compiler_commands - .par_iter() - .map(|compiler_command| { - let output = Command::new("sh").arg("-c").arg(compiler_command).output(); - if let Ok(output) = output { - if output.status.success() { - true - } else { - error!( - "Failed to compile code for intrinsics: \n\nstdout:\n{}\n\nstderr:\n{}", - std::str::from_utf8(&output.stdout).unwrap_or(""), - std::str::from_utf8(&output.stderr).unwrap_or("") - ); - false - } - } else { - error!("Command failed: {output:#?}"); - false - } - }) - .find_any(|x| !x) - .is_none() -} - pub fn generate_c_test_loop( w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, @@ -118,6 +90,7 @@ pub fn create_c_test_function( pub fn write_mod_cpp( w: &mut impl std::io::Write, notice: &str, + architecture: &str, intrinsics: &[impl IntrinsicDefinition], ) -> std::io::Result<()> { write!(w, "{notice}")?; @@ -142,10 +115,19 @@ template T1 cast(T2 x) {{ }} std::ostream& operator<<(std::ostream& os, float16_t value); -std::ostream& operator<<(std::ostream& os, poly128_t value); + + + "# )?; + writeln!(w, "#ifdef __{architecture}__")?; + writeln!( + w, + "std::ostream& operator<<(std::ostream& os, poly128_t value);" + )?; + writeln!(w, "#endif")?; + for intrinsic in intrinsics { create_c_test_function(w, intrinsic)?; } diff --git a/crates/intrinsic-test/src/common/gen_rust.rs b/crates/intrinsic-test/src/common/gen_rust.rs index 142e1d7ace..13107af6ad 100644 --- a/crates/intrinsic-test/src/common/gen_rust.rs +++ b/crates/intrinsic-test/src/common/gen_rust.rs @@ -57,12 +57,17 @@ pub fn write_main_rs<'a>( writeln!(w, "#![feature(f16)]")?; writeln!(w, "#![allow(unused)]")?; + // NOTE: on CI, cargo is + writeln!(w, "#![allow(non_upper_case_globals)]")?; + writeln!(w, "#![allow(non_camel_case_types)]")?; + writeln!(w, "#![allow(non_snake_case)]")?; + writeln!(w, "{cfg}")?; writeln!(w, "{definitions}")?; writeln!(w, "use core_arch::arch::{architecture}::*;")?; - for module in 0..available_parallelism { + for module in 0..Ord::min(available_parallelism, intrinsics.clone().count()) { writeln!(w, "mod mod_{module};")?; writeln!(w, "use mod_{module}::*;")?; } @@ -90,13 +95,15 @@ pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Opti /* If there has been a linker explicitly set from the command line then * we want to set it via setting it in the RUSTFLAGS*/ + trace!("Building cargo command"); + let mut cargo_command = Command::new("cargo"); cargo_command.current_dir("rust_programs"); - if let Some(toolchain) = toolchain { - if !toolchain.is_empty() { - cargo_command.arg(toolchain); - } + if let Some(toolchain) = toolchain + && !toolchain.is_empty() + { + cargo_command.arg(toolchain); } cargo_command.args(["build", "--target", target, "--release"]); @@ -110,7 +117,16 @@ pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Opti } cargo_command.env("RUSTFLAGS", rust_flags); + + trace!("running cargo"); + + if log::log_enabled!(log::Level::Trace) { + cargo_command.stdout(std::process::Stdio::inherit()); + cargo_command.stderr(std::process::Stdio::inherit()); + } + let output = cargo_command.output(); + trace!("cargo is done"); if let Ok(output) = output { if output.status.success() { @@ -200,6 +216,7 @@ pub fn create_rust_test_module( w: &mut impl std::io::Write, intrinsic: &dyn IntrinsicDefinition, ) -> std::io::Result<()> { + trace!("generating `{}`", intrinsic.name()); let indentation = Indentation::default(); writeln!(w, "pub fn run_{}() {{", intrinsic.name())?; diff --git a/crates/intrinsic-test/src/main.rs b/crates/intrinsic-test/src/main.rs index 925917be66..cc7d85c040 100644 --- a/crates/intrinsic-test/src/main.rs +++ b/crates/intrinsic-test/src/main.rs @@ -13,6 +13,12 @@ fn main() { let args: Cli = clap::Parser::parse(); let processed_cli_options = ProcessedCli::new(args); + warn!( + "available parallelism: {:?} {}", + std::thread::available_parallelism(), + rayon::current_num_threads(), + ); + let test_environment_result: Option> = match processed_cli_options.target.as_str() { "aarch64-unknown-linux-gnu" @@ -30,15 +36,15 @@ fn main() { let test_environment = test_environment_result.unwrap(); - info!("building Rust binaries"); + warn!("building Rust binaries"); if !test_environment.build_rust_file() { std::process::exit(3); } - info!("building C binaries"); + warn!("building C binaries"); if !test_environment.build_c_file() { std::process::exit(2); } - info!("comparing outputs"); + warn!("comparing outputs"); if !test_environment.compare_outputs() { std::process::exit(1); } From 3fceb9e32fb40b3bdd6a2b3b47f83a5e54310958 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 9 Jul 2025 22:41:12 +0200 Subject: [PATCH 14/14] a hopeless attempt --- .../aarch64_be-unknown-linux-gnu/Dockerfile | 2 +- crates/intrinsic-test/src/arm/compile.rs | 8 ++- crates/intrinsic-test/src/arm/mod.rs | 55 +++++++++++++++---- crates/intrinsic-test/src/common/compile_c.rs | 42 ++++++-------- crates/intrinsic-test/src/main.rs | 8 +-- 5 files changed, 70 insertions(+), 45 deletions(-) diff --git a/ci/docker/aarch64_be-unknown-linux-gnu/Dockerfile b/ci/docker/aarch64_be-unknown-linux-gnu/Dockerfile index 74f770556d..3eba02766b 100644 --- a/ci/docker/aarch64_be-unknown-linux-gnu/Dockerfile +++ b/ci/docker/aarch64_be-unknown-linux-gnu/Dockerfile @@ -24,7 +24,7 @@ RUN mkdir /toolchains && mv "./${TOOLCHAIN}" /toolchains ENV AARCH64_BE_TOOLCHAIN="/toolchains/${TOOLCHAIN}" ENV AARCH64_BE_LIBC="${AARCH64_BE_TOOLCHAIN}/aarch64_be-none-linux-gnu/libc" -ENV CARGO_TARGET_AARCH64_BE_UNKNOWN_LINUX_GNU_LINKER="${AARCH64_BE_TOOLCHAIN}/bin/aarch64_be-none-linux-gnu-gcc" +ENV CARGO_TARGET_AARCH64_BE_UNKNOWN_LINUX_GNU_LINKER="${AARCH64_BE_TOOLCHAIN}/bin/aarch64_be-none-linux-gnu-g++" ENV CARGO_TARGET_AARCH64_BE_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64_be -cpu max -L ${AARCH64_BE_LIBC}" ENV OBJDUMP="${AARCH64_BE_TOOLCHAIN}/bin/aarch64_be-none-linux-gnu-objdump" ENV STDARCH_TEST_SKIP_FEATURE=tme diff --git a/crates/intrinsic-test/src/arm/compile.rs b/crates/intrinsic-test/src/arm/compile.rs index 32c7865f2e..79f8d337c3 100644 --- a/crates/intrinsic-test/src/arm/compile.rs +++ b/crates/intrinsic-test/src/arm/compile.rs @@ -1,4 +1,4 @@ -use crate::common::compile_c::CompilationCommandBuilder; +use crate::common::compile_c::{CompilationCommand, CompilationCommandBuilder}; pub fn compile_c_arm( compiler: &str, @@ -67,12 +67,14 @@ pub fn compile_c_arm( command.command_mut().arg("-c"); } command.command_mut().args(["-o", output]); + + if let CompilationCommand::CustomLinker { linker, .. } = &mut command { + linker.arg(format!("c_programs/{output}")); + } } else { trace!("running {compiler}"); } - trace!("running {compiler}\n{:?}", &command); - if log::log_enabled!(log::Level::Trace) { command.command_mut().stdout(std::process::Stdio::inherit()); command.command_mut().stderr(std::process::Stdio::inherit()); diff --git a/crates/intrinsic-test/src/arm/mod.rs b/crates/intrinsic-test/src/arm/mod.rs index 622f57a039..34633a8a3c 100644 --- a/crates/intrinsic-test/src/arm/mod.rs +++ b/crates/intrinsic-test/src/arm/mod.rs @@ -96,18 +96,51 @@ impl SupportedArchitectureTest for ArmArchitectureTest { ) .unwrap(); - let mut inputs = vec![format!("main.cpp")]; - for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) { - inputs.push(format!("mod_{i}.o")); + if let Some(linker) = &self.cli_options.linker { + compile_c_arm( + compiler, + target, + cxx_toolchain_dir, + &["main.cpp".to_string()], + Some("intrinsic-test-programs.o"), + ); + + let mut cmd = std::process::Command::new(linker); + cmd.current_dir("c_programs"); + + let mut inputs = vec![]; + for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) { + inputs.push(format!("mod_{i}.o")); + } + cmd.args(inputs); + + cmd.arg("intrinsic-test-programs.o"); + + cmd.arg("-o"); + cmd.arg("intrinsic-test-programs"); + + if log::log_enabled!(log::Level::Trace) { + cmd.stdout(std::process::Stdio::inherit()); + cmd.stderr(std::process::Stdio::inherit()); + } + + assert!(cmd.output().unwrap().status.success()); + } else { + let mut inputs = vec![format!("main.cpp")]; + for i in 0..Ord::min(available_parallelism, self.intrinsics.len()) { + inputs.push(format!("mod_{i}.o")); + } + + compile_c_arm( + compiler, + target, + cxx_toolchain_dir, + &inputs, + Some("intrinsic-test-programs"), + ); } - compile_c_arm( - compiler, - target, - cxx_toolchain_dir, - &inputs, - Some("intrinsic-test-programs"), - ) + true } fn build_rust_file(&self) -> bool { @@ -153,8 +186,6 @@ impl SupportedArchitectureTest for ArmArchitectureTest { .map(|(i, chunk)| { use std::io::Write; - dbg!(chunk_size, chunk.len()); - let rust_filename = format!("rust_programs/src/mod_{i}.rs"); trace!("generating `{rust_filename}`"); let mut file = File::create(rust_filename).unwrap(); diff --git a/crates/intrinsic-test/src/common/compile_c.rs b/crates/intrinsic-test/src/common/compile_c.rs index 769326ef03..dfb18693b3 100644 --- a/crates/intrinsic-test/src/common/compile_c.rs +++ b/crates/intrinsic-test/src/common/compile_c.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - #[derive(Clone)] pub struct CompilationCommandBuilder { compiler: String, @@ -9,7 +7,6 @@ pub struct CompilationCommandBuilder { optimization: String, include_paths: Vec, project_root: Option, - output: String, linker: Option, extra_flags: Vec, } @@ -24,7 +21,6 @@ impl CompilationCommandBuilder { optimization: "2".to_string(), include_paths: Vec::new(), project_root: None, - output: String::new(), linker: None, extra_flags: Vec::new(), } @@ -91,10 +87,10 @@ impl CompilationCommandBuilder { #[allow(clippy::large_enum_variant)] pub enum CompilationCommand { Simple(std::process::Command), + #[allow(unused)] CustomLinker { cmd: std::process::Command, linker: std::process::Command, - cleanup: PathBuf, }, } @@ -112,12 +108,9 @@ impl CompilationCommand { CompilationCommand::CustomLinker { mut cmd, mut linker, - cleanup, } => { let output = cmd.output()?; - linker.current_dir("c_programs"); - if log::log_enabled!(log::Level::Trace) { linker.stdout(std::process::Stdio::inherit()); linker.stderr(std::process::Stdio::inherit()); @@ -129,9 +122,9 @@ impl CompilationCommand { linker.get_program(), ); } - if cleanup.exists() { - std::fs::remove_file(cleanup)?; - } + // if cleanup.exists() { + // std::fs::remove_file(cleanup)?; + // } Ok(output) } @@ -141,8 +134,8 @@ impl CompilationCommand { impl CompilationCommandBuilder { pub fn into_command(self) -> CompilationCommand { - let project_root = self.project_root.unwrap_or_default(); - let project_root_str = project_root.as_str(); + // let project_root = self.project_root.unwrap_or_default(); + // let project_root_str = project_root.as_str(); let mut cmd = std::process::Command::new(self.compiler); @@ -159,7 +152,7 @@ impl CompilationCommandBuilder { cmd.arg(format!("--target={target}")); } - if let (Some(linker), Some(cxx_toolchain_dir)) = (&self.linker, &self.cxx_toolchain_dir) { + if let (Some(_linker), Some(cxx_toolchain_dir)) = (&self.linker, &self.cxx_toolchain_dir) { cmd.arg("-c"); cmd.args( self.include_paths @@ -167,20 +160,19 @@ impl CompilationCommandBuilder { .map(|path| "--include-directory=".to_string() + cxx_toolchain_dir + path), ); - let output = "dummy_value"; - let mut linker_cmd = std::process::Command::new(linker); - linker_cmd.arg(format!("{project_root_str}/{output}")); + // let linker_cmd = std::process::Command::new(linker); + // linker_cmd.arg(format!("{project_root_str}/{output}")); - linker_cmd.arg("-o"); - linker_cmd.arg(format!("{project_root_str}/{}", self.output)); + // let remove_path = PathBuf::new(); + // PathBuf::from(format!("{project_root_str}/{output}")); - let remove_path = PathBuf::from(format!("{project_root_str}/{output}")); + // CompilationCommand::CustomLinker { + // cmd, + // linker: linker_cmd, + // // cleanup: remove_path, + // } - CompilationCommand::CustomLinker { - cmd, - linker: linker_cmd, - cleanup: remove_path, - } + CompilationCommand::Simple(cmd) } else { CompilationCommand::Simple(cmd) } diff --git a/crates/intrinsic-test/src/main.rs b/crates/intrinsic-test/src/main.rs index cc7d85c040..d5504f13bc 100644 --- a/crates/intrinsic-test/src/main.rs +++ b/crates/intrinsic-test/src/main.rs @@ -36,14 +36,14 @@ fn main() { let test_environment = test_environment_result.unwrap(); - warn!("building Rust binaries"); - if !test_environment.build_rust_file() { - std::process::exit(3); - } warn!("building C binaries"); if !test_environment.build_c_file() { std::process::exit(2); } + warn!("building Rust binaries"); + if !test_environment.build_rust_file() { + std::process::exit(3); + } warn!("comparing outputs"); if !test_environment.compare_outputs() { std::process::exit(1);