diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 91039d0c8dcc1..60b1ff3e4413e 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1041,6 +1041,21 @@ impl Step for PlainSourceTarball { .env("RUSTC_BOOTSTRAP", "1") .current_dir(plain_dst_src); + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES + .iter() + .chain(build_helper::RUSTC_PGO_CRATES) + .map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); + for manifest_path in pkgs_for_pgo_training { + cmd.arg("--sync").arg(manifest_path); + } + let config = if !builder.config.dry_run() { t!(String::from_utf8(t!(cmd.output()).stdout)) } else { diff --git a/src/tools/build_helper/README.md b/src/tools/build_helper/README.md new file mode 100644 index 0000000000000..f81b631c3fdbf --- /dev/null +++ b/src/tools/build_helper/README.md @@ -0,0 +1 @@ +Types and functions shared across tools in this workspace. diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs index 2abda5d3ebf27..15807d1c0d8f8 100644 --- a/src/tools/build_helper/src/lib.rs +++ b/src/tools/build_helper/src/lib.rs @@ -1,5 +1,30 @@ +//! Types and functions shared across tools in this workspace. + pub mod ci; pub mod git; pub mod metrics; pub mod stage0_parser; pub mod util; + +/// The default set of crates for opt-dist to collect LLVM profiles. +pub const LLVM_PGO_CRATES: &[&str] = &[ + "syn-1.0.89", + "cargo-0.60.0", + "serde-1.0.136", + "ripgrep-13.0.0", + "regex-1.5.5", + "clap-3.1.6", + "hyper-0.14.18", +]; + +/// The default set of crates for opt-dist to collect rustc profiles. +pub const RUSTC_PGO_CRATES: &[&str] = &[ + "externs", + "ctfe-stress-5", + "cargo-0.60.0", + "token-stream-stress", + "match-stress", + "tuple-stress", + "diesel-1.4.8", + "bitmaps-3.1.0", +]; diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 09263dc645e2d..89f4d8957c8bf 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -3,30 +3,10 @@ use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; use anyhow::Context; +use build_helper::{LLVM_PGO_CRATES, RUSTC_PGO_CRATES}; use camino::{Utf8Path, Utf8PathBuf}; use humansize::BINARY; -const LLVM_PGO_CRATES: &[&str] = &[ - "syn-1.0.89", - "cargo-0.60.0", - "serde-1.0.136", - "ripgrep-13.0.0", - "regex-1.5.5", - "clap-3.1.6", - "hyper-0.14.18", -]; - -const RUSTC_PGO_CRATES: &[&str] = &[ - "externs", - "ctfe-stress-5", - "cargo-0.60.0", - "token-stream-stress", - "match-stress", - "tuple-stress", - "diesel-1.4.8", - "bitmaps-3.1.0", -]; - fn init_compiler_benchmarks( env: &Environment, profiles: &[&str], diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index efcd2a181ffa2..7b93b09dda24b 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -67,6 +67,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) //("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored //("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None), + ("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None), ("src/tools/x", &[], None), // tidy-alphabetical-end ]; @@ -142,6 +143,22 @@ const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[ // tidy-alphabetical-end ]; +const EXCEPTIONS_RUSTC_PERF: ExceptionList = &[ + // tidy-alphabetical-start + ("alloc-no-stdlib", "BSD-3-Clause"), + ("alloc-stdlib", "BSD-3-Clause"), + ("brotli", "BSD-3-Clause/MIT"), + ("brotli-decompressor", "BSD-3-Clause/MIT"), + ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), + ("inferno", "CDDL-1.0"), + ("instant", "BSD-3-Clause"), + ("ring", NON_STANDARD_LICENSE), // see EXCEPTIONS_NON_STANDARD_LICENSE_DEPS for more. + ("ryu", "Apache-2.0 OR BSL-1.0"), + ("snap", "BSD-3-Clause"), + ("subtle", "BSD-3-Clause"), + // tidy-alphabetical-end +]; + const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-start ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"), @@ -178,6 +195,20 @@ const EXCEPTIONS_UEFI_QEMU_TEST: ExceptionList = &[ ("r-efi", "MIT OR Apache-2.0 OR LGPL-2.1-or-later"), // LGPL is not acceptible, but we use it under MIT OR Apache-2.0 ]; +/// Placeholder for non-standard license file. +const NON_STANDARD_LICENSE: &str = "NON_STANDARD_LICENSE"; + +/// These dependencies have non-standard licenses but are genenrally permitted. +const EXCEPTIONS_NON_STANDARD_LICENSE_DEPS: &[&str] = &[ + // `ring` is included because it is an optional dependency of `hyper`, + // which is a training data in rustc-perf for optimized build. + // The license of it is generally `ISC AND MIT AND OpenSSL`, + // though the `package.license` field is not set. + // + // See https://github.com/briansmith/ring/issues/902 + "ring", +]; + /// These are the root crates that are part of the runtime. The licenses for /// these and all their dependencies *must not* be in the exception list. const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", "panic_unwind"]; @@ -610,6 +641,11 @@ fn check_license_exceptions(metadata: &Metadata, exceptions: &[(&str, &str)], ba for pkg in metadata.packages.iter().filter(|p| p.name == *name) { match &pkg.license { None => { + if *license == NON_STANDARD_LICENSE + && EXCEPTIONS_NON_STANDARD_LICENSE_DEPS.contains(&pkg.name.as_str()) + { + continue; + } tidy_error!( bad, "dependency exception `{}` does not declare a license expression", diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index ff71ca537256f..2118de5f20411 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -4,7 +4,11 @@ use std::fs; use std::path::Path; /// List of allowed sources for packages. -const ALLOWED_SOURCES: &[&str] = &["\"registry+https://github.com/rust-lang/crates.io-index\""]; +const ALLOWED_SOURCES: &[&str] = &[ + r#""registry+https://github.com/rust-lang/crates.io-index""#, + // This is `rust_team_data` used by `site` in src/tools/rustc-perf, + r#""git+https://github.com/rust-lang/team#a5260e76d3aa894c64c56e6ddc8545b9a98043ec""#, +]; /// Checks for external package sources. `root` is the path to the directory that contains the /// workspace `Cargo.toml`.