Merge branch 'altair' into participation-cache

.github/custom/config.toml

@@ -0,0 +1,3 @@
+# Custom Cargo config to be used for the udeps CI job
+[http]
+multiplexing = false

.github/workflows/test-suite.yml

@@ -12,7 +12,7 @@ env:
   # Deny warnings in CI
   RUSTFLAGS: "-D warnings"
   # The Nightly version used for cargo-udeps, might need updating from time to time.
-  PINNED_NIGHTLY: nightly-2021-03-01
+  PINNED_NIGHTLY: nightly-2021-06-09
 jobs:
   target-branch-check:
     name: target-branch-check
@@ -208,6 +208,10 @@ jobs:
       run: rustup toolchain install $PINNED_NIGHTLY
     - name: Install cargo-udeps
       run: cargo install cargo-udeps --locked
+    - name: Create Cargo config dir
+      run: mkdir -p .cargo
+    - name: Install custom Cargo config
+      run: cp -f .github/custom/config.toml .cargo/config.toml
     - name: Run cargo udeps to identify unused crates in the dependency graph
       run: make udeps
     env:

account_manager/src/validator/create.rs

@@ -236,7 +236,7 @@ pub fn cli_run<T: EthSpec>(
             .map_err(|e| {
                 format!(
                     "Error registering validator {}: {:?}",
-                    voting_pubkey.to_hex_string(),
+                    voting_pubkey.as_hex_string(),
                     e
                 )
             })?;
@@ -250,7 +250,7 @@ pub fn cli_run<T: EthSpec>(
             .build()
             .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
 
-        println!("{}/{}\t{}", i + 1, n, voting_pubkey.to_hex_string());
+        println!("{}/{}\t{}", i + 1, n, voting_pubkey.as_hex_string());
     }
 
     Ok(())

account_manager/src/validator/import.rs

@@ -242,7 +242,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
             .map_err(|e| {
                 format!(
                     "Error registering validator {}: {:?}",
-                    voting_pubkey.to_hex_string(),
+                    voting_pubkey.as_hex_string(),
                     e
                 )
             })?;

account_manager/src/validator/slashing_protection.rs

@@ -1,7 +1,7 @@
 use clap::{App, Arg, ArgMatches};
 use environment::Environment;
 use slashing_protection::{
-    interchange::Interchange, InterchangeImportOutcome, SlashingDatabase,
+    interchange::Interchange, InterchangeError, InterchangeImportOutcome, SlashingDatabase,
     SLASHING_PROTECTION_FILENAME,
 };
 use std::fs::File;
@@ -15,6 +15,8 @@ pub const EXPORT_CMD: &str = "export";
 pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
 pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
 
+pub const MINIFY_FLAG: &str = "minify";
+
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Import or export slashing protection data to or from another client")
@@ -26,6 +28,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .takes_value(true)
                         .value_name("FILE")
                         .help("The slashing protection interchange file to import (.json)"),
+                )
+                .arg(
+                    Arg::with_name(MINIFY_FLAG)
+                        .long(MINIFY_FLAG)
+                        .takes_value(true)
+                        .default_value("true")
+                        .possible_values(&["false", "true"])
+                        .help(
+                            "Minify the input file before processing. This is *much* faster, \
+                             but will not detect slashable data in the input.",
+                        ),
                 ),
         )
         .subcommand(
@@ -36,6 +49,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .takes_value(true)
                         .value_name("FILE")
                         .help("The filename to export the interchange file to"),
+                )
+                .arg(
+                    Arg::with_name(MINIFY_FLAG)
+                        .long(MINIFY_FLAG)
+                        .takes_value(true)
+                        .default_value("false")
+                        .possible_values(&["false", "true"])
+                        .help(
+                            "Minify the output file. This will make it smaller and faster to \
+                             import, but not faster to generate.",
+                        ),
                 ),
         )
 }
@@ -64,6 +88,7 @@ pub fn cli_run<T: EthSpec>(
     match matches.subcommand() {
         (IMPORT_CMD, Some(matches)) => {
             let import_filename: PathBuf = clap_utils::parse_required(&matches, IMPORT_FILE_ARG)?;
+            let minify: bool = clap_utils::parse_required(&matches, MINIFY_FLAG)?;
             let import_file = File::open(&import_filename).map_err(|e| {
                 format!(
                     "Unable to open import file at {}: {:?}",
@@ -72,8 +97,18 @@ pub fn cli_run<T: EthSpec>(
                 )
             })?;
 
-            let interchange = Interchange::from_json_reader(&import_file)
+            eprint!("Loading JSON file into memory & deserializing");
+            let mut interchange = Interchange::from_json_reader(&import_file)
                 .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
+            eprintln!(" [done].");
+
+            if minify {
+                eprint!("Minifying input file for faster loading");
+                interchange = interchange
+                    .minify()
+                    .map_err(|e| format!("Minification failed: {:?}", e))?;
+                eprintln!(" [done].");
+            }
 
             let slashing_protection_database =
                 SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
@@ -84,16 +119,6 @@ pub fn cli_run<T: EthSpec>(
                     )
                 })?;
 
-            let outcomes = slashing_protection_database
-                .import_interchange_info(interchange, genesis_validators_root)
-                .map_err(|e| {
-                    format!(
-                        "Error during import: {:?}\n\
-                         IT IS NOT SAFE TO START VALIDATING",
-                        e
-                    )
-                })?;
-
             let display_slot = |slot: Option<Slot>| {
                 slot.map_or("none".to_string(), |slot| format!("{}", slot.as_u64()))
             };
@@ -105,48 +130,77 @@ pub fn cli_run<T: EthSpec>(
                 (source, target) => format!("{}=>{}", display_epoch(source), display_epoch(target)),
             };
 
-            let mut num_failed = 0;
-
-            for outcome in &outcomes {
-                match outcome {
-                    InterchangeImportOutcome::Success { pubkey, summary } => {
-                        eprintln!("- {:?} SUCCESS min block: {}, max block: {}, min attestation: {}, max attestation: {}",
-                            pubkey,
-                            display_slot(summary.min_block_slot),
-                            display_slot(summary.max_block_slot),
-                            display_attestation(summary.min_attestation_source, summary.min_attestation_target),
-                            display_attestation(summary.max_attestation_source,
-                            summary.max_attestation_target),
-                        );
+            match slashing_protection_database
+                .import_interchange_info(interchange, genesis_validators_root)
+            {
+                Ok(outcomes) => {
+                    eprintln!("All records imported successfully:");
+                    for outcome in &outcomes {
+                        match outcome {
+                            InterchangeImportOutcome::Success { pubkey, summary } => {
+                                eprintln!("- {:?}", pubkey);
+                                eprintln!(
+                                    "    - min block: {}",
+                                    display_slot(summary.min_block_slot)
+                                );
+                                eprintln!(
+                                    "    - min attestation: {}",
+                                    display_attestation(
+                                        summary.min_attestation_source,
+                                        summary.min_attestation_target
+                                    )
+                                );
+                                eprintln!(
+                                    "    - max attestation: {}",
+                                    display_attestation(
+                                        summary.max_attestation_source,
+                                        summary.max_attestation_target
+                                    )
+                                );
+                            }
+                            InterchangeImportOutcome::Failure { pubkey, error } => {
+                                panic!(
+                                    "import should be atomic, but key {:?} was imported despite error: {:?}",
+                                    pubkey, error
+                                );
+                            }
+                        }
                     }
-                    InterchangeImportOutcome::Failure { pubkey, error } => {
-                        eprintln!("- {:?} ERROR: {:?}", pubkey, error);
-                        num_failed += 1;
+                }
+                Err(InterchangeError::AtomicBatchAborted(outcomes)) => {
+                    eprintln!("ERROR, slashable data in input:");
+                    for outcome in &outcomes {
+                        if let InterchangeImportOutcome::Failure { pubkey, error } = outcome {
+                            eprintln!("- {:?}", pubkey);
+                            eprintln!("    - error: {:?}", error);
+                        }
                     }
+                    return Err(
+                        "ERROR: import aborted due to slashable data, see above.\n\
+                         Please see https://lighthouse-book.sigmaprime.io/slashing-protection.html#slashable-data-in-import\n\
+                         IT IS NOT SAFE TO START VALIDATING".to_string()
+                    );
+                }
+                Err(e) => {
+                    return Err(format!(
+                        "Fatal error during import: {:?}\n\
+                         IT IS NOT SAFE TO START VALIDATING",
+                        e
+                    ));
                 }
             }
 
-            if num_failed == 0 {
-                eprintln!("Import completed successfully.");
-                eprintln!(
-                    "Please double-check that the minimum and maximum blocks and slots above \
-                     match your expectations."
-                );
-            } else {
-                eprintln!(
-                    "WARNING: history was NOT imported for {} of {} records",
-                    num_failed,
-                    outcomes.len()
-                );
-                eprintln!("IT IS NOT SAFE TO START VALIDATING");
-                eprintln!("Please see https://lighthouse-book.sigmaprime.io/slashing-protection.html#slashable-data-in-import");
-                return Err("Partial import".to_string());
-            }
+            eprintln!("Import completed successfully.");
+            eprintln!(
+                "Please double-check that the minimum and maximum blocks and attestations above \
+                 match your expectations."
+            );
 
             Ok(())
         }
         (EXPORT_CMD, Some(matches)) => {
             let export_filename: PathBuf = clap_utils::parse_required(&matches, EXPORT_FILE_ARG)?;
+            let minify: bool = clap_utils::parse_required(&matches, MINIFY_FLAG)?;
 
             if !slashing_protection_db_path.exists() {
                 return Err(format!(
@@ -164,10 +218,17 @@ pub fn cli_run<T: EthSpec>(
                     )
                 })?;
 
-            let interchange = slashing_protection_database
+            let mut interchange = slashing_protection_database
                 .export_interchange_info(genesis_validators_root)
                 .map_err(|e| format!("Error during export: {:?}", e))?;
 
+            if minify {
+                eprintln!("Minifying output file");
+                interchange = interchange
+                    .minify()
+                    .map_err(|e| format!("Unable to minify output: {:?}", e))?;
+            }
+
             let output_file = File::create(export_filename)
                 .map_err(|e| format!("Error creating output file: {:?}", e))?;