Doppelganger detection (sigp#2230)

## Issue Addressed

Resolves sigp#2069 

## Proposed Changes

- Adds a `--doppelganger-detection` flag
- Adds a `lighthouse/seen_validators` endpoint, which will make it so the lighthouse VC is not interopable with other client beacon nodes if the `--doppelganger-detection` flag is used, but hopefully this will become standardized. Relevant Eth2 API repo issue: ethereum/beacon-APIs#64
- If the `--doppelganger-detection` flag is used, the VC will wait until the beacon node is synced, and then wait an additional 2 epochs. The reason for this is to make sure the beacon node is able to subscribe to the subnets our validators should be attesting on. I think an alternative would be to have the beacon node subscribe to all subnets for 2+ epochs on startup by default.

## Additional Info

I'd like to add tests and would appreciate feedback. 

TODO:  handle validators started via the API, potentially make this default behavior

Co-authored-by: realbigsean <seananderson33@gmail.com>
Co-authored-by: Michael Sproul <michael@sigmaprime.io>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

.github/workflows/test-suite.yml

@@ -146,6 +146,28 @@ jobs:
         run: sudo npm install -g ganache-cli
       - name: Run the syncing simulator
         run: cargo run --release --bin simulator syncing-sim
+  doppelganger-protection-test:
+      name: doppelganger-protection-test
+      runs-on: ubuntu-latest
+      needs: cargo-fmt
+      steps:
+          - uses: actions/checkout@v1
+          - name: Get latest version of stable Rust
+            run: rustup update stable
+          - name: Install ganache-cli
+            run: sudo npm install -g ganache-cli
+          - name: Install lighthouse and lcli
+            run: |
+                make
+                make install-lcli
+          - name: Run the doppelganger protection success test script
+            run: |
+                cd scripts/tests
+                ./doppelganger_protection.sh success
+          - name: Run the doppelganger protection failure test script
+            run: |
+                cd scripts/tests
+                ./doppelganger_protection.sh failure
   check-benchmarks:
     name: check-benchmarks
     runs-on: ubuntu-latest

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -3443,6 +3443,28 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut file = std::fs::File::create(file_name).unwrap();
         self.dump_as_dot(&mut file);
     }
+
+    /// Checks if attestations have been seen from the given `validator_index` at the
+    /// given `epoch`.
+    pub fn validator_seen_at_epoch(&self, validator_index: usize, epoch: Epoch) -> bool {
+        // It's necessary to assign these checks to intermediate variables to avoid a deadlock.
+        //
+        // See: https://github.com/sigp/lighthouse/pull/2230#discussion_r620013993
+        let attested = self
+            .observed_attesters
+            .read()
+            .index_seen_at_epoch(validator_index, epoch);
+        let aggregated = self
+            .observed_aggregators
+            .read()
+            .index_seen_at_epoch(validator_index, epoch);
+        let produced_block = self
+            .observed_block_producers
+            .read()
+            .index_seen_at_epoch(validator_index as u64, epoch);
+
+        attested || aggregated || produced_block
+    }
 }
 
 impl<T: BeaconChainTypes> Drop for BeaconChain<T> {

beacon_node/beacon_chain/src/observed_attesters.rs

@@ -381,6 +381,16 @@ impl<T: Item, E: EthSpec> AutoPruningEpochContainer<T, E> {
     pub(crate) fn get_lowest_permissible(&self) -> Epoch {
         self.lowest_permissible_epoch
     }
+
+    /// Returns `true` if the given `index` has been stored in `self` at `epoch`.
+    ///
+    /// This is useful for doppelganger detection.
+    pub fn index_seen_at_epoch(&self, index: usize, epoch: Epoch) -> bool {
+        self.items
+            .get(&epoch)
+            .map(|item| item.contains(index))
+            .unwrap_or(false)
+    }
 }
 
 /// A container that stores some number of `V` items.

beacon_node/beacon_chain/src/observed_block_producers.rs

@@ -3,7 +3,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
-use types::{BeaconBlockRef, EthSpec, Slot, Unsigned};
+use types::{BeaconBlockRef, Epoch, EthSpec, Slot, Unsigned};
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -114,6 +114,15 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
         self.finalized_slot = finalized_slot;
         self.items.retain(|slot, _set| *slot > finalized_slot);
     }
+
+    /// Returns `true` if the given `validator_index` has been stored in `self` at `epoch`.
+    ///
+    /// This is useful for doppelganger detection.
+    pub fn index_seen_at_epoch(&self, validator_index: u64, epoch: Epoch) -> bool {
+        self.items.iter().any(|(slot, producers)| {
+            slot.epoch(E::slots_per_epoch()) == epoch && producers.contains(&validator_index)
+        })
+    }
 }
 
 #[cfg(test)]

beacon_node/http_api/src/lib.rs

@@ -1907,6 +1907,49 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
+    // POST lighthouse/liveness
+    let post_lighthouse_liveness = warp::path("lighthouse")
+        .and(warp::path("liveness"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(chain_filter.clone())
+        .and_then(
+            |request_data: api_types::LivenessRequestData, chain: Arc<BeaconChain<T>>| {
+                blocking_json_task(move || {
+                    // Ensure the request is for either the current, previous or next epoch.
+                    let current_epoch = chain
+                        .epoch()
+                        .map_err(warp_utils::reject::beacon_chain_error)?;
+                    let prev_epoch = current_epoch.saturating_sub(Epoch::new(1));
+                    let next_epoch = current_epoch.saturating_add(Epoch::new(1));
+
+                    if request_data.epoch < prev_epoch || request_data.epoch > next_epoch {
+                        return Err(warp_utils::reject::custom_bad_request(format!(
+                            "request epoch {} is more than one epoch from the current epoch {}",
+                            request_data.epoch, current_epoch
+                        )));
+                    }
+
+                    let liveness: Vec<api_types::LivenessResponseData> = request_data
+                        .indices
+                        .iter()
+                        .cloned()
+                        .map(|index| {
+                            let is_live =
+                                chain.validator_seen_at_epoch(index as usize, request_data.epoch);
+                            api_types::LivenessResponseData {
+                                index: index as u64,
+                                epoch: request_data.epoch,
+                                is_live,
+                            }
+                        })
+                        .collect();
+
+                    Ok(api_types::GenericResponse::from(liveness))
+                })
+            },
+        );
+
     // GET lighthouse/health
     let get_lighthouse_health = warp::path("lighthouse")
         .and(warp::path("health"))
@@ -2249,6 +2292,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(post_beacon_pool_voluntary_exits.boxed())
                 .or(post_validator_duties_attester.boxed())
                 .or(post_validator_aggregate_and_proofs.boxed())
+                .or(post_lighthouse_liveness.boxed())
                 .or(post_validator_beacon_committee_subscriptions.boxed()),
         ))
         .recover(warp_utils::reject::handle_rejection)

beacon_node/http_api/tests/tests.rs

@@ -2149,6 +2149,71 @@ impl ApiTester {
         self
     }
 
+    pub async fn test_post_lighthouse_liveness(self) -> Self {
+        let epoch = self.chain.epoch().unwrap();
+        let head_state = self.chain.head_beacon_state().unwrap();
+        let indices = (0..head_state.validators().len())
+            .map(|i| i as u64)
+            .collect::<Vec<_>>();
+
+        // Construct the expected response
+        let expected: Vec<LivenessResponseData> = head_state
+            .validators()
+            .iter()
+            .enumerate()
+            .map(|(index, _)| LivenessResponseData {
+                index: index as u64,
+                is_live: false,
+                epoch,
+            })
+            .collect();
+
+        let result = self
+            .client
+            .post_lighthouse_liveness(indices.as_slice(), epoch)
+            .await
+            .unwrap()
+            .data;
+
+        assert_eq!(result, expected);
+
+        // Attest to the current slot
+        self.client
+            .post_beacon_pool_attestations(self.attestations.as_slice())
+            .await
+            .unwrap();
+
+        let result = self
+            .client
+            .post_lighthouse_liveness(indices.as_slice(), epoch)
+            .await
+            .unwrap()
+            .data;
+
+        let committees = head_state
+            .get_beacon_committees_at_slot(self.chain.slot().unwrap())
+            .unwrap();
+        let attesting_validators: Vec<usize> = committees
+            .into_iter()
+            .map(|committee| committee.committee.iter().cloned())
+            .flatten()
+            .collect();
+        // All attesters should now be considered live
+        let expected = expected
+            .into_iter()
+            .map(|mut a| {
+                if attesting_validators.contains(&(a.index as usize)) {
+                    a.is_live = true;
+                }
+                a
+            })
+            .collect::<Vec<_>>();
+
+        assert_eq!(result, expected);
+
+        self
+    }
+
     pub async fn test_get_events(self) -> Self {
         // Subscribe to all events
         let topics = vec![
@@ -2635,5 +2700,7 @@ async fn lighthouse_endpoints() {
         .test_get_lighthouse_beacon_states_ssz()
         .await
         .test_get_lighthouse_staking()
+        .await
+        .test_post_lighthouse_liveness()
         .await;
 }

book/src/api-lighthouse.md

@@ -329,3 +329,28 @@ curl -X GET "http://localhost:5052/lighthouse/beacon/states/0/ssz" | jq
 ```
 
 *Example omitted for brevity, the body simply contains SSZ bytes.*
+
+### `/lighthouse/liveness`
+
+POST request that checks if any of the given validators have attested in the given epoch. Returns a list
+of objects, each including the validator index, epoch, and `is_live` status of a requested validator.
+
+This endpoint is used in doppelganger detection, and will only provide accurate information for the
+current, previous, or next epoch.
+
+
+```bash
+curl -X POST "http://localhost:5052/lighthouse/liveness" -d '{"indices":["0","1"],"epoch":"1"}' -H  "content-type: application/json" | jq
+```
+
+```json
+{
+    "data": [
+        {
+            "index": "0",
+            "epoch": "1",
+            "is_live": true
+        }
+    ]
+}
+```

book/src/api-vc-endpoints.md

@@ -351,7 +351,7 @@ Typical Responses | 200
       "checksum": {
         "function": "sha256",
         "params": {
-          
+
         },
         "message": "abadc1285fd38b24a98ac586bda5b17a8f93fc1ff0778803dc32049578981236"
       },

common/eth2/src/lib.rs

@@ -83,6 +83,7 @@ impl fmt::Display for Error {
 pub struct Timeouts {
     pub attestation: Duration,
     pub attester_duties: Duration,
+    pub liveness: Duration,
     pub proposal: Duration,
     pub proposer_duties: Duration,
 }
@@ -92,6 +93,7 @@ impl Timeouts {
         Timeouts {
             attestation: timeout,
             attester_duties: timeout,
+            liveness: timeout,
             proposal: timeout,
             proposer_duties: timeout,
         }
@@ -1103,6 +1105,30 @@ impl BeaconNodeHttpClient {
             .await
     }
 
+    /// `POST lighthouse/liveness`
+    pub async fn post_lighthouse_liveness(
+        &self,
+        ids: &[u64],
+        epoch: Epoch,
+    ) -> Result<GenericResponse<Vec<LivenessResponseData>>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("liveness");
+
+        self.post_with_timeout_and_response(
+            path,
+            &LivenessRequestData {
+                indices: ids.to_vec(),
+                epoch,
+            },
+            self.timeouts.liveness,
+        )
+        .await
+    }
+
     /// `POST validator/duties/attester/{epoch}`
     pub async fn post_validator_duties_attester(
         &self,

common/eth2/src/types.rs

@@ -847,6 +847,21 @@ impl FromStr for Accept {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LivenessRequestData {
+    pub epoch: Epoch,
+    #[serde(with = "serde_utils::quoted_u64_vec")]
+    pub indices: Vec<u64>,
+}
+
+#[derive(PartialEq, Debug, Serialize, Deserialize)]
+pub struct LivenessResponseData {
+    #[serde(with = "serde_utils::quoted_u64")]
+    pub index: u64,
+    pub epoch: Epoch,
+    pub is_live: bool,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

lcli/src/main.rs

@@ -344,6 +344,13 @@ fn main() {
                               non-default.",
                         ),
                 )
+                .arg(
+                    Arg::with_name("seconds-per-slot")
+                        .long("seconds-per-slot")
+                        .value_name("SECONDS")
+                        .takes_value(true)
+                        .help("Eth2 slot time"),
+                )
                 .arg(
                     Arg::with_name("seconds-per-eth1-block")
                         .long("seconds-per-eth1-block")

lcli/src/new_testnet.rs

@@ -43,6 +43,7 @@ pub fn run<T: EthSpec>(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul
     maybe_update!("genesis-delay", genesis_delay);
     maybe_update!("eth1-id", deposit_chain_id);
     maybe_update!("eth1-id", deposit_network_id);
+    maybe_update!("seconds-per-slot", seconds_per_slot);
     maybe_update!("seconds-per-eth1-block", seconds_per_eth1_block);
 
     if let Some(v) = parse_ssz_optional(matches, "genesis-fork-version")? {

lighthouse/tests/validator_client.rs

@@ -378,3 +378,16 @@ pub fn malloc_tuning_flag() {
         // effects of it.
         .run();
 }
+#[test]
+fn doppelganger_protection_flag() {
+    CommandLineTest::new()
+        .flag("enable-doppelganger-protection", None)
+        .run()
+        .with_config(|config| assert!(config.enable_doppelganger_protection));
+}
+#[test]
+fn no_doppelganger_protection_flag() {
+    CommandLineTest::new()
+        .run()
+        .with_config(|config| assert!(!config.enable_doppelganger_protection));
+}